/*
   Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License, version 2.0,
   as published by the Free Software Foundation.

   This program is also distributed with certain software (including
   but not limited to OpenSSL) that is licensed under separate terms,
   as designated in a particular file or component or in included license
   documentation.  The authors of MySQL hereby grant you an additional
   permission to link the program and your derivative works with the
   separately licensed software that they have included with MySQL.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License, version 2.0, for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
*/

#ifdef _WIN32
#define DEFAULT_PREFIX "c:/atrt"
#endif

#include <NdbAutoPtr.hpp>
#include <NdbOut.hpp>
#include "atrt.hpp"
#include "test_execution_resources.hpp"

#include <util/File.hpp>
#include <FileLogHandler.hpp>
#include <SysLogHandler.hpp>

#include <NdbSleep.h>
#include "my_alloc.h"  // MEM_ROOT
#include <ndb_version.h>
#include <vector>
#include <ndb_version.h>
#include "typelib.h"

#define PATH_SEPARATOR DIR_SEPARATOR
#define TESTCASE_RETRIES_THRESHOLD_WARNING 5
#define ATRT_VERSION_NUMBER 5

/** Global variables */
static const char progname[] = "ndb_atrt";
static const char *g_gather_progname = 0;
static const char *g_analyze_progname = 0;
static const char *g_setup_progname = 0;

static const char *g_log_filename = 0;
static const char *g_test_case_filename = 0;
static const char *g_report_filename = 0;

static int g_do_setup = 0;
static int g_do_deploy = 0;
static int g_do_sshx = 0;
static int g_do_start = 0;
static int g_do_quit = 0;

static int g_help = 0;
static int g_verbosity = 1;
static FILE *g_report_file = 0;
static FILE *g_test_case_file = stdin;
static int g_mode = 0;

Logger g_logger;
atrt_config g_config;
const char *g_user = 0;
int g_baseport = 10000;
int g_fqpn = 0;
int g_fix_nodeid = 0;
int g_default_ports = 0;
int g_mt = 0;
int g_mt_rr = 0;
int g_restart = 0;
int g_default_max_retries = 0;
static int g_default_force_cluster_restart = 0;
FailureMode g_default_behaviour_on_failure = Restart;
const char *default_behaviour_on_failure[] = {"Restart", "Abort", "Skip",
                                              "Continue", NullS};
TYPELIB behaviour_typelib = {array_elements(default_behaviour_on_failure) - 1,
                             "default_behaviour_on_failure",
                             default_behaviour_on_failure, NULL};

const char *g_cwd = 0;
const char *g_basedir = 0;
const char *g_my_cnf = 0;
const char *g_prefix = NULL;
const char *g_prefix0 = NULL;
const char *g_prefix1 = NULL;
const char *g_clusters = 0;
const char *g_config_type = NULL;  // "cnf" or "ini"
const char *g_site = NULL;
BaseString g_replicate;
const char *save_file = 0;
const char *save_group_suffix = 0;
const char *g_dummy;
char *g_env_path = 0;
const char *g_mysqld_host = 0;

TestExecutionResources g_resources;

static BaseString get_atrt_path(const char *arg);

const char *g_search_path[] = {"bin", "libexec",   "sbin", "scripts",
                               "lib", "lib/mysql", 0};
static bool find_scripts(const char *path);
static bool find_config_ini_files();

static struct my_option g_options[] = {
    {"help", '?', "Display this help and exit.", (uchar **)&g_help,
     (uchar **)&g_help, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
    {"version", 'V', "Output version information and exit.", 0, 0, 0,
     GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
    {"site", 256, "Site", (uchar **)&g_site, (uchar **)&g_site, 0, GET_STR,
     REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"clusters", 256, "Cluster", (uchar **)&g_clusters, (uchar **)&g_clusters,
     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"config-type", 256, "cnf (default) or ini", (uchar **)&g_config_type,
     (uchar **)&g_config_type, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"mysqld", 256, "atrt mysqld", (uchar **)&g_mysqld_host,
     (uchar **)&g_mysqld_host, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"replicate", 1024, "replicate", (uchar **)&g_dummy, (uchar **)&g_dummy, 0,
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"log-file", 256, "log-file", (uchar **)&g_log_filename,
     (uchar **)&g_log_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"testcase-file", 'f', "testcase-file", (uchar **)&g_test_case_filename,
     (uchar **)&g_test_case_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0,
     0},
    {"report-file", 'r', "report-file", (uchar **)&g_report_filename,
     (uchar **)&g_report_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"basedir", 256, "Base path", (uchar **)&g_basedir, (uchar **)&g_basedir, 0,
     GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"baseport", 256, "Base port", (uchar **)&g_baseport, (uchar **)&g_baseport,
     0, GET_INT, REQUIRED_ARG, g_baseport, 0, 0, 0, 0, 0},
    {"prefix", 256, "atrt install dir", (uchar **)&g_prefix,
     (uchar **)&g_prefix, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"prefix0", 256, "mysql install dir", (uchar **)&g_prefix0,
     (uchar **)&g_prefix0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"prefix1", 256, "mysql install dir 1", (uchar **)&g_prefix1,
     (uchar **)&g_prefix1, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
    {"verbose", 'v', "Verbosity", (uchar **)&g_verbosity,
     (uchar **)&g_verbosity, 0, GET_INT, REQUIRED_ARG, g_verbosity, 0, 0, 0, 0,
     0},
    {"configure", 256, "configure", (uchar **)&g_do_setup,
     (uchar **)&g_do_setup, 0, GET_INT, REQUIRED_ARG, g_do_setup, 0, 0, 0, 0,
     0},
    {"deploy", 256, "deploy", (uchar **)&g_do_deploy, (uchar **)&g_do_deploy, 0,
     GET_INT, REQUIRED_ARG, g_do_deploy, 0, 0, 0, 0, 0},
    {"sshx", 256, "sshx", (uchar **)&g_do_sshx, (uchar **)&g_do_sshx, 0,
     GET_INT, REQUIRED_ARG, g_do_sshx, 0, 0, 0, 0, 0},
    {"start", 256, "start", (uchar **)&g_do_start, (uchar **)&g_do_start, 0,
     GET_INT, REQUIRED_ARG, g_do_start, 0, 0, 0, 0, 0},
    {"fqpn", 256, "Fully qualified path-names ", (uchar **)&g_fqpn,
     (uchar **)&g_fqpn, 0, GET_INT, REQUIRED_ARG, g_fqpn, 0, 0, 0, 0, 0},
    {"fix-nodeid", 256, "Fix nodeid for each started process ",
     (uchar **)&g_fix_nodeid, (uchar **)&g_fix_nodeid, 0, GET_INT, REQUIRED_ARG,
     g_fqpn, 0, 0, 0, 0, 0},
    {"default-ports", 256, "Use default ports when possible",
     (uchar **)&g_default_ports, (uchar **)&g_default_ports, 0, GET_INT,
     REQUIRED_ARG, g_default_ports, 0, 0, 0, 0, 0},
    {"mode", 256, "Mode 0=interactive 1=regression 2=bench", (uchar **)&g_mode,
     (uchar **)&g_mode, 0, GET_INT, REQUIRED_ARG, g_mode, 0, 0, 0, 0, 0},
    {"quit", 256, "Quit before starting tests", (uchar **)&g_do_quit,
     (uchar **)&g_do_quit, 0, GET_BOOL, NO_ARG, g_do_quit, 0, 0, 0, 0, 0},
    {"mt", 256, "Use ndbmtd (0 = never, 1 = round-robin, 2 = only)",
     (uchar **)&g_mt, (uchar **)&g_mt, 0, GET_INT, REQUIRED_ARG, g_mt, 0, 0, 0,
     0, 0},
    {"default-max-retries", 256,
     "default number of retries after a test case fails (can be overwritten in "
     "the test suite file)",
     (uchar **)&g_default_max_retries, (uchar **)&g_default_max_retries, 0,
     GET_INT, REQUIRED_ARG, g_default_max_retries, 0, 0, 0, 0, 0},
    {"default-force-cluster-restart", 0,
     "Force cluster to restart for each testrun (can be overwritten in test "
     "suite file)",
     (uchar **)&g_default_force_cluster_restart,
     (uchar **)&g_default_force_cluster_restart, 0, GET_BOOL, NO_ARG,
     g_default_force_cluster_restart, 0, 0, 0, 0, 0},
    {"default-behaviour-on-failure", 256, "default to do when a test fails",
     (uchar **)&g_default_behaviour_on_failure,
     (uchar **)&g_default_behaviour_on_failure, &behaviour_typelib, GET_ENUM,
     REQUIRED_ARG, g_default_behaviour_on_failure, 0, 0, 0, 0, 0},
    {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}};

const int p_ndb = atrt_process::AP_NDB_MGMD | atrt_process::AP_NDBD;
const int p_servers = atrt_process::AP_MYSQLD | atrt_process::AP_CUSTOM;
const int p_clients = atrt_process::AP_CLIENT | atrt_process::AP_NDB_API;

static int check_testcase_file_main(int argc, char **argv);
static void print_testcase_file_syntax();

int main(int argc, char **argv) {
  ndb_init();

  AtrtExitCodes return_code = AtrtExitCodes::TESTSUITE_SUCCESS;

  g_logger.setCategory(progname);
  g_logger.enable(Logger::LL_ALL);
  g_logger.createConsoleHandler();

  // If program is called with --check-testcase-files as first option
  // it is assumed that the rest of command line arguments are
  // testcase-filenames and those files will be syntax checked.
  if (argc >= 2 && strcmp(argv[1], "--check-testcase-files") == 0) {
    exit(check_testcase_file_main(argc, argv));
  }

  MEM_ROOT alloc = MEM_ROOT{PSI_NOT_INSTRUMENTED, 512};
  if (!parse_args(argc, argv, &alloc)) {
    g_logger.critical("Failed to parse arguments");
    return atrt_exit(ATRT_FAILURE);
  }

  g_logger.info("Starting ATRT version : %s", getAtrtVersion().c_str());

  if (g_mt != 0) {
    g_resources.setRequired(g_resources.NDBMTD);
  }

  {
    std::vector<std::string> error;
    std::vector<std::string> info;
    if (!g_resources.loadPaths(g_prefix0, g_prefix1, &error, &info)) {
      g_logger.critical("Failed to find required binaries for execution");

      for (auto msg : error) {
        g_logger.critical("%s", msg.c_str());
      }
      return atrt_exit(ATRT_FAILURE);
    }

    for (auto msg : info) {
      g_logger.info("%s", msg.c_str());
    }
  }

  {
    BaseString atrt_path = get_atrt_path(argv[0]);
    assert(atrt_path != "");

    if (!find_scripts(atrt_path.c_str())) {
      g_logger.critical("Failed to find required atrt scripts for execution");
      return atrt_exit(ATRT_FAILURE);
    }
  }

  g_config.m_config_type = atrt_config::CNF;
  if (g_config_type != NULL && strcmp(g_config_type, "ini") == 0) {
    g_logger.info("Using config.ini for cluster configuration");
    g_config.m_config_type = atrt_config::INI;

    if (!find_config_ini_files()) {
      g_logger.critical("Failed to find required config.ini files");
      return atrt_exit(ATRT_FAILURE);
    }
  }

  g_config.m_generated = false;
  g_config.m_replication = g_replicate;
  if (!setup_config(g_config, g_mysqld_host)) {
    g_logger.critical("Failed to setup configuration");
    return atrt_exit(ATRT_FAILURE);
  }

  if (!g_config.m_processes.size()) {
    g_logger.critical("Error: No processes defined in cluster configuration");
    return atrt_exit(ATRT_FAILURE);
  }

  if (!configure(g_config, g_do_setup)) {
    g_logger.critical("Failed to configure");
    return atrt_exit(ATRT_FAILURE);
  }

  g_logger.info("Setting up directories...");
  if (!setup_directories(g_config, g_do_setup)) {
    g_logger.critical("Failed to set up directories");
    return atrt_exit(ATRT_FAILURE);
  }

  if (g_do_setup) {
    g_logger.info("Setting up files...");
    if (!setup_files(g_config, g_do_setup, g_do_sshx)) {
      g_logger.critical("Failed to set up files");
      return atrt_exit(ATRT_FAILURE);
    }
  }

  if (g_do_deploy) {
    g_logger.info("Deploying files...");
    if (!deploy(g_do_deploy, g_config)) {
      g_logger.critical("Failed to deploy");
      return atrt_exit(ATRT_FAILURE);
    }
  }

  if (g_do_quit) {
    return atrt_exit(TESTSUITE_SUCCESS);
  }

  if (!setup_hosts(g_config)) {
    g_logger.critical("Failed to setup hosts");
    return atrt_exit(ATRT_FAILURE);
  }

  if (g_do_sshx) {
    g_logger.info("Starting xterm-ssh");
    if (!sshx(g_config, g_do_sshx)) {
      g_logger.critical("Failed to start xterm-ssh");
      return atrt_exit(ATRT_FAILURE);
    }

    g_logger.info("Done...sleeping");
    while (true) {
      if (!do_command(g_config)) {
        g_logger.critical("Failed to do ssh command");
        return atrt_exit(ATRT_FAILURE);
      }

      NdbSleep_SecSleep(1);
    }
    return atrt_exit(TESTSUITE_SUCCESS);
  }

  /**
   * contact each ndb_cpcd
   */
  g_logger.info("Connecting to hosts...");
  if (!connect_hosts(g_config)) {
    g_logger.critical("Failed to connect to CPCD on hosts");
    return atrt_exit(ATRT_FAILURE);
  }

  /**
   * Collect all the testcases
   */
  std::vector<atrt_testcase> testcases;
  if (!read_test_cases(g_test_case_file, &testcases)) {
    g_logger.critical("Failed to read all the testcases");
    return atrt_exit(ATRT_FAILURE);
  }

  /**
   * Start the cluster
   */
  g_logger.info("Starting server processes...");

  if (!start_clusters(g_config)) {
    shutdown_processes(g_config, atrt_process::AP_ALL);
    return atrt_exit(ATRT_FAILURE);
  }

  if (!check_cluster_status(g_config, atrt_process::AP_ALL)) {
    g_logger.critical("Cluster start up failed(%d)", ERR_CRITICAL);
    shutdown_processes(g_config, atrt_process::AP_ALL);
    return atrt_exit(ATRT_FAILURE);
  }

  g_logger.info("All servers start completed");

  /**
   * Run all tests
   */

  g_logger.debug("Entering main loop");
  FailureMode current_failure_mode = FailureMode::Continue;
  for (auto testcase : testcases) {
    g_logger.info("#%d - %s", testcase.test_no, testcase.m_name.c_str());

    TestResult test_result;
    if (current_failure_mode == FailureMode::Skip) {
      test_result = {0, 0, ERR_TEST_SKIPPED};
    } else {
      test_result = run_test_case(testcase);
      if (test_result.result != ErrorCodes::ERR_OK) {
        current_failure_mode = testcase.m_behaviour_on_failure;
      }
    }
    update_atrt_result_code(test_result, &return_code);

    if (g_report_file != 0) {
      fprintf(g_report_file, "%s ; %d ; %d ; %ld ; %d\n",
              testcase.m_name.c_str(), testcase.test_no, test_result.result,
              test_result.elapsed, test_result.testruns);
      fflush(g_report_file);
    }

    if (g_mode == 0 && test_result.result != ERR_OK) {
      g_logger.info("Encountered failed test in interactive mode");
    }

    const char *test_status = get_test_status(test_result.result);
    g_logger.info("#%d %s(%d)", testcase.test_no, test_status,
                  test_result.result);

    if (current_failure_mode == FailureMode::Abort) {
      g_logger.info("Aborting the test suite execution!");
      break;
    }
  }

  if (g_report_file != 0) {
    fclose(g_report_file);
    g_report_file = 0;
  }

  g_logger.info("Finishing, result: %d", return_code);
  return return_code;
}

extern "C" bool get_one_option(int arg, const struct my_option *opt,
                               char *value) {
  if (arg == 1024) {
    if (g_replicate.length()) g_replicate.append(";");
    g_replicate.append(value);
    return 0;
  }
  return 0;
}

bool parse_args(int argc, char **argv, MEM_ROOT *alloc) {
  bool fail_after_help = false;
  char buf[2048];

  if (argc >= 2 &&
      (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)) {
    ndbout << getAtrtVersion().c_str() << endl;
    exit(0);
  }

  if (getcwd(buf, sizeof(buf)) == 0) {
    g_logger.error("Unable to get current working directory");
    return false;
  }
  g_cwd = strdup(buf);

  struct stat sbuf;
  BaseString mycnf;
  mycnf.append(g_cwd);
  mycnf.append(DIR_SEPARATOR);

  if (argc > 1 && lstat(argv[argc - 1], &sbuf) == 0) {
    mycnf.append(argv[argc - 1]);
  } else {
    mycnf.append("my.cnf");
    if (lstat(mycnf.c_str(), &sbuf) != 0) {
      g_logger.error(
          "Could not find out which config file to use! "
          "Pass it as last argument to atrt: 'atrt <config file>' "
          "(default: '%s')",
          mycnf.c_str());
      fail_after_help = true;
    }
  }

  to_fwd_slashes((char *)g_cwd);

  g_logger.info("Bootstrapping using %s", mycnf.c_str());

  const char *groups[] = {"atrt", 0};
  int ret = load_defaults(mycnf.c_str(), groups, &argc, &argv, alloc);

  if (ret) {
    g_logger.error("Failed to load defaults, returned (%d)", ret);
    return false;
  }

  save_file = my_defaults_file;
  save_group_suffix = my_defaults_group_suffix;

  if (my_defaults_extra_file) {
    g_logger.error("--defaults-extra-file(%s) is not supported...",
                   my_defaults_extra_file);
    return false;
  }

  ret = handle_options(&argc, &argv, g_options, get_one_option);
  if (ret) {
    g_logger.error("handle_options failed, ret: %d, argc: %d, *argv: '%s'", ret,
                   argc, *argv);
    return false;
  }

  if (argc >= 2) {
    const char *arg = argv[argc - 2];
    while (*arg) {
      switch (*arg) {
        case 'c':
          g_do_setup = (g_do_setup == 0) ? 1 : g_do_setup;
          break;
        case 'C':
          g_do_setup = 2;
          break;
        case 'd':
          g_do_deploy = 3;
          break;
        case 'D':
          g_do_deploy = 2;  // only binaries
          break;
        case 'x':
          g_do_sshx = atrt_process::AP_CLIENT | atrt_process::AP_NDB_API;
          break;
        case 'X':
          g_do_sshx = atrt_process::AP_ALL;
          break;
        case 's':
          g_do_start = p_ndb;
          break;
        case 'S':
          g_do_start = p_ndb | p_servers;
          break;
        case 'f':
          g_fqpn = 1;
          break;
        case 'z':
          g_fix_nodeid = 1;
          break;
        case 'q':
          g_do_quit = 1;
          break;
        case 'r':
          g_restart = 1;
          break;
        default:
          g_logger.error("Unknown switch '%c'", *arg);
          return false;
      }
      arg++;
    }
  }

  if (g_log_filename != 0) {
    g_logger.removeConsoleHandler();
    g_logger.addHandler(new FileLogHandler(g_log_filename));
  }

  {
    int tmp = Logger::LL_WARNING - g_verbosity;
    tmp = (tmp < Logger::LL_DEBUG ? Logger::LL_DEBUG : tmp);
    g_logger.disable(Logger::LL_ALL);
    g_logger.enable(Logger::LL_ON);
    g_logger.enable((Logger::LoggerLevel)tmp, Logger::LL_ALERT);
  }

  if (!g_basedir) {
    g_basedir = g_cwd;
    g_logger.info("basedir not specified, using %s", g_basedir);
  } else {
    g_logger.info("basedir, %s", g_basedir);
  }

  const char *default_prefix;
  if (g_prefix != NULL) {
    default_prefix = g_prefix;
  } else if (g_prefix0 != NULL) {
    default_prefix = g_prefix0;
  } else {
    default_prefix = DEFAULT_PREFIX;
  }

  if (g_prefix == NULL) {
    g_prefix = DEFAULT_PREFIX;
  }

  if (g_prefix0 == NULL) {
    g_prefix0 = DEFAULT_PREFIX;
  }

  /**
   * Add path to atrt-*.sh
   */
  {
    BaseString tmp;
    const char *env = getenv("PATH");
    if (env && strlen(env)) {
      tmp.assfmt("PATH=%s:%s/mysql-test/ndb", env, g_prefix);
    } else {
      tmp.assfmt("PATH=%s/mysql-test/ndb", g_prefix);
    }
    to_native(tmp);
    g_env_path = strdup(tmp.c_str());
    putenv(g_env_path);
  }

  if (g_help) {
    my_print_help(g_options);
    my_print_variables(g_options);
    print_testcase_file_syntax();
    return 0;
  }
  if (fail_after_help) {
    return false;
  }

  if (g_test_case_filename) {
    g_test_case_file = fopen(g_test_case_filename, "r");
    if (g_test_case_file == 0) {
      g_logger.critical("Unable to open file: %s", g_test_case_filename);
      return false;
    }
    if (g_do_setup == 0) g_do_setup = 2;

    if (g_do_start == 0) g_do_start = p_ndb | p_servers;

    if (g_mode == 0) g_mode = 1;

    if (g_do_sshx) {
      g_logger.critical("ssx specified...not possible with testfile");
      return false;
    }
  } else {
    g_logger.info(
        "No test case file given with -f <test file>, "
        "running in interactive mode from stdin");
  }

  if (g_do_setup == 0) {
    BaseString tmp;
    tmp.append(g_basedir);
    tmp.append(PATH_SEPARATOR);
    tmp.append("my.cnf");
    if (lstat(tmp.c_str(), &sbuf) != 0) {
      g_logger.error(
          "Could not find a my.cnf file in the basedir '%s', "
          "you probably need to configure it with "
          "'atrt --configure=1 <config_file>'",
          g_basedir);
      return false;
    }

    if (!S_ISREG(sbuf.st_mode)) {
      g_logger.error("%s is not a regular file", tmp.c_str());
      return false;
    }

    g_my_cnf = strdup(tmp.c_str());
    g_logger.info("Using %s", tmp.c_str());
  } else {
    g_my_cnf = strdup(mycnf.c_str());
  }

  if (g_prefix1) {
    g_logger.info("Using --prefix1=\"%s\"", g_prefix1);
  }

  if (g_report_filename) {
    g_report_file = fopen(g_report_filename, "w");
    if (g_report_file == 0) {
      g_logger.critical("Unable to create report file: %s", g_report_filename);
      return false;
    }
  }

  if (g_clusters == 0) {
    g_logger.critical("No clusters specified");
    return false;
  }

  /* Read username from environment, default to sakila */
  g_user = strdup(getenv("LOGNAME"));
  if (g_user == 0) {
    g_user = "sakila";
    g_logger.info("No default user specified, will use 'sakila'.");
    g_logger.info("Please set LOGNAME environment variable for other username");
  }

  return true;
}

std::string getAtrtVersion() {
  int mysql_version = ndbGetOwnVersion();
  std::string version = std::to_string(ndbGetMajor(mysql_version)) + "." +
                        std::to_string(ndbGetMinor(mysql_version)) + "." +
                        std::to_string(ndbGetBuild(mysql_version)) + "." +
                        std::to_string(ATRT_VERSION_NUMBER);
  return version;
}

bool connect_hosts(atrt_config &config) {
  for (unsigned i = 0; i < config.m_hosts.size(); i++) {
    if (config.m_hosts[i]->m_hostname.length() == 0) continue;

    if (config.m_hosts[i]->m_cpcd->connect() != 0) {
      g_logger.error("Unable to connect to cpc %s:%d",
                     config.m_hosts[i]->m_cpcd->getHost(),
                     config.m_hosts[i]->m_cpcd->getPort());
      return false;
    }
    g_logger.debug("Connected to %s:%d", config.m_hosts[i]->m_cpcd->getHost(),
                   config.m_hosts[i]->m_cpcd->getPort());
  }

  return true;
}

bool connect_ndb_mgm(atrt_process &proc) {
  NdbMgmHandle handle = ndb_mgm_create_handle();
  if (handle == 0) {
    g_logger.critical("Unable to create mgm handle");
    return false;
  }
  BaseString tmp = proc.m_host->m_hostname;
  const char *val;
  proc.m_options.m_loaded.get("--PortNumber=", &val);
  tmp.appfmt(":%s", val);

  if (ndb_mgm_set_connectstring(handle, tmp.c_str())) {
    g_logger.critical("Unable to create parse connectstring");
    return false;
  }

  if (ndb_mgm_connect(handle, 30, 1, 0) != -1) {
    proc.m_ndb_mgm_handle = handle;
    return true;
  }

  g_logger.critical("Unable to connect to ndb mgm %s", tmp.c_str());
  return false;
}

bool connect_ndb_mgm(atrt_config &config) {
  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    if ((proc.m_type & atrt_process::AP_NDB_MGMD) != 0) {
      if (!connect_ndb_mgm(proc)) {
        return false;
      }
    }
  }

  return true;
}

static int remap(int i) {
  if (i == NDB_MGM_NODE_STATUS_NO_CONTACT) return NDB_MGM_NODE_STATUS_UNKNOWN;
  if (i == NDB_MGM_NODE_STATUS_UNKNOWN) return NDB_MGM_NODE_STATUS_NO_CONTACT;
  return i;
}

bool wait_ndb(atrt_config &config, int goal) {
  goal = remap(goal);

  size_t cnt = 0;
  for (unsigned i = 0; i < config.m_clusters.size(); i++) {
    atrt_cluster *cluster = config.m_clusters[i];

    if (strcmp(cluster->m_name.c_str(), ".atrt") == 0) {
      /**
       * skip atrt mysql
       */
      cnt++;
      continue;
    }

    /**
     * Get mgm handle for cluster
     */
    NdbMgmHandle handle = 0;
    for (unsigned j = 0; j < cluster->m_processes.size(); j++) {
      atrt_process &proc = *cluster->m_processes[j];
      if ((proc.m_type & atrt_process::AP_NDB_MGMD) != 0) {
        handle = proc.m_ndb_mgm_handle;
        break;
      }
    }

    if (handle == 0) {
      return true;
    }

    if (goal == NDB_MGM_NODE_STATUS_STARTED) {
      /**
       * 1) wait NOT_STARTED
       * 2) send start
       * 3) wait STARTED
       */
      if (!wait_ndb(config, NDB_MGM_NODE_STATUS_NOT_STARTED)) return false;

      ndb_mgm_start(handle, 0, 0);
    }

    struct ndb_mgm_cluster_state *state;

    time_t now = time(0);
    time_t end = now + 360;
    int min = remap(NDB_MGM_NODE_STATUS_NO_CONTACT);
    int min2 = goal;

    while (now < end) {
      /**
       * 1) retreive current state
       */
      state = 0;
      do {
        state = ndb_mgm_get_status(handle);
        if (state == 0) {
          const int err = ndb_mgm_get_latest_error(handle);
          g_logger.error("Unable to poll db state: %d %s %s",
                         ndb_mgm_get_latest_error(handle),
                         ndb_mgm_get_latest_error_msg(handle),
                         ndb_mgm_get_latest_error_desc(handle));
          if (err == NDB_MGM_SERVER_NOT_CONNECTED && connect_ndb_mgm(config)) {
            g_logger.error("Reconnected...");
            continue;
          }
          return false;
        }
      } while (state == 0);
      NdbAutoPtr<void> tmp(state);

      min2 = goal;
      for (int j = 0; j < state->no_of_nodes; j++) {
        if (state->node_states[j].node_type == NDB_MGM_NODE_TYPE_NDB) {
          const int s = remap(state->node_states[j].node_status);
          min2 = (min2 < s ? min2 : s);

          if (s < remap(NDB_MGM_NODE_STATUS_NO_CONTACT) ||
              s > NDB_MGM_NODE_STATUS_STARTED) {
            g_logger.critical("Strange DB status during start: %d %d", j, min2);
            return false;
          }

          if (min2 < min) {
            g_logger.critical("wait ndb failed node: %d %d %d %d",
                              state->node_states[j].node_id, min, min2, goal);
          }
        }
      }

      if (min2 < min) {
        g_logger.critical("wait ndb failed %d %d %d", min, min2, goal);
        return false;
      }

      if (min2 == goal) {
        cnt++;
        goto next;
      }

      min = min2;
      now = time(0);
    }

    g_logger.critical("wait ndb timed out %d %d %d", min, min2, goal);
    break;

  next:;
  }

  return cnt == config.m_clusters.size();
}

bool start_process(atrt_process &proc, bool run_setup) {
  if (proc.m_proc.m_id != -1) {
    g_logger.critical("starting already started process: %u",
                      (unsigned)proc.m_index);
    return false;
  }

  if (run_setup) {
    BaseString tmp = g_setup_progname;
    tmp.appfmt(" %s %s/ %s", proc.m_host->m_hostname.c_str(),
               proc.m_proc.m_cwd.c_str(), proc.m_proc.m_cwd.c_str());

    g_logger.debug("system(%s)", tmp.c_str());
    const int r1 = sh(tmp.c_str());
    if (r1 != 0) {
      g_logger.critical("Failed to setup process");
      return false;
    }
  }

  /**
   * For MySQL server program we need to pass the correct basedir.
   */
  const bool mysqld = proc.m_type & atrt_process::AP_MYSQLD;
  if (mysqld) {
    BaseString basedir;
    /**
     * If MYSQL_BASE_DIR is set use that for basedir.
     */
    ssize_t pos = proc.m_proc.m_env.indexOf("MYSQL_BASE_DIR=");
    if (pos > 0) {
      pos = proc.m_proc.m_env.indexOf(" MYSQL_BASE_DIR=");
      if (pos != -1) pos++;
    }
    if (pos >= 0) {
      pos += strlen("MYSQL_BASE_DIR=");
      ssize_t endpos = proc.m_proc.m_env.indexOf(' ', pos);
      if (endpos == -1) endpos = proc.m_proc.m_env.length();
      basedir = proc.m_proc.m_env.substr(pos, endpos);
    } else {
      /**
       * If no MYSQL_BASE_DIR set, derive basedir from program path.
       * Assumming that program path is on the form
       *   <basedir>/{bin,sql}/mysqld
       */
      const BaseString sep("/");
      Vector<BaseString> dir_parts;
      int num_of_parts = proc.m_proc.m_path.split(dir_parts, sep);
      dir_parts.erase(num_of_parts - 1);  // remove trailing /mysqld
      dir_parts.erase(num_of_parts - 2);  // remove trailing /bin
      num_of_parts -= 2;
      basedir.assign(dir_parts, sep);
    }
    if (proc.m_proc.m_args.indexOf("--basedir=") == -1) {
      proc.m_proc.m_args.appfmt(" --basedir=%s", basedir.c_str());
      g_logger.info("appended '--basedir=%s' to mysqld process",
                    basedir.c_str());
    }
  }
  BaseString save_args(proc.m_proc.m_args);
  {
    Properties reply;
    if (proc.m_host->m_cpcd->define_process(proc.m_proc, reply) != 0) {
      BaseString msg;
      reply.get("errormessage", msg);
      g_logger.error("Unable to define process: %s", msg.c_str());
      if (mysqld) {
        proc.m_proc.m_args = save_args; /* restore args */
      }
      return false;
    }
  }
  if (mysqld) {
    proc.m_proc.m_args = save_args; /* restore args */
  }
  {
    Properties reply;
    if (proc.m_host->m_cpcd->start_process(proc.m_proc.m_id, reply) != 0) {
      BaseString msg;
      reply.get("errormessage", msg);
      g_logger.error("Unable to start process: %s", msg.c_str());
      return false;
    }
  }
  return true;
}

bool start_processes(atrt_config &config, int types) {
  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    if (IF_WIN(!(proc.m_type & atrt_process::AP_MYSQLD), 1) &&
        (types & proc.m_type) != 0 && proc.m_proc.m_path != "") {
      if (!start_process(proc)) {
        return false;
      }
    }
  }
  return true;
}

bool stop_process(atrt_process &proc) {
  if (proc.m_proc.m_id == -1) {
    return true;
  }

  if (proc.m_type == atrt_process::AP_MYSQLD) {
    disconnect_mysqld(proc);
  }

  {
    Properties reply;
    if (proc.m_host->m_cpcd->stop_process(proc.m_proc.m_id, reply) != 0) {
      Uint32 status;
      reply.get("status", &status);
      if (status != 4) {
        BaseString msg;
        reply.get("errormessage", msg);
        g_logger.error(
            "Unable to stop process id: %d host: %s cmd: %s, "
            "msg: %s, status: %d",
            proc.m_proc.m_id, proc.m_host->m_hostname.c_str(),
            proc.m_proc.m_path.c_str(), msg.c_str(), status);
        return false;
      }
    }
  }
  {
    Properties reply;
    if (proc.m_host->m_cpcd->undefine_process(proc.m_proc.m_id, reply) != 0) {
      BaseString msg;
      reply.get("errormessage", msg);
      g_logger.error("Unable to stop process id: %d host: %s cmd: %s, msg: %s",
                     proc.m_proc.m_id, proc.m_host->m_hostname.c_str(),
                     proc.m_proc.m_path.c_str(), msg.c_str());
      return false;
    }
  }

  return true;
}

bool stop_processes(atrt_config &config, int types) {
  int failures = 0;

  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    if ((types & proc.m_type) != 0) {
      if (!stop_process(proc)) {
        failures++;
      }
    }
  }
  return failures == 0;
}

bool update_status(atrt_config &config, int types, bool fail_on_missing) {
  Vector<Vector<SimpleCpcClient::Process> > m_procs;

  Vector<SimpleCpcClient::Process> dummy;
  m_procs.fill(config.m_hosts.size(), dummy);
  for (unsigned i = 0; i < config.m_hosts.size(); i++) {
    if (config.m_hosts[i]->m_hostname.length() == 0) continue;

    Properties p;
    config.m_hosts[i]->m_cpcd->list_processes(m_procs[i], p);
  }

  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];

    if (proc.m_proc.m_id == -1 || (proc.m_type & types) == 0) {
      continue;
    }

    Vector<SimpleCpcClient::Process> &h_procs = m_procs[proc.m_host->m_index];
    bool found = false;
    for (unsigned j = 0; j < h_procs.size() && !found; j++) {
      if (proc.m_proc.m_id == h_procs[j].m_id) {
        found = true;
        proc.m_proc.m_status = h_procs[j].m_status;
      }
    }

    if (found) continue;

    if (!fail_on_missing) {
      proc.m_proc.m_id = -1;
      proc.m_proc.m_status.clear();
    } else {
      g_logger.error("update_status: not found");
      g_logger.error("id: %d host: %s cmd: %s", proc.m_proc.m_id,
                     proc.m_host->m_hostname.c_str(),
                     proc.m_proc.m_path.c_str());
      for (unsigned j = 0; j < h_procs.size(); j++) {
        g_logger.error("found: %d %s", h_procs[j].m_id,
                       h_procs[j].m_path.c_str());
      }
      return false;
    }
  }
  return true;
}

int check_ndb_or_servers_failures(atrt_config &config) {
  int failed_processes = 0;
  const int types = p_ndb | p_servers;
  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    bool skip =
        proc.m_atrt_stopped || IF_WIN(proc.m_type & atrt_process::AP_MYSQLD, 0);
    bool isRunning = proc.m_proc.m_status == "running";
    if ((types & proc.m_type) != 0 && !isRunning && !skip) {
      g_logger.critical("%s #%d not running on %s", proc.m_name.c_str(),
                        proc.m_index, proc.m_host->m_hostname.c_str());
      failed_processes |= proc.m_type;
    }
  }
  if ((failed_processes & p_ndb) && (failed_processes & p_servers)) {
    return ERR_NDB_AND_SERVERS_FAILED;
  }
  if ((failed_processes & p_ndb) != 0) {
    return ERR_NDB_FAILED;
  }
  if ((failed_processes & p_servers) != 0) {
    return ERR_SERVERS_FAILED;
  }
  return 0;
}

bool is_client_running(atrt_config &config) {
  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    if ((p_clients & proc.m_type) != 0 && proc.m_proc.m_status == "running") {
      return true;
    }
  }
  return false;
}

bool wait_for_processes_to_stop(atrt_config &config, int types, int retries,
                                int wait_between_retries_s) {
  for (int attempts = 0; attempts < retries; attempts++) {
    bool last_attempt = attempts == (retries - 1);

    update_status(config, types, false);

    int found = 0;
    for (unsigned i = 0; i < config.m_processes.size(); i++) {
      atrt_process &proc = *config.m_processes[i];
      if ((types & proc.m_type) == 0 || proc.m_proc.m_id == -1) continue;

      found++;

      if (!last_attempt) continue;  // skip logging
      g_logger.error(
          "Failed to stop process id: %d host: %s status: %s cmd: %s",
          proc.m_proc.m_id, proc.m_host->m_hostname.c_str(),
          proc.m_proc.m_status.c_str(), proc.m_proc.m_path.c_str());
    }

    if (found == 0) return true;

    if (!last_attempt) NdbSleep_SecSleep(wait_between_retries_s);
  }

  return false;
}

bool wait_for_process_to_stop(atrt_config &config, atrt_process &proc,
                              int retries, int wait_between_retries_s) {
  for (int attempts = 0; attempts < retries; attempts++) {
    update_status(config, proc.m_type, false);

    if (proc.m_proc.m_id == -1) return true;

    bool last_attempt = attempts == (retries - 1);
    if (!last_attempt) {
      NdbSleep_SecSleep(wait_between_retries_s);
      continue;
    }

    g_logger.error("Failed to stop process id: %d host: %s status: %s cmd: %s",
                   proc.m_proc.m_id, proc.m_host->m_hostname.c_str(),
                   proc.m_proc.m_status.c_str(), proc.m_proc.m_path.c_str());
  }

  return false;
}

bool shutdown_processes(atrt_config &config, int types) {
  const char *p_type = get_process_type_name(types);

  g_logger.info("Stopping %s processes", p_type);

  if (!stop_processes(config, types)) {
    g_logger.critical("Failed to stop %s processes", p_type);
    return false;
  }

  if (!wait_for_processes_to_stop(config, types)) {
    g_logger.critical("Failed to stop %s processes", p_type);
    return false;
  }
  return true;
}

const char* get_process_type_name(int types) {
  switch (types) {
    case p_clients:
      return "client";
    case p_ndb:
      return "ndb";
    case p_servers:
      return "server";
    default:
      return "all";
  }
}

const char *get_test_status(int result) {
  switch (result) {
    case ErrorCodes::ERR_OK:
      return "OK";
    case ErrorCodes::ERR_TEST_SKIPPED:
      return "SKIPPED";
    case ErrorCodes::ERR_CRITICAL:
      return "CRITICAL";
  }
  return "FAILED";
}

bool check_cluster_status(atrt_config &config, int types) {
  if (!update_status(config, types)) {
    g_logger.critical("Failed to get updated status for all processes");
    return false;
  }
  if (check_ndb_or_servers_failures(config) != 0) {
    return false;
  }
  return true;
}

int atrt_exit(int return_code) {
  g_logger.info("Finishing, result: %d", return_code);
  return return_code;
}

bool setup_hosts_filesystem(atrt_config &config) {
  if (!setup_directories(config, 2)) {
    g_logger.critical("Failed to setup directories");
    return false;
  }

  if (!setup_files(config, 2, 1)) {
    g_logger.critical("Failed to setup files");
    return false;
  }

  if (!setup_hosts(config)) {
    g_logger.critical("Failed to setup hosts");
    return false;
  }

  return true;
}

bool start_clusters(atrt_config &config) {
  g_logger.debug("Setup complete, starting servers");
  if (!start(config, p_ndb | p_servers)) {
    g_logger.critical("Failed to start server processes");
    return false;
  }

  if (!setup_db(config)) {
    g_logger.critical("Failed to setup database");
    return false;
  }

  return true;
}

bool read_test_cases(FILE *file, std::vector<atrt_testcase> *testcases) {
  int lineno = 1;
  int test_no = 1;
  while (!feof(file)) {
    atrt_testcase testcase;
    const int num_element_lines = read_test_case(file, lineno, testcase);
    if (num_element_lines == 0) {
      continue;
    }
    if (num_element_lines == ERR_CORRUPT_TESTCASE) {
      g_logger.critical("Corrupted testcase at line %d (error %d)", lineno,
                        num_element_lines);
      return false;
    }
    testcase.test_no = test_no++;
    testcases->push_back(std::move(testcase));
  }

  if (file != stdin) {
    fclose(file);
  }

  return true;
}

TestResult run_test_case(const atrt_testcase &testcase) {
  TestResult test_result = {0, 0, 0};
  for (; test_result.testruns <= testcase.m_max_retries;
       test_result.testruns++) {
    if (test_result.testruns > 0) {
      if (test_result.result == ERR_OK ||
          test_result.result == ERR_TEST_SKIPPED) {
        break;
      }
      g_logger.info("Retrying #%d - %s (%d/%d)...", testcase.test_no,
                    testcase.m_name.c_str(), test_result.testruns,
                    testcase.m_max_retries);
    }

    if (testcase.m_force_cluster_restart || test_result.result != ERR_OK) {
      if (!shutdown_processes(g_config, ~0)) {
        g_logger.critical("Failed to stop all processes");
        test_result.result = ERR_CRITICAL;
        continue;  // attempt test retry
      }

      if (!setup_hosts_filesystem(g_config)) {
        test_result.result = ERR_CRITICAL;
        continue;
      }

      g_logger.info("(Re)starting server processes...");
      if (!start_clusters(g_config)) {
        g_logger.critical("Cluster could not be started");
        test_result.result = ERR_CRITICAL;
        continue;
      }

      g_logger.info("All servers start completed");
    }

    if (!check_cluster_status(g_config, atrt_process::AP_ALL)) {
      g_logger.critical("Cluster processes failed before test starts");
      test_result.result = ERR_CRITICAL;
      continue;
    }

    {
      // Assign processes to programs
      if (!setup_test_case(g_config, testcase)) {
        g_logger.critical("Failed to setup test case");
        test_result.result = ERR_CRITICAL;
        continue;
      }

      if (!start_processes(g_config, p_clients)) {
        g_logger.critical("Failed to start client processes");
        test_result.result = ERR_CRITICAL;
        continue;
      }

      const time_t start = time(0);
      time_t now = start;
      do {
        if (!update_status(g_config, atrt_process::AP_ALL)) {
          g_logger.critical("Failed to get updated status for all processes");
          test_result.result = ERR_CRITICAL;
          break;
        }

        test_result.result = check_ndb_or_servers_failures(g_config);
        if (test_result.result) {
          break;
        }

        if (!is_client_running(g_config)) {
          break;
        }

        if (!do_command(g_config)) {
          test_result.result = ERR_COMMAND_FAILED;
          g_logger.critical("Failure on client command execution");
          break;
        }

        now = time(0);
        if (now > (start + testcase.m_max_time)) {
          g_logger.info("Timeout '%s' after %ld seconds",
                        testcase.m_name.c_str(), testcase.m_max_time);
          test_result.result = ERR_MAX_TIME_ELAPSED;
          break;
        }
        NdbSleep_SecSleep(1);
      } while (true);

      test_result.elapsed = time(0) - start;
      if (!shutdown_processes(g_config, p_clients)) {
        g_logger.critical("Failed to stop client processes");
        test_result.result = ERR_CRITICAL;
        continue;  // retry test due to failure
      }
    }
  }

  int tmp, *rp = test_result.result ? &tmp : &test_result.result;
  if (!gather_result(g_config, rp)) {
    g_logger.critical("Failed to gather result after test run");
    test_result.result = ERR_CRITICAL;
  }

  BaseString res_dir;
  res_dir.assfmt("result.%d", testcase.test_no);
  remove_dir(res_dir.c_str(), true);

  if (testcase.m_report || test_result.result != ERR_OK) {
    if (rename("result", res_dir.c_str()) != 0) {
      g_logger.critical("Failed to rename %s as %s", "result", res_dir.c_str());
      test_result.result = ERR_CRITICAL;
    }
  } else {
    remove_dir("result", true);
  }

  bool stop_cluster = reset_config(g_config) ||
                      (test_result.result != ERR_OK &&
                       testcase.m_behaviour_on_failure == FailureMode::Restart);
  if (stop_cluster) {
    if (!shutdown_processes(g_config, ~0)) {
      g_logger.critical("Failed to stop all processes");
      test_result.result = ERR_CRITICAL;
    }

    if (!setup_hosts_filesystem(g_config)) {
      test_result.result = ERR_CRITICAL;
    }

    g_logger.info("Restarting cluster processes...");
    if (!start_clusters(g_config)) {
      g_logger.critical("Cluster could not be started");
      test_result.result = ERR_CRITICAL;
    }
  }

  if (!check_cluster_status(g_config, atrt_process::AP_ALL)) {
    test_result.result = ERR_CRITICAL;
  }

  return test_result;
}

void update_atrt_result_code(const TestResult &test_result,
                             AtrtExitCodes *return_code) {
  if (*return_code == ATRT_FAILURE) return;

  switch (test_result.result) {
    case ErrorCodes::ERR_OK:
      break;
    case ErrorCodes::ERR_CRITICAL:
      *return_code = ATRT_FAILURE;
      break;
    default:
      *return_code = TESTSUITE_FAILURES;
      break;
  }
}

int insert(const char *pair, Properties &p) {
  BaseString tmp(pair);

  Vector<BaseString> split;
  tmp.split(split, ":=", 2);

  if (split.size() != 2) return -1;

  p.put(split[0].trim().c_str(), split[1].trim().c_str());

  return 0;
}

/*
 * read_test_case - extract one testcase from file
 *
 * On success return a positive number with actual lines describing
 * the test case not counting blank lines and comments.
 * On end of file it returns 0.
 * On failure a nehative number is returned.
 */
int read_test_case(FILE *file, int &line, atrt_testcase &tc) {
  Properties p;
  int elements = 0;
  char buf[1024];

  while (!feof(file)) {
    if (file == stdin) printf("atrt> ");
    if (!fgets(buf, 1024, file)) break;

    line++;
    BaseString tmp = buf;

    if (tmp.length() > 0 && tmp.c_str()[0] == '#') continue;

    tmp.trim(" \t\n\r");

    if (tmp.length() == 0) {
      break;  // End of test case definition
    }

    if (insert(tmp.c_str(), p) != 0) {
      // Element line had no : or =
      if (elements == 0 && file == stdin) {
        // Assume a single line command with command and arguments
        // separated with a space
        Vector<BaseString> split;
        tmp.split(split, " ", 2);
        tc.m_cmd.m_exe = split[0];
        if (split.size() == 2)
          tc.m_cmd.m_args = split[1];
        else
          tc.m_cmd.m_args = "";
        tc.m_max_time = 60000;
        return 1;
      }
      g_logger.critical("Invalid test file: Corrupt line: %d: %s", line, buf);
      return ERR_CORRUPT_TESTCASE;
    }

    elements++;
  }

  if (elements == 0) {
    // End of file
    return 0;
  }

  int used_elements = 0;

  if (!p.get("cmd", tc.m_cmd.m_exe)) {
    g_logger.critical(
        "Invalid test file: cmd is missing in test case above line: %d", line);
    return ERR_CORRUPT_TESTCASE;
  }
  used_elements++;

  if (!p.get("args", tc.m_cmd.m_args))
    tc.m_cmd.m_args = "";
  else
    used_elements++;

  const char *mt = 0;
  if (!p.get("max-time", &mt))
    tc.m_max_time = 60000;
  else {
    tc.m_max_time = atoi(mt);
    used_elements++;
  }

  if (p.get("type", &mt)) {
    tc.m_report = (strcmp(mt, "bench") == 0);
    used_elements++;
  } else
    tc.m_report = false;

  if (p.get("run-all", &mt)) {
    tc.m_run_all = (strcmp(mt, "yes") == 0);
    used_elements++;
  } else
    tc.m_run_all = false;

  const char *str;
  if (p.get("mysqld", &str)) {
    tc.m_mysqld_options.assign(str);
    used_elements++;
  } else {
    tc.m_mysqld_options.assign("");
  }

  tc.m_cmd.m_cmd_type = atrt_process::AP_NDB_API;
  if (p.get("cmd-type", &str)) {
    if (strcmp(str, "mysql") == 0)
      tc.m_cmd.m_cmd_type = atrt_process::AP_CLIENT;
    used_elements++;
  }

  if (!p.get("name", &mt)) {
    tc.m_name.assfmt("%s %s", tc.m_cmd.m_exe.c_str(), tc.m_cmd.m_args.c_str());
  } else {
    tc.m_name.assign(mt);
    used_elements++;
  }

  tc.m_force_cluster_restart = g_default_force_cluster_restart;
  if (p.get("force-cluster-restart", &str)) {
    tc.m_force_cluster_restart = (strcmp(str, "yes") == 0);
    used_elements++;
  }

  tc.m_max_retries = g_default_max_retries;
  if (p.get("max-retries", &mt)) {
    tc.m_max_retries = atoi(mt);
    used_elements++;
  }

  if (tc.m_max_retries < 0) {
    g_logger.error("No of retries must not be less than zero for test '%s'",
                   tc.m_name.c_str());
    return ERR_CORRUPT_TESTCASE;
  }

  if (tc.m_max_retries > TESTCASE_RETRIES_THRESHOLD_WARNING)
    g_logger.warning(
        "No of retries should be less than or equal to %d for test '%s'",
        TESTCASE_RETRIES_THRESHOLD_WARNING, tc.m_name.c_str());

  tc.m_behaviour_on_failure = (FailureMode)g_default_behaviour_on_failure;
  if (p.get("on-failure", &str)) {
    std::map<std::string, FailureMode> failure_mode_values = {
        {"Restart", FailureMode::Restart},
        {"Abort", FailureMode::Abort},
        {"Skip", FailureMode::Skip},
        {"Continue", FailureMode::Continue}};
    if (failure_mode_values.find(str) == failure_mode_values.end()) {
      g_logger.critical("Invalid Failure mode!!");
      return ERR_CORRUPT_TESTCASE;
    }
    tc.m_behaviour_on_failure = failure_mode_values[str];
    used_elements++;
  }

  if (used_elements != elements) {
    g_logger.critical(
        "Invalid test file: unknown properties in test case above line: %d",
        line);
    return ERR_CORRUPT_TESTCASE;
  }

  return elements;
}

bool setup_test_case(atrt_config &config, const atrt_testcase &tc) {
  if (!remove_dir("result", true)) {
    g_logger.critical("setup_test_case: Failed to clear result");
    return false;
  }

  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    if (proc.m_type == atrt_process::AP_NDB_API ||
        proc.m_type == atrt_process::AP_CLIENT) {
      proc.m_proc.m_path.assign("");
      proc.m_proc.m_args.assign("");
    }
  }

  BaseString cmd;
  char *p = find_bin_path(tc.m_cmd.m_exe.c_str());
  if (p == 0) {
    g_logger.critical("Failed to locate '%s'", tc.m_cmd.m_exe.c_str());
    return false;
  }
  cmd.assign(p);
  free(p);

  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    if (proc.m_type == tc.m_cmd.m_cmd_type && proc.m_proc.m_path == "") {
      proc.m_save.m_proc = proc.m_proc;
      proc.m_save.m_saved = true;

      proc.m_proc.m_env.appfmt(" ATRT_TIMEOUT=%ld", tc.m_max_time);
      if (0)  // valgrind
      {
        proc.m_proc.m_path = "/usr/bin/valgrind";
        proc.m_proc.m_args.appfmt("%s %s", cmd.c_str(),
                                  tc.m_cmd.m_args.c_str());
      } else {
        proc.m_proc.m_path = cmd;
        proc.m_proc.m_args.assign(tc.m_cmd.m_args.c_str());
      }
      if (!tc.m_run_all) break;
    }
  }

  if (tc.m_mysqld_options != "") {
    g_logger.info("restarting mysqld with extra options: %s",
                  tc.m_mysqld_options.c_str());

    /**
     * Apply testcase specific mysqld options
     */
    for (unsigned i = 0; i < config.m_processes.size(); i++) {
      atrt_process &proc = *config.m_processes[i];
      if (proc.m_type == atrt_process::AP_MYSQLD) {
        if (!stop_process(proc)) {
          return false;
        }

        if (!wait_for_process_to_stop(config, proc)) {
          return false;
        }

        proc.m_save.m_proc = proc.m_proc;
        proc.m_save.m_saved = true;
        proc.m_proc.m_args.appfmt(" %s", tc.m_mysqld_options.c_str());

        if (!start_process(proc)) {
          return false;
        }

        if (!connect_mysqld(proc)) {
          return false;
        }
      }
    }
  }

  return true;
}

bool gather_result(atrt_config &config, int *result) {
  BaseString tmp = g_gather_progname;

  for (unsigned i = 0; i < config.m_hosts.size(); i++) {
    if (config.m_hosts[i]->m_hostname.length() == 0) continue;

    tmp.appfmt(" %s:%s", config.m_hosts[i]->m_hostname.c_str(),
               config.m_hosts[i]->m_basedir.c_str());
  }

  g_logger.debug("system(%s)", tmp.c_str());
  const int r1 = sh(tmp.c_str());
  if (r1 != 0) {
    g_logger.critical("Failed to gather result!");
    return false;
  }

  g_logger.debug("system(%s)", g_analyze_progname);
  const int r2 = sh(g_analyze_progname);

  if (r2 == -1 || r2 == (127 << 8)) {
    g_logger.critical("Failed to analyze results");
    return false;
  }

  *result = r2;
  return true;
}

bool setup_hosts(atrt_config &config) {
  if (!remove_dir("result", true)) {
    g_logger.critical("setup_hosts: Failed to clear result");
    return false;
  }

  for (unsigned i = 0; i < config.m_hosts.size(); i++) {
    if (config.m_hosts[i]->m_hostname.length() == 0) continue;
    BaseString tmp = g_setup_progname;
    tmp.appfmt(" %s %s/ %s/", config.m_hosts[i]->m_hostname.c_str(), g_basedir,
               config.m_hosts[i]->m_basedir.c_str());

    g_logger.debug("system(%s)", tmp.c_str());
    const int r1 = sh(tmp.c_str());
    if (r1 != 0) {
      g_logger.critical("Failed to setup %s",
                        config.m_hosts[i]->m_hostname.c_str());
      return false;
    }
  }
  return true;
}

static bool do_rsync(const char *dir, const char *dst) {
  BaseString tmp = g_setup_progname;
  tmp.appfmt(" %s %s/ %s", dst, dir, dir);

  g_logger.info("rsyncing %s to %s", dir, dst);
  g_logger.debug("system(%s)", tmp.c_str());
  const int r1 = sh(tmp.c_str());
  if (r1 != 0) {
    g_logger.critical("Failed to rsync %s to %s", dir, dst);
    return false;
  }

  return true;
}

bool deploy(int d, atrt_config &config) {
  for (unsigned i = 0; i < config.m_hosts.size(); i++) {
    if (config.m_hosts[i]->m_hostname.length() == 0) continue;

    if (d & 1) {
      if (!do_rsync(g_basedir, config.m_hosts[i]->m_hostname.c_str()))
        return false;
    }

    if (d & 2) {
      if (!do_rsync(g_prefix0, config.m_hosts[i]->m_hostname.c_str()))
        return false;

      if (g_prefix1 &&
          !do_rsync(g_prefix1, config.m_hosts[i]->m_hostname.c_str()))
        return false;
    }
  }

  return true;
}

bool sshx(atrt_config &config, unsigned mask) {
  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];

    BaseString tmp;
    const char *type = 0;
    switch (proc.m_type) {
      case atrt_process::AP_NDB_MGMD:
        type = (mask & proc.m_type) ? "ndb_mgmd" : 0;
        break;
      case atrt_process::AP_NDBD:
        type = (mask & proc.m_type) ? "ndbd" : 0;
        break;
      case atrt_process::AP_MYSQLD:
        type = (mask & proc.m_type) ? "mysqld" : 0;
        break;
      case atrt_process::AP_NDB_API:
        type = (mask & proc.m_type) ? "ndbapi" : 0;
        break;
      case atrt_process::AP_CLIENT:
        type = (mask & proc.m_type) ? "client" : 0;
        break;
      default:
        type = "<unknown>";
    }

    if (type == 0) continue;

#ifdef _WIN32
#define SYS_SSH                    \
  "bash '-c echo\"%s(%s) on %s\";" \
  "ssh -t %s sh %s/ssh-login.sh' &"
#else
#define SYS_SSH                   \
  "xterm -title \"%s(%s) on %s\"" \
  " -e 'ssh -t -X %s sh %s/ssh-login.sh' &"
#endif

    tmp.appfmt(SYS_SSH, type, proc.m_cluster->m_name.c_str(),
               proc.m_host->m_hostname.c_str(), proc.m_host->m_hostname.c_str(),
               proc.m_proc.m_cwd.c_str());

    g_logger.debug("system(%s)", tmp.c_str());
    const int r1 = sh(tmp.c_str());
    if (r1 != 0) {
      g_logger.critical("Failed sshx (%s)", tmp.c_str());
      return false;
    }
    NdbSleep_MilliSleep(300);  // To prevent xlock problem
  }

  return true;
}

bool start(atrt_config &config, unsigned proc_mask) {
  if (proc_mask & atrt_process::AP_NDB_MGMD)
    if (!start_processes(g_config, atrt_process::AP_NDB_MGMD)) return false;

  if (proc_mask & atrt_process::AP_NDBD) {
    if (!connect_ndb_mgm(g_config)) {
      return false;
    }

    if (!start_processes(g_config, atrt_process::AP_NDBD)) return false;

    if (!wait_ndb(g_config, NDB_MGM_NODE_STATUS_NOT_STARTED)) return false;

    for (Uint32 i = 0; i < 3; i++)
      if (wait_ndb(g_config, NDB_MGM_NODE_STATUS_STARTED)) goto started;
    return false;
  }

started:
  if (!start_processes(g_config, p_servers & proc_mask)) return false;

  return true;
}

bool reset_config(atrt_config &config) {
  bool changed = false;
  for (unsigned i = 0; i < config.m_processes.size(); i++) {
    atrt_process &proc = *config.m_processes[i];
    if (proc.m_save.m_saved) {
      if (proc.m_proc.m_id != -1) {
        if (!stop_process(proc)) return false;
        if (!wait_for_process_to_stop(config, proc)) return false;

        changed = true;
      }

      proc.m_save.m_saved = false;
      proc.m_proc = proc.m_save.m_proc;
    }
  }
  return changed;
}

bool find_scripts(const char *atrt_path) {
  g_logger.info("Locating scripts...");

  struct script_path {
    const char *name;
    const char **path;
  };
  std::vector<struct script_path> scripts = {
      {"atrt-gather-result.sh", &g_gather_progname},
      {"atrt-analyze-result.sh", &g_analyze_progname},
      {"atrt-setup.sh", &g_setup_progname}};

  for (auto &script : scripts) {
    BaseString script_full_path;
    script_full_path.assfmt("%s/%s", atrt_path, script.name);
    if (!File_class::exists(script_full_path.c_str())) {
      g_logger.critical("atrt script %s could not be found in %s", script.name,
                        atrt_path);
      return false;
    }
    *script.path = strdup(script_full_path.c_str());
  }
  return true;
}

static bool find_config_ini_files() {
  g_logger.info("Locating config.ini files...");

  BaseString tmp(g_clusters);
  Vector<BaseString> clusters;
  tmp.split(clusters, ",");

  bool found = true;
  for (unsigned int i = 0; i < clusters.size(); i++) {
    BaseString config_ini_path(g_cwd);
    const char *cluster_name = clusters[i].c_str();
    config_ini_path.appfmt("%sconfig%s.ini", PATH_SEPARATOR, cluster_name);
    to_native(config_ini_path);

    if (!exists_file(config_ini_path.c_str())) {
      g_logger.critical("Failed to locate '%s'", config_ini_path.c_str());
      found = false;
    }
  }

  return found;
}

BaseString get_atrt_path(const char *arg) {
  char *fullPath = realpath(arg, nullptr);
  if (fullPath == nullptr) return {};

  BaseString path;
  char *last_folder_sep = strrchr(fullPath, '/');
  if (last_folder_sep != nullptr) {
    *last_folder_sep = '\0';
    path.assign(fullPath);
  }

  free(fullPath);
  return path;
}

template class Vector<Vector<SimpleCpcClient::Process> >;
template class Vector<atrt_host *>;
template class Vector<atrt_cluster *>;
template class Vector<atrt_process *>;

int check_testcase_file_main(int argc, char **argv) {
  bool ok = true;
  int argi = 1;
  if (strcmp(argv[argi], "--check-testcase-files") == 0) {
    argi++;
  }
  if (argi == argc) {
    ok = false;
    g_logger.critical("Error: No files to check!\n");
  } else
    for (; argi < argc; argi++) {
      FILE *f = fopen(argv[argi], "r");
      if (f == NULL) {
        ok = false;
        g_logger.critical("Unable to open file: %s (%d: %s)", argv[argi], errno,
                          strerror(errno));
        continue;
      }
      atrt_testcase tc_dummy;
      int line_num = 0;
      int ntests = 0;
      int num_element_lines;
      while ((num_element_lines = read_test_case(f, line_num, tc_dummy)) > 0) {
        ntests++;
      }
      // If line count does not change that indicates end of file.
      if (num_element_lines >= 0) {
        printf("%s: Contains %d tests in %d lines.\n", argv[argi], ntests,
               line_num);
      } else {
        ok = false;
        g_logger.critical("%s: Error at line %d (error %d)\n", argv[argi],
                          line_num, num_element_lines);
      }
      fclose(f);
    }
  return ok ? 0 : 1;
}

void print_testcase_file_syntax() {
  printf(
      "\n"
      "Test cases to run are described in files passed with the\n"
      "--testcase-file (-f) option.\n"
      "\n"
      "A testcase is defined with some properties, one property per line,\n"
      "and terminated with exactly one empty line.  No other empty lines\n"
      "are allowed in the file.  Lines starting with # are comments and\n"
      "are ignored, note they are not counted as empty lines.\n"
      "\n"
      "The properties are:\n"
      "cmd      - Names the test executable.  The only mandatory property.\n"
      "args     - The arguments to test executable.\n"
      "max-time - Maximum run time for test in seconds (default 60000).\n"
      "type     - Declare the type of the test.  The only recognized value\n"
      "           is 'bench' which implies that results are stored also for\n"
      "           successful tests.  Normally if this option is not used\n"
      "           only results from failed tests will be stored.\n"
      "run-all  - If 'yes' atrt will start the same command for each defined\n"
      "           api/mysqld, normally it only starts one instance.\n"
      "mysqld   - Arguments that atrt will use when starting mysqld.\n"
      "cmd-type - If 'mysql' change test process type from ndbapi to client.\n"
      "name     - Change name of test.  Default is given by cmd and args.\n"
      "force-cluster-restart - If 'yes' force restart the cluster before\n"
      "                        running test.\n"
      "max-retries - Maximum number of retries after test failed.\n"
      ""
      "\n"
      "Example:\n"
      "# BASIC FUNCTIONALITY\n"
      "max-time: 500\n"
      "cmd: testBasic\n"
      "args: -n PkRead\n"
      "\n"
      "# 4k record DD\n"
      "max-time: 600\n"
      "cmd: flexAsynch\n"
      "args: -dd -temp -con 2 -t 8 -r 2 -p 64 -ndbrecord -a 25 -s 40\n"
      "type: bench\n"
      "\n"
      "# sql\n"
      "max-time: 600\n"
      "cmd: ndb-sql-perf.sh\n"
      "args: ndb-sql-perf-select.sh t1 1 64\n"
      "mysqld: --ndb-cluster-connection-pool=1\n"
      "type: bench\n"
      "cmd-type: mysql\n"
      "\n");
}