578 lines
17 KiB
C++
578 lines
17 KiB
C++
/************************************************************************
|
|
*
|
|
* Copyright (c) 2016 Alibaba.com, Inc. All Rights Reserved
|
|
* $Id: consensus-trx.cc,v 1.0 Nov 30, 2017 5:05:02 PM jarry.zj(jarry.zj@alibaba-inc.com) $
|
|
*
|
|
************************************************************************/
|
|
|
|
/**
|
|
* @file consensus-trx.cc
|
|
* @author jarry.zj(jarry.zj@alibaba-inc.com)
|
|
* @date Nov 30, 2017 5:05:02 PM
|
|
* @version 1.0
|
|
* @brief unit test for consensus with commit dependency (large trx)
|
|
*
|
|
**/
|
|
|
|
#include <atomic>
|
|
#include <thread>
|
|
#include <gtest/gtest.h>
|
|
#include <string>
|
|
#include "easyNet.h"
|
|
#include "service.h"
|
|
#include "paxos.h"
|
|
#include "paxos_log.h"
|
|
#include "paxos_server.h"
|
|
#include "paxos_configuration.h"
|
|
#include "paxos.pb.h"
|
|
#include "files.h"
|
|
|
|
#include "rd_paxos_log.h"
|
|
#include "file_paxos_log.h"
|
|
|
|
using namespace alisql;
|
|
|
|
/*
|
|
* testcase list:
|
|
* followers should keep consistency in all cases
|
|
* normal case: large trx commitindex update atomically
|
|
* leader crash during a large trx (cannot transfer leader during a large trx )
|
|
* write new logs during a large trx recovery
|
|
* debug very slow case:
|
|
* just very slow, keep heartbeat
|
|
* leader transfer during a large trx recovery (weight election should also not work)
|
|
* crash during a large trx recovery
|
|
* another leader requestvote during a large trx recovery
|
|
*/
|
|
|
|
TEST(trx, basic)
|
|
{
|
|
std::vector<std::string> strConfig;
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
|
|
const uint64_t timeout= 2000;
|
|
std::shared_ptr<PaxosLog> rlog, rlog1;
|
|
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
|
|
Paxos *paxos1= new Paxos(timeout, rlog);
|
|
paxos1->init(strConfig, 1);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
|
|
Paxos *paxos2= new Paxos(timeout, rlog);
|
|
paxos2->init(strConfig, 2);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
|
|
Paxos *paxos3= new Paxos(timeout, rlog);
|
|
paxos3->init(strConfig, 3);
|
|
|
|
sleep(3);
|
|
paxos1->requestVote();
|
|
sleep(1);
|
|
|
|
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
|
|
|
|
LogEntry le;
|
|
uint64_t cindex; // currentIndex
|
|
le.set_index(0);
|
|
le.set_optype(kNormal);
|
|
le.set_value("first");
|
|
paxos1->replicateLog(le);
|
|
cindex = le.index();
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex);
|
|
|
|
le.Clear();
|
|
le.set_optype(kNormal);
|
|
le.set_value("second");
|
|
paxos1->replicateLog(le);
|
|
for (int i=0; i<9; ++i)
|
|
{
|
|
le.Clear();
|
|
le.set_optype(kCommitDep);
|
|
le.set_value("part-" + std::to_string(i));
|
|
paxos1->replicateLog(le);
|
|
usleep(100000);
|
|
}
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex + 1); // still the old one
|
|
le.Clear();
|
|
le.set_optype(kCommitDepEnd);
|
|
le.set_value("part-end");
|
|
paxos1->replicateLog(le);
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex + 11);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), paxos1->getLastLogIndex());
|
|
le.Clear();
|
|
le.set_optype(kNormal);
|
|
le.set_value("final");
|
|
paxos1->replicateLog(le);
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), paxos1->getLastLogIndex());
|
|
EXPECT_EQ(paxos1->getCommitIndex(), paxos2->getCommitIndex());
|
|
|
|
delete paxos1;
|
|
delete paxos2;
|
|
delete paxos3;
|
|
deleteDir("paxosLogTestDir1");
|
|
deleteDir("paxosLogTestDir2");
|
|
deleteDir("paxosLogTestDir3");
|
|
}
|
|
|
|
TEST(trx, leader_crash_during_trx)
|
|
{
|
|
std::vector<std::string> strConfig;
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
|
|
const uint64_t timeout= 2000;
|
|
std::shared_ptr<PaxosLog> rlog, rlog1;
|
|
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
|
|
Paxos *paxos1= new Paxos(timeout, rlog);
|
|
paxos1->init(strConfig, 1);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
|
|
Paxos *paxos2= new Paxos(timeout, rlog);
|
|
paxos2->init(strConfig, 2);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
|
|
Paxos *paxos3= new Paxos(timeout, rlog);
|
|
paxos3->init(strConfig, 3);
|
|
|
|
sleep(3);
|
|
paxos1->requestVote();
|
|
sleep(1);
|
|
|
|
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
|
|
|
|
LogEntry le;
|
|
uint64_t cindex = paxos1->getLastLogIndex(); // currentIndex
|
|
for (int i=0; i<10; ++i)
|
|
{
|
|
le.Clear();
|
|
le.set_optype(kCommitDep);
|
|
le.set_value("part-" + std::to_string(i));
|
|
paxos1->replicateLog(le);
|
|
usleep(100000);
|
|
}
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex); // still the old one
|
|
cindex = paxos1->getLastLogIndex();
|
|
delete paxos1;
|
|
while(paxos2->getState() != Paxos::LEADER && paxos3->getState() != Paxos::LEADER) {}
|
|
Paxos *leader;
|
|
if (paxos2->getState() == Paxos::LEADER) leader = paxos2;
|
|
if (paxos3->getState() == Paxos::LEADER) leader = paxos3;
|
|
le.Clear();
|
|
le.set_optype(kNormal);
|
|
le.set_value("normal case");
|
|
leader->replicateLog(le); // may fail
|
|
EXPECT_EQ(le.term(), 0);
|
|
sleep(1);
|
|
EXPECT_EQ(leader->getCommitIndex(), cindex + 1);
|
|
for (int i=0; i<11; ++i)
|
|
{
|
|
// check all kCommitDep is replaced
|
|
leader->getLog()->getEntry(cindex-i, le, false);
|
|
EXPECT_NE(le.optype(), kCommitDep);
|
|
}
|
|
le.Clear();
|
|
le.set_optype(kNormal);
|
|
le.set_value("final");
|
|
leader->replicateLog(le);
|
|
sleep(1);
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos2->getLastLogIndex());
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos3->getCommitIndex());
|
|
|
|
delete paxos2;
|
|
delete paxos3;
|
|
deleteDir("paxosLogTestDir1");
|
|
deleteDir("paxosLogTestDir2");
|
|
deleteDir("paxosLogTestDir3");
|
|
}
|
|
|
|
TEST(trx, trx_recovery_slow)
|
|
{
|
|
std::vector<std::string> strConfig;
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
|
|
const uint64_t timeout= 2000;
|
|
std::shared_ptr<PaxosLog> rlog, rlog1;
|
|
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
|
|
Paxos *paxos1= new Paxos(timeout, rlog);
|
|
paxos1->init(strConfig, 1);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
|
|
Paxos *paxos2= new Paxos(timeout, rlog);
|
|
paxos2->init(strConfig, 2);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
|
|
Paxos *paxos3= new Paxos(timeout, rlog);
|
|
paxos3->init(strConfig, 3);
|
|
|
|
sleep(3);
|
|
paxos1->requestVote();
|
|
sleep(1);
|
|
|
|
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
|
|
|
|
LogEntry le;
|
|
uint64_t cindex = paxos1->getLastLogIndex(); // currentIndex
|
|
for (int i=0; i<3; ++i)
|
|
{
|
|
le.Clear();
|
|
le.set_optype(kCommitDep);
|
|
le.set_value("part-" + std::to_string(i));
|
|
paxos1->replicateLog(le);
|
|
usleep(100000);
|
|
}
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex); // still the old one
|
|
cindex = paxos1->getLastLogIndex();
|
|
paxos2->debugResetLogSlow = true;
|
|
paxos3->debugResetLogSlow = true;
|
|
delete paxos1;
|
|
while(paxos2->getState() != Paxos::LEADER && paxos3->getState() != Paxos::LEADER) {}
|
|
Paxos *leader, *other;
|
|
if (paxos2->getState() == Paxos::LEADER) {
|
|
leader = paxos2;
|
|
other = paxos3;
|
|
}
|
|
if (paxos3->getState() == Paxos::LEADER) {
|
|
leader = paxos3;
|
|
other = paxos2;
|
|
}
|
|
std::atomic<bool> foo(false);
|
|
other->setStateChangeCb([&foo](Paxos::StateType state, uint64_t term, uint64_t index) {
|
|
foo = true;
|
|
});
|
|
le.Clear();
|
|
le.set_optype(kNormal);
|
|
le.set_value("normal case");
|
|
leader->replicateLog(le); // should fail
|
|
EXPECT_EQ(le.term(), 0);
|
|
sleep(6);
|
|
for (int i=0 ; i< 5; ++i)
|
|
{
|
|
leader->replicateLog(le); // success
|
|
EXPECT_NE(le.term(), 0);
|
|
}
|
|
sleep(2);
|
|
/* recovery is slow but still send heartbeat */
|
|
/* ensure other never statechange (requestVote) */
|
|
EXPECT_EQ(foo.load(), false);
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos2->getLastLogIndex());
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos3->getCommitIndex());
|
|
|
|
easy_warn_log("Restart paxos1.........\n");
|
|
/* now I bring paxos1 back */
|
|
strConfig.clear();
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
paxos1= new Paxos(timeout, rlog1);
|
|
paxos1->init(strConfig, 1);
|
|
sleep(2);
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos2->getLastLogIndex());
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos3->getCommitIndex());
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos1->getCommitIndex());
|
|
delete paxos1;
|
|
delete paxos2;
|
|
delete paxos3;
|
|
deleteDir("paxosLogTestDir1");
|
|
deleteDir("paxosLogTestDir2");
|
|
deleteDir("paxosLogTestDir3");
|
|
}
|
|
|
|
TEST(trx, trx_recovery_slow2)
|
|
{
|
|
std::vector<std::string> strConfig;
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
|
|
const uint64_t timeout= 2000;
|
|
std::shared_ptr<PaxosLog> rlog, rlog1;
|
|
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
|
|
Paxos *paxos1= new Paxos(timeout, rlog);
|
|
paxos1->init(strConfig, 1);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
|
|
Paxos *paxos2= new Paxos(timeout, rlog);
|
|
paxos2->init(strConfig, 2);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
|
|
Paxos *paxos3= new Paxos(timeout, rlog);
|
|
paxos3->init(strConfig, 3);
|
|
|
|
sleep(3);
|
|
paxos1->requestVote();
|
|
sleep(1);
|
|
|
|
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
|
|
|
|
LogEntry le;
|
|
uint64_t cindex = paxos1->getLastLogIndex(); // currentIndex
|
|
for (int i=0; i<3; ++i)
|
|
{
|
|
le.Clear();
|
|
le.set_optype(kCommitDep);
|
|
le.set_value("part-" + std::to_string(i));
|
|
paxos1->replicateLog(le);
|
|
usleep(100000);
|
|
}
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex); // still the old one
|
|
cindex = paxos1->getLastLogIndex();
|
|
paxos2->debugResetLogSlow = true;
|
|
paxos3->debugResetLogSlow = true;
|
|
delete paxos1;
|
|
while(paxos2->getState() != Paxos::LEADER && paxos3->getState() != Paxos::LEADER) {}
|
|
Paxos *leader, *other;
|
|
if (paxos2->getState() == Paxos::LEADER) {
|
|
leader = paxos2;
|
|
other = paxos3;
|
|
}
|
|
if (paxos3->getState() == Paxos::LEADER) {
|
|
leader = paxos3;
|
|
other = paxos2;
|
|
}
|
|
easy_warn_log("Restart paxos1.........\n");
|
|
/* now I bring paxos1 back, during trx recovery */
|
|
strConfig.clear();
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
paxos1= new Paxos(timeout, rlog1);
|
|
paxos1->init(strConfig, 1);
|
|
|
|
le.Clear();
|
|
le.set_optype(kNormal);
|
|
le.set_value("normal case");
|
|
leader->replicateLog(le); // should fail
|
|
EXPECT_EQ(le.term(), 0);
|
|
sleep(6);
|
|
for (int i=0 ; i< 5; ++i)
|
|
{
|
|
leader->replicateLog(le); // success
|
|
EXPECT_NE(le.term(), 0);
|
|
}
|
|
sleep(2);
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos2->getLastLogIndex());
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos3->getCommitIndex());
|
|
EXPECT_EQ(paxos2->getCommitIndex(), paxos1->getCommitIndex());
|
|
delete paxos1;
|
|
delete paxos2;
|
|
delete paxos3;
|
|
deleteDir("paxosLogTestDir1");
|
|
deleteDir("paxosLogTestDir2");
|
|
deleteDir("paxosLogTestDir3");
|
|
}
|
|
|
|
TEST(trx, trx_recovery_slow_leader_transfer)
|
|
{
|
|
std::vector<std::string> strConfig;
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
|
|
const uint64_t timeout= 2000;
|
|
std::shared_ptr<PaxosLog> rlog, rlog1, rlog2;
|
|
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
|
|
Paxos *paxos1= new Paxos(timeout, rlog);
|
|
paxos1->init(strConfig, 1);
|
|
rlog2= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
|
|
Paxos *paxos2= new Paxos(timeout, rlog);
|
|
paxos2->init(strConfig, 2);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
|
|
Paxos *paxos3= new Paxos(timeout, rlog);
|
|
paxos3->init(strConfig, 3);
|
|
|
|
sleep(3);
|
|
paxos1->requestVote();
|
|
sleep(1);
|
|
|
|
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
|
|
|
|
LogEntry le;
|
|
uint64_t cindex = paxos1->getLastLogIndex(); // currentIndex
|
|
for (int i=0; i<3; ++i)
|
|
{
|
|
le.Clear();
|
|
le.set_optype(kCommitDep);
|
|
le.set_value("part-" + std::to_string(i));
|
|
paxos1->replicateLog(le);
|
|
usleep(100000);
|
|
}
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex); // still the old one
|
|
cindex = paxos1->getLastLogIndex();
|
|
paxos2->debugResetLogSlow = true;
|
|
paxos3->debugResetLogSlow = true;
|
|
delete paxos1;
|
|
while(paxos2->getState() != Paxos::LEADER && paxos3->getState() != Paxos::LEADER) {}
|
|
Paxos *leader, *other;
|
|
if (paxos2->getState() == Paxos::LEADER) {
|
|
leader = paxos2;
|
|
other = paxos3;
|
|
}
|
|
if (paxos3->getState() == Paxos::LEADER) {
|
|
leader = paxos3;
|
|
other = paxos2;
|
|
}
|
|
easy_warn_log("Restart paxos1.........\n");
|
|
/* now I bring paxos1 back, during trx recovery */
|
|
strConfig.clear();
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
paxos1= new Paxos(timeout, rlog1);
|
|
paxos1->init(strConfig, 1);
|
|
|
|
le.Clear();
|
|
le.set_optype(kNormal);
|
|
le.set_value("normal case");
|
|
leader->replicateLog(le); // may fail
|
|
EXPECT_EQ(le.term(), 0);
|
|
EXPECT_EQ(leader->leaderTransfer(other->getLocalServer()->serverId), PaxosErrorCode::PE_CONFLICTS); /* leader transfer is disabled */
|
|
/* now crash leader */
|
|
delete leader;
|
|
sleep(6);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), paxos1->getLastLogIndex());
|
|
EXPECT_EQ(paxos1->getCommitIndex(), other->getCommitIndex());
|
|
|
|
delete paxos1;
|
|
delete other;
|
|
deleteDir("paxosLogTestDir1");
|
|
deleteDir("paxosLogTestDir2");
|
|
deleteDir("paxosLogTestDir3");
|
|
}
|
|
|
|
TEST(trx, trx_recovery_slow_crash)
|
|
{
|
|
std::vector<std::string> strConfig;
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
|
|
const uint64_t timeout= 2000;
|
|
std::shared_ptr<PaxosLog> rlog, rlog1;
|
|
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
|
|
Paxos *paxos1= new Paxos(timeout, rlog);
|
|
paxos1->init(strConfig, 1);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
|
|
Paxos *paxos2= new Paxos(timeout, rlog);
|
|
paxos2->init(strConfig, 2);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
|
|
Paxos *paxos3= new Paxos(timeout, rlog);
|
|
paxos3->init(strConfig, 3);
|
|
|
|
sleep(3);
|
|
paxos1->requestVote();
|
|
sleep(1);
|
|
|
|
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
|
|
|
|
LogEntry le;
|
|
uint64_t cindex = paxos1->getLastLogIndex(); // currentIndex
|
|
for (int i=0; i<10; ++i)
|
|
{
|
|
le.Clear();
|
|
le.set_optype(kCommitDep);
|
|
le.set_value("part-" + std::to_string(i));
|
|
paxos1->replicateLog(le);
|
|
usleep(100000);
|
|
}
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex); // still the old one
|
|
cindex = paxos1->getLastLogIndex();
|
|
paxos2->debugResetLogSlow = true;
|
|
paxos3->debugResetLogSlow = true;
|
|
delete paxos1;
|
|
while(paxos2->getState() != Paxos::LEADER && paxos3->getState() != Paxos::LEADER) {}
|
|
Paxos *leader, *other;
|
|
if (paxos2->getState() == Paxos::LEADER) {
|
|
leader = paxos2;
|
|
other = paxos3;
|
|
}
|
|
if (paxos3->getState() == Paxos::LEADER) {
|
|
leader = paxos3;
|
|
other = paxos2;
|
|
}
|
|
other->debugResetLogSlow = false;
|
|
sleep(1);
|
|
delete leader;
|
|
other->forceSingleLeader();
|
|
sleep(6);
|
|
EXPECT_EQ(other->getLastLogIndex(), other->getCommitIndex());
|
|
EXPECT_TRUE(other->getLastLogIndex() > cindex);
|
|
|
|
delete other;
|
|
deleteDir("paxosLogTestDir1");
|
|
deleteDir("paxosLogTestDir2");
|
|
deleteDir("paxosLogTestDir3");
|
|
}
|
|
|
|
TEST(trx, trx_recovery_slow_recv_requestVote)
|
|
{
|
|
std::vector<std::string> strConfig;
|
|
strConfig.emplace_back("127.0.0.1:11001");
|
|
strConfig.emplace_back("127.0.0.1:11002");
|
|
strConfig.emplace_back("127.0.0.1:11003");
|
|
|
|
const uint64_t timeout= 2000;
|
|
std::shared_ptr<PaxosLog> rlog, rlog1;
|
|
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
|
|
Paxos *paxos1= new Paxos(timeout, rlog);
|
|
paxos1->init(strConfig, 1);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
|
|
Paxos *paxos2= new Paxos(timeout, rlog);
|
|
paxos2->init(strConfig, 2);
|
|
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
|
|
Paxos *paxos3= new Paxos(timeout, rlog);
|
|
paxos3->init(strConfig, 3);
|
|
|
|
sleep(3);
|
|
paxos1->requestVote();
|
|
sleep(1);
|
|
|
|
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
|
|
|
|
LogEntry le;
|
|
uint64_t cindex = paxos1->getLastLogIndex(); // currentIndex
|
|
for (int i=0; i<10; ++i)
|
|
{
|
|
le.Clear();
|
|
le.set_optype(kCommitDep);
|
|
le.set_value("part-" + std::to_string(i));
|
|
paxos1->replicateLog(le);
|
|
usleep(100000);
|
|
}
|
|
sleep(1);
|
|
EXPECT_EQ(paxos1->getCommitIndex(), cindex); // still the old one
|
|
cindex = paxos1->getLastLogIndex();
|
|
paxos2->debugResetLogSlow = true;
|
|
paxos3->debugResetLogSlow = true;
|
|
delete paxos1;
|
|
while(paxos2->getState() != Paxos::LEADER && paxos3->getState() != Paxos::LEADER) {}
|
|
Paxos *leader, *other;
|
|
if (paxos2->getState() == Paxos::LEADER) {
|
|
leader = paxos2;
|
|
other = paxos3;
|
|
}
|
|
if (paxos3->getState() == Paxos::LEADER) {
|
|
leader = paxos3;
|
|
other = paxos2;
|
|
}
|
|
other->debugResetLogSlow = false;
|
|
// other->requestVote(false); /* will trigger stickness */
|
|
if (leader == paxos2)
|
|
leader->leaderTransfer(3);
|
|
else
|
|
leader->leaderTransfer(2);
|
|
sleep(1);
|
|
EXPECT_EQ(other->getLastLogIndex(), other->getCommitIndex());
|
|
EXPECT_EQ(other->getLastLogIndex(), leader->getLastLogIndex());
|
|
|
|
delete paxos2;
|
|
delete paxos3;
|
|
deleteDir("paxosLogTestDir1");
|
|
deleteDir("paxosLogTestDir2");
|
|
deleteDir("paxosLogTestDir3");
|
|
}
|