4493d40a-92aa-4162-9dfc-bfc4ec0996e6
/
polardbxengine
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
polardbxengine/extra/IS/consensus/unittest/consensus-learner-t.cc

2329 lines
74 KiB
C++
Raw Blame History

/************************************************************************
*
* Copyright (c) 2017 Alibaba.com, Inc. All Rights Reserved
* $Id: consensus1-t.cc,v 1.0 03/21/2017 09:19:25 PM yingqiang.zyq(yingqiang.zyq@alibaba-inc.com) $
*
************************************************************************/
/**
* @file consensus-learner-t.cc
* @author yingqiang.zyq(yingqiang.zyq@alibaba-inc.com)
* @date 03/21/2017 09:19:25 PM
* @version 1.0
* @brief
*
**/
#include <atomic>
#include <thread>
#include <gtest/gtest.h>
#include "easyNet.h"
#include "service.h"
#include "paxos.h"
#include "paxos_log.h"
#include "paxos_server.h"
#include "paxos_configuration.h"
#include "paxos.pb.h"
#include "files.h"
#include "rd_paxos_log.h"
#include "file_paxos_log.h"
using namespace alisql;
TEST(consensus, learnerSource)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::string strTmp4("127.0.0.1:11005");
std::string strTmp5("127.0.0.1:11006");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir15", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos5= new Paxos(timeout, rlog);
paxos5->initAsLearner(strTmp4);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir16", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos6= new Paxos(timeout, rlog);
paxos6->initAsLearner(strTmp5);
const Paxos::StatsType &stats= paxos2->getStats();
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
strConfig.push_back(strTmp4);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00"));
/* extra safety test */
EXPECT_EQ(paxos1->configureMember(100, true, 9), PE_WEIGHTLEARNER);
EXPECT_STREQ(pxserror(paxos1->configureMember("127.0.0.1:11004", true, 9)), "Configure forcesync or weight to a learner is not allowed.");
paxos1->configureLearner(100, 2);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$02;127.0.0.1:11005$00"));
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string("127.0.0.1:11004$02;127.0.0.1:11005$00"));
EXPECT_EQ(paxos2->getConfig()->getServer(100)->learnerSource, 2);
/* test getServerIdFromAddr */
EXPECT_EQ(paxos1->getServerIdFromAddr("127.0.0.1:11001"), 1);
EXPECT_EQ(paxos1->getServerIdFromAddr("127.0.0.1:11002"), 2);
EXPECT_EQ(paxos1->getServerIdFromAddr("127.0.0.1:11003"), 3);
EXPECT_EQ(paxos1->getServerIdFromAddr("127.0.0.1:11004"), 100);
EXPECT_EQ(paxos1->getServerIdFromAddr("127.0.0.1:11005"), 101);
paxos2->clearStats();
le.set_index(0);
le.set_optype(0);
le.set_value("aaa");
paxos1->replicateLog(le);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(2);
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog<< " countMsgRequestVote:"<<stats.countMsgRequestVote<< " countOnMsgAppendLog:"<< stats.countOnMsgAppendLog<< " countHeartbeat:"<< stats.countHeartbeat << " countOnMsgRequestVote:"<<stats.countOnMsgRequestVote<< " countOnHeartbeat:"<<stats.countOnHeartbeat<< " countReplicateLog:"<<stats.countReplicateLog<< std::endl<< std::flush;
EXPECT_EQ(stats.countMsgAppendLog, 2);
EXPECT_EQ(paxos4->getCurrentLeader(), 2);
EXPECT_EQ(paxos5->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
EXPECT_EQ(paxos5->getLastLogIndex(), paxos1->getLastLogIndex());
std::vector<Paxos::ClusterInfoType> cis;
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(cis[3].learnerSource, 2);
EXPECT_EQ(cis[4].learnerSource, 0);
EXPECT_EQ(cis[0].learnerSource, 0);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_TRUE(cis[3].matchIndex <= cis[1].matchIndex);
EXPECT_TRUE(cis[3].matchIndex >= cis[1].matchIndex - 2);
EXPECT_TRUE(cis[3].nextIndex <= cis[1].nextIndex);
EXPECT_TRUE(cis[3].nextIndex >= cis[1].nextIndex - 2);
paxos1->configureLearner(100, 3);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(2);
EXPECT_EQ(paxos4->getCurrentLeader(), 3);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_TRUE(cis[3].matchIndex <= cis[1].matchIndex);
EXPECT_TRUE(cis[3].matchIndex >= cis[1].matchIndex - 2);
EXPECT_TRUE(cis[3].nextIndex <= cis[1].nextIndex);
EXPECT_TRUE(cis[3].nextIndex >= cis[1].nextIndex - 2);
paxos1->configureLearner(100, 0);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
// Already fixed! Error may happy when server 1 is leader, and we change learnerSource from 1 to 0.
paxos1->configureLearner(100, 1);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
paxos1->configureLearner(100, 0);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
// set the learnerSource to another learner
paxos1->configureLearner(100, 101);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 101);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
uint64_t tmpInt= 0;
std::cout<<"Cluster Info for server "<<paxos1->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
tmpInt= cis[0].matchIndex;
std::cout<<"Cluster Info for server "<<paxos5->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos5->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(tmpInt, cis[1].matchIndex);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
std::cout<<"Cluster Info for server "<<paxos1->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_LE(tmpInt, cis[3].matchIndex);
std::cout<<"Cluster Info for server "<<paxos5->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos5->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
// check if the new added member has the forceSync electionWeight and learnerSource info.
paxos1->configureMember(2, false, 8);
sleep(1);
paxos1->configureLearner(100, 2);
sleep(1);
strConfig.clear();
strConfig.push_back(strTmp5);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
paxos1->changeMember(Paxos::CCAddNode, strTmp5);
sleep(5);
EXPECT_EQ(paxos6->getCurrentLeader(), 1);
EXPECT_EQ(paxos6->getLastLogIndex(), paxos1->getLastLogIndex());
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
paxos6->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(cis[1].electionWeight, 8);
EXPECT_EQ(cis[4].learnerSource, 2);
delete paxos6;
delete paxos5;
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
deleteDir("paxosLogTestDir15");
deleteDir("paxosLogTestDir16");
}
TEST(consensus, follower2learner)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::string strTmp4("127.0.0.1:11005");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog4;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir15", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos5= new Paxos(timeout, rlog);
paxos5->initAsLearner(strTmp4);
le.set_optype(0);
//const Paxos::StatsType &stats= paxos2->getStats();
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
strConfig.push_back(strTmp4);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(0, paxos1->changeMember(Paxos::CCAddNode, strTmp3));
sleep(1);
EXPECT_EQ(0, paxos1->changeMember(Paxos::CCAddNode, strTmp4));
sleep(1);
auto config1= std::dynamic_pointer_cast<StableConfiguration>(paxos1->getConfig());
EXPECT_EQ(config1->membersToString(("127.0.0.1:11001")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N;127.0.0.1:11004#5N;127.0.0.1:11005#5N@1"));
std::vector<Paxos::ClusterInfoType> cis;
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(0, paxos1->downgradeMember(4));
sleep(2);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(Paxos::LEARNER, paxos4->getState());
std::string tmpStr;
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004#5N"));
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00"));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
sleep(1);
EXPECT_EQ(0, paxos1->changeMember(Paxos::CCAddNode, strTmp3));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
sleep(1);
EXPECT_EQ(0, paxos1->downgradeMember(4));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(100, cis[4].serverId);
EXPECT_EQ("127.0.0.1:11004", cis[4].ipPort);
sleep(1);
std::cout<<"strTmp3:"<<strTmp3<<std::endl<<std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(0, paxos1->changeMember(Paxos::CCAddNode, strTmp3));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
sleep(1);
// can't downgrade a learner
EXPECT_EQ(PaxosErrorCode::PE_DOWNGRADLEARNER, paxos1->downgradeMember(100));
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(5, cis.size());
// when a node change from follower => learner => follower, this node can't create connect to other node.
// That means can't became leader.
sleep(1);
paxos1->leaderTransfer(4);
sleep(6);
EXPECT_EQ(paxos4->getState(), Paxos::LEADER);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
delete paxos5;
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
deleteDir("paxosLogTestDir15");
}
TEST(consensus, downgrade_follower_donnot_recieve_msg)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
le.set_optype(0);
//const Paxos::StatsType &stats= paxos2->getStats();
sleep(3);
paxos1->requestVote();
for (uint i = 0; i < 30; ++i) {
if (paxos1->getState() != Paxos::LEADER)
sleep(1);
else
break;
}
ASSERT_TRUE(paxos1->getState() == Paxos::LEADER);
std::vector<Paxos::ClusterInfoType> cis;
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
ASSERT_TRUE(0 == paxos1->downgradeMember(2));
auto s100= paxos1->getConfig()->getServer(100);
ASSERT_TRUE(s100 != 0);
s100->disconnect(nullptr);
paxos2->requestVote(false);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(cis[2].serverId, 100);
EXPECT_EQ(cis[1].serverId, 3);
le.clear_term();
paxos1->replicateLog(le);
sleep(2);
EXPECT_EQ(paxos2->getCurrentLeader(), 1);
EXPECT_EQ(paxos2->getState(), Paxos::LEARNER);
EXPECT_EQ(paxos2->getLastLogIndex(), paxos1->getLastLogIndex());
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
}
TEST(consensus, restart_learner)
{
std::vector<std::string> strConfig;
std::string tmpStr;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::string strTmp4("127.0.0.1:11005");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog4;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir15", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos5= new Paxos(timeout, rlog);
paxos5->initAsLearner(strTmp4);
le.set_optype(0);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
//EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
cis.clear();
paxos4->getClusterInfo(cis);
sleep(1);
Paxos::printClusterInfo(cis);
std::cout<< "11004 restart"<< std::endl<< std::flush;
delete paxos4;
sleep(2);
paxos4= new Paxos(timeout, rlog4);
std::string emptryString("");
paxos4->initAsLearner(emptryString);
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00"));
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004#5N"));
cis.clear();
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
sleep(2);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
strConfig.clear();
strConfig.push_back(strTmp4);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
std::cout<< "11004 restart"<< std::endl<< std::flush;
delete paxos4;
sleep(1);
paxos4= new Paxos(timeout, rlog4);
paxos4->initAsLearner(emptryString);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(2);
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00;127.0.0.1:11005$00"));
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004#5N"));
cis.clear();
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
delete paxos4;
delete paxos5;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
deleteDir("paxosLogTestDir15");
}
TEST(consensus, restart_follower)
{
std::vector<std::string> strConfig;
std::string tmpStr;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog2;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog2= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
le.set_optype(0);
sleep(3);
paxos3->requestVote();
sleep(1);
EXPECT_EQ(paxos3->getState(), Paxos::LEADER);
EXPECT_EQ(paxos3->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
//EXPECT_EQ(paxos3->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00"));
EXPECT_EQ(paxos3->getConfig()->learnersToString(), std::string(""));
sleep(1);
le.clear_term();
paxos3->replicateLog(le);
cis.clear();
paxos3->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
ASSERT_TRUE(0 == paxos3->downgradeMember(1));
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEARNER);
cis.clear();
paxos3->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "11002 restart"<< std::endl<< std::flush;
delete paxos2;
sleep(2);
paxos2= new Paxos(timeout, rlog2);
strConfig.clear();
paxos2->init(strConfig, 0);
sleep(2);
rlog2->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11001$00"));
rlog2->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("0;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
cis.clear();
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
sleep(2);
le.clear_term();
paxos3->replicateLog(le);
sleep(1);
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
}
TEST(consensus, configureChange_affect_learner_meta)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::string strTmp4("127.0.0.1:11005");
std::string strTmp6("127.0.0.1:11006");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog2, rlog4, rlog5, rlog6;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog2= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
rlog5= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir15", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos5= new Paxos(timeout, rlog);
paxos5->initAsLearner(strTmp4);
rlog6= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir16", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos6= new Paxos(timeout, rlog);
paxos6->initAsLearner(strTmp6);
le.set_optype(0);
//const Paxos::StatsType &stats= paxos2->getStats();
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
strConfig.clear();
strConfig.push_back(strTmp4);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
auto config1= std::dynamic_pointer_cast<StableConfiguration>(paxos1->getConfig());
EXPECT_EQ(config1->membersToString(("127.0.0.1:11001")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@1"));
EXPECT_EQ(config1->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00"));
std::vector<Paxos::ClusterInfoType> cis;
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::string tmpStr;
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004#5N"));
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00;127.0.0.1:11005$00"));
// follower -> learner
EXPECT_EQ(0, paxos1->downgradeMember(2));
sleep(2);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
std::cout<< "s1:"<< std::endl<< std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "s2:"<< std::endl<< std::flush;
cis.clear();
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
std::cout<< "s4:"<< std::endl<< std::flush;
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(Paxos::LEARNER, paxos4->getState());
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004#5N"));
rlog2->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00;127.0.0.1:11005$00;127.0.0.1:11002$00"));
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00;127.0.0.1:11005$00;127.0.0.1:11002$00"));
rlog5->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00;127.0.0.1:11005$00;127.0.0.1:11002$00"));
// learner -> follower
EXPECT_EQ(0, paxos1->changeMember(Paxos::CCAddNode, strTmp3));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
std::cout<< "s1:"<< std::endl<< std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "s2:"<< std::endl<< std::flush;
cis.clear();
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
std::cout<< "s4:"<< std::endl<< std::flush;
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(Paxos::FOLLOWER, paxos4->getState());
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11001#5N;127.0.0.1:11004#5N;127.0.0.1:11003#5N@2"));
rlog2->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("0;127.0.0.1:11005$00;127.0.0.1:11002$00"));
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("0;127.0.0.1:11005$00;127.0.0.1:11002$00"));
rlog5->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("0;127.0.0.1:11005$00;127.0.0.1:11002$00"));
// follower -> learner
EXPECT_EQ(0, paxos1->downgradeMember(3));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
std::cout<< "s1:"<< std::endl<< std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "s2:"<< std::endl<< std::flush;
cis.clear();
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
std::cout<< "s4:"<< std::endl<< std::flush;
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(Paxos::FOLLOWER, paxos4->getState());
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11001#5N;127.0.0.1:11004#5N@2"));
rlog2->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11003$00;127.0.0.1:11005$00;127.0.0.1:11002$00"));
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11003$00;127.0.0.1:11005$00;127.0.0.1:11002$00"));
rlog5->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11003$00;127.0.0.1:11005$00;127.0.0.1:11002$00"));
// configureLearner
paxos1->configureLearner(100, 102);
for (int i=0; i< 7; ++i)
{
// add 103-109
strConfig.clear();
strConfig.push_back("127.0.0.1:1200" + std::to_string(i));
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
}
// add 110
strConfig.clear();
strConfig.push_back(strTmp6);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
for (int i=0; i< 7; ++i)
{
// delete 103-109
strConfig.clear();
strConfig.push_back("127.0.0.1:1200" + std::to_string(i));
paxos1->changeLearners(Paxos::CCDelNode, strConfig);
}
paxos1->configureLearner(101, 110);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
std::cout<< "s1:"<< std::endl<< std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "s2:"<< std::endl<< std::flush;
cis.clear();
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
std::cout<< "s4:"<< std::endl<< std::flush;
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(Paxos::FOLLOWER, paxos4->getState());
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11001#5N;127.0.0.1:11004#5N@2"));
rlog2->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11003$:2;127.0.0.1:11005$;0;127.0.0.1:11002$00;0;0;0;0;0;0;0;127.0.0.1:11006$00"));
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11003$:2;127.0.0.1:11005$;0;127.0.0.1:11002$00;0;0;0;0;0;0;0;127.0.0.1:11006$00"));
rlog5->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11003$:2;127.0.0.1:11005$;0;127.0.0.1:11002$00;0;0;0;0;0;0;0;127.0.0.1:11006$00"));
//restart learner
std::cout<< "11005 restart"<< std::endl<< std::flush;
delete paxos5;
sleep(2);
paxos5= new Paxos(timeout, rlog5);
std::string emptryString("");
paxos5->initAsLearner(emptryString);
rlog5->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11003$:2;127.0.0.1:11005$;0;127.0.0.1:11002$00;0;0;0;0;0;0;0;127.0.0.1:11006$00"));
rlog5->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11005#5N"));
cis.clear();
std::cout<< "s5:"<< std::endl<< std::flush;
paxos5->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
sleep(2);
delete paxos6;
delete paxos5;
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
deleteDir("paxosLogTestDir15");
deleteDir("paxosLogTestDir16");
}
TEST(consensus, learner_old_meta_new_version)
{
std::vector<std::string> strConfig;
std::string tmpStr;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog4;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
le.set_optype(0);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
cis.clear();
paxos4->getClusterInfo(cis);
sleep(1);
Paxos::printClusterInfo(cis);
std::cout<< "11004 restart"<< std::endl<< std::flush;
delete paxos4;
sleep(2);
//change meta to old format
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004#5N"));
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004$00"));
rlog4->setMetaData(std::string(Paxos::keyMemberConfigure), std::string(""));
paxos4= new Paxos(timeout, rlog4);
std::string emptryString("");
paxos4->initAsLearner(emptryString);
sleep(2);
rlog4->getMetaData(std::string(Paxos::keyLearnerConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string(""));
rlog4->getMetaData(std::string(Paxos::keyMemberConfigure), tmpStr);
EXPECT_EQ(tmpStr, std::string("127.0.0.1:11004#5N"));
cis.clear();
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
sleep(2);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
}
TEST(consensus, SyncLearnerAll)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::string strTmp4("127.0.0.1:11005");
std::string strTmp5("127.0.0.1:11006");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog6;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir15", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos5= new Paxos(timeout, rlog);
paxos5->initAsLearner(strTmp4);
rlog6= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir16", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos6= new Paxos(timeout, rlog);
paxos6->initAsLearner(strTmp5);
const Paxos::StatsType &stats= paxos2->getStats();
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
strConfig.push_back(strTmp4);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
std::vector<Paxos::ClusterInfoType> cis;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00"));
le.set_index(0);
le.set_optype(0);
le.set_value("aaa");
paxos1->replicateLog(le);
sleep(1);
auto lli= paxos1->getLastLogIndex();
le.set_optype(kMock);
for (uint i= 2; i <= lli-1; ++i)
{
le.set_index(2);
rlog6->append(le);
}
le.set_optype(0);
EXPECT_EQ(lli-1, rlog6->getLastLogIndex());
strConfig.clear();
strConfig.push_back(strTmp5);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(2);
EXPECT_EQ(paxos6->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00;127.0.0.1:11006$00"));
/* for old added learner to sync meta */
std::vector<std::string> tmp;
paxos1->changeLearners(Paxos::CCSyncLearnerAll, tmp);
sleep(2);
EXPECT_EQ(paxos6->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00;127.0.0.1:11006$00"));
/*
paxos1->configureLearner(100, 2);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$02;127.0.0.1:11005$00"));
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string("127.0.0.1:11004$02;127.0.0.1:11005$00"));
EXPECT_EQ(paxos2->getConfig()->getServer(100)->learnerSource, 2);
paxos2->clearStats();
le.set_index(0);
le.set_optype(0);
le.set_value("aaa");
paxos1->replicateLog(le);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(2);
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog<< " countMsgRequestVote:"<<stats.countMsgRequestVote<< " countOnMsgAppendLog:"<< stats.countOnMsgAppendLog<< " countHeartbeat:"<< stats.countHeartbeat << " countOnMsgRequestVote:"<<stats.countOnMsgRequestVote<< " countOnHeartbeat:"<<stats.countOnHeartbeat<< " countReplicateLog:"<<stats.countReplicateLog<< std::endl<< std::flush;
EXPECT_EQ(stats.countMsgAppendLog, 2);
EXPECT_EQ(paxos4->getCurrentLeader(), 2);
EXPECT_EQ(paxos5->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
EXPECT_EQ(paxos5->getLastLogIndex(), paxos1->getLastLogIndex());
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(cis[3].learnerSource, 2);
EXPECT_EQ(cis[4].learnerSource, 0);
EXPECT_EQ(cis[0].learnerSource, 0);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_TRUE(cis[3].matchIndex <= cis[1].matchIndex);
EXPECT_TRUE(cis[3].matchIndex >= cis[1].matchIndex - 2);
EXPECT_TRUE(cis[3].nextIndex <= cis[1].nextIndex);
EXPECT_TRUE(cis[3].nextIndex >= cis[1].nextIndex - 2);
paxos1->configureLearner(100, 3);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(2);
EXPECT_EQ(paxos4->getCurrentLeader(), 3);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_TRUE(cis[3].matchIndex <= cis[1].matchIndex);
EXPECT_TRUE(cis[3].matchIndex >= cis[1].matchIndex - 2);
EXPECT_TRUE(cis[3].nextIndex <= cis[1].nextIndex);
EXPECT_TRUE(cis[3].nextIndex >= cis[1].nextIndex - 2);
paxos1->configureLearner(100, 0);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
// Already fixed! Error may happy when server 1 is leader, and we change learnerSource from 1 to 0.
paxos1->configureLearner(100, 1);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
paxos1->configureLearner(100, 0);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 1);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
// set the learnerSource to another learner
paxos1->configureLearner(100, 101);
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos4->getCurrentLeader(), 101);
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
uint64_t tmpInt= 0;
std::cout<<"Cluster Info for server "<<paxos1->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
tmpInt= cis[0].matchIndex;
std::cout<<"Cluster Info for server "<<paxos5->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos5->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(tmpInt, cis[1].matchIndex);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
std::cout<<"Cluster Info for server "<<paxos1->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_LE(tmpInt, cis[3].matchIndex);
std::cout<<"Cluster Info for server "<<paxos5->getLocalServer()->serverId<<std::endl<<std::flush;
cis.clear();
paxos5->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
// check if the new added member has the forceSync electionWeight and learnerSource info.
paxos1->configureMember(2, false, 8);
sleep(1);
paxos1->configureLearner(100, 2);
sleep(1);
strConfig.clear();
strConfig.push_back(strTmp5);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
paxos1->changeMember(Paxos::CCAddNode, strTmp5);
sleep(1);
EXPECT_EQ(paxos6->getCurrentLeader(), 1);
EXPECT_EQ(paxos6->getLastLogIndex(), paxos1->getLastLogIndex());
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
cis.clear();
paxos6->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(cis[1].electionWeight, 8);
EXPECT_EQ(cis[4].learnerSource, 2);
*/
delete paxos6;
delete paxos5;
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
deleteDir("paxosLogTestDir15");
deleteDir("paxosLogTestDir16");
}
TEST(consensus, cctimeout)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->setConfigureChangeTimeout(1000);
EXPECT_EQ(paxos1->changeLearners(Paxos::CCAddNode, strConfig), 0);
delete paxos2;
delete paxos3;
easy_error_log("start changeMember");
EXPECT_EQ(paxos1->changeMember(Paxos::CCAddNode, strTmp3), PaxosErrorCode::PE_TIMEOUT);
easy_error_log("changeMember timeout");
delete paxos1;
delete paxos4;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
}
static void msleep(uint64_t t)
{
struct timeval sleeptime;
if (t == 0)
return;
sleeptime.tv_sec= t / 1000;
sleeptime.tv_usec= (t - (sleeptime.tv_sec * 1000)) * 1000;
select(0, 0, 0, 0, &sleeptime);
}
TEST(consensus, learnerSource4)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::string strTmp4("127.0.0.1:11005");
std::string strTmp5("127.0.0.1:11006");
std::string strTmp6("127.0.0.1:11007");
std::string strTmp7("127.0.0.1:11008");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir15", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos5= new Paxos(timeout, rlog);
paxos5->initAsLearner(strTmp4);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir16", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos6= new Paxos(timeout, rlog);
paxos6->initAsLearner(strTmp5);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir17", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos7= new Paxos(timeout, rlog);
paxos7->initAsLearner(strTmp6);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir18", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos8= new Paxos(timeout, rlog);
paxos8->initAsLearner(strTmp7);
le.set_optype(0);
paxos2->setMaxMergeReportTimeout(500);
paxos4->setMaxMergeReportTimeout(500);
paxos5->setMaxMergeReportTimeout(500);
paxos6->setMaxMergeReportTimeout(500);
paxos7->setMaxMergeReportTimeout(500);
paxos8->setMaxMergeReportTimeout(500);
const Paxos::StatsType &stats= paxos2->getStats();
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
strConfig.push_back(strTmp4);
strConfig.push_back(strTmp5);
strConfig.push_back(strTmp6);
strConfig.push_back(strTmp7);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00;127.0.0.1:11005$00;127.0.0.1:11006$00;127.0.0.1:11007$00;127.0.0.1:11008$00"));
paxos1->configureLearner(100, 2);
sleep(1);
paxos1->configureLearner(101, 100);
sleep(1);
paxos1->configureLearner(102, 101);
sleep(1);
paxos1->configureLearner(103, 102);
sleep(1);
paxos1->configureLearner(104, 103);
sleep(2);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$02;127.0.0.1:11005$:0;127.0.0.1:11006$:1;127.0.0.1:11007$:2;127.0.0.1:11008$:3"));
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string("127.0.0.1:11004$02;127.0.0.1:11005$:0;127.0.0.1:11006$:1;127.0.0.1:11007$:2;127.0.0.1:11008$:3"));
EXPECT_EQ(paxos2->getConfig()->getServer(100)->learnerSource, 2);
EXPECT_EQ(paxos2->getConfig()->getServer(101)->learnerSource, 100);
EXPECT_EQ(paxos2->getConfig()->getServer(102)->learnerSource, 101);
EXPECT_EQ(paxos2->getConfig()->getServer(103)->learnerSource, 102);
EXPECT_EQ(paxos2->getConfig()->getServer(104)->learnerSource, 103);
paxos2->clearStats();
std::vector<Paxos::ClusterInfoType> cis;
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(cis[3].learnerSource, 2);
EXPECT_EQ(cis[4].learnerSource, 100);
EXPECT_EQ(cis[0].learnerSource, 0);
auto index= paxos1->getLastLogIndex();
le.set_index(0);
le.set_optype(0);
le.set_value("aaa");
paxos1->replicateLog(le);
for (uint64_t i= 0; i<14; ++i)
{
le.clear_term();
paxos1->replicateLog(le);
msleep(200);
}
sleep(2);
std::cout<< "paxos1:"<<std::endl<< std::flush;
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos2:"<<std::endl<< std::flush;
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos4(100):"<<std::endl<< std::flush;
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos5(101):"<<std::endl<< std::flush;
paxos5->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos6(102):"<<std::endl<< std::flush;
paxos6->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos7(103):"<<std::endl<< std::flush;
paxos7->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(index+15, paxos1->getLastLogIndex());
EXPECT_EQ(index+15, paxos4->getLastLogIndex());
EXPECT_EQ(index+15, paxos5->getLastLogIndex());
//change learnersource
paxos1->configureLearner(103, 2);
sleep(1);
EXPECT_EQ(paxos2->getConfig()->getServer(103)->learnerSource, 2);
index= paxos1->getLastLogIndex();
le.set_index(0);
le.set_optype(0);
le.set_value("aaa");
paxos1->replicateLog(le);
for (uint64_t i= 0; i<14; ++i)
{
le.clear_term();
paxos1->replicateLog(le);
msleep(200);
}
sleep(2);
std::cout<< "paxos1:"<<std::endl<< std::flush;
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos2:"<<std::endl<< std::flush;
paxos2->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos4(100):"<<std::endl<< std::flush;
paxos4->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos5(101):"<<std::endl<< std::flush;
paxos5->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos6(102):"<<std::endl<< std::flush;
paxos6->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
std::cout<< "paxos7(103):"<<std::endl<< std::flush;
paxos7->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(index+15, paxos1->getLastLogIndex());
EXPECT_EQ(index+15, paxos4->getLastLogIndex());
EXPECT_EQ(index+15, paxos5->getLastLogIndex());
delete paxos8;
delete paxos7;
delete paxos6;
delete paxos5;
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
deleteDir("paxosLogTestDir15");
deleteDir("paxosLogTestDir16");
deleteDir("paxosLogTestDir17");
deleteDir("paxosLogTestDir18");
}
/**
* bug: concurrent configureChange may clear other's flag,
* cause later configureChange hang
*/
TEST(consensus, concurrent_cc)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
std::thread op1([paxos1](){
std::vector<std::string> strConfig;
strConfig.push_back("127.0.0.1:11004");
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
});
std::thread op2([paxos1](){
std::vector<std::string> strConfig;
strConfig.push_back("127.0.0.1:11005");
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
});
std::thread op3([paxos1](){
std::vector<std::string> strConfig;
strConfig.push_back("127.0.0.1:11006");
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
});
op1.join();
op2.join();
op3.join();
paxos1->setConfigureChangeTimeout(5*1000); //5s
delete paxos2;
delete paxos3;
std::thread op4([paxos1](){
std::vector<std::string> strConfig;
strConfig.push_back("127.0.0.1:11007");
paxos1->changeLearners(Paxos::CCAddNode, strConfig); // timeout
});
std::thread op5([paxos1](){
std::vector<std::string> strConfig;
strConfig.push_back("127.0.0.1:11008");
paxos1->changeLearners(Paxos::CCAddNode, strConfig); // timeout
});
op4.join();
op5.join();
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
}
TEST(consensus, learner_downgrade_term)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog4;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
le.set_optype(0);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
sleep(1);
le.clear_term();
paxos1->replicateLog(le);
cis.clear();
paxos4->getClusterInfo(cis);
sleep(1);
Paxos::printClusterInfo(cis);
std::cout<< "11004 restart"<< std::endl<< std::flush;
delete paxos4;
sleep(2);
//force to give learner a larger term
uint64_t cterm1 = 0, cterm2 = 0;
rlog4->getMetaData(std::string(Paxos::keyCurrentTerm), &cterm1);
rlog1->getMetaData(std::string(Paxos::keyCurrentTerm), &cterm2);
EXPECT_EQ(cterm1, cterm2);
rlog4->setMetaData(std::string(Paxos::keyCurrentTerm), cterm1 + 10);
paxos4= new Paxos(timeout, rlog4);
std::string emptryString("");
paxos4->initAsLearner(emptryString);
sleep(2);
rlog4->getMetaData(std::string(Paxos::keyCurrentTerm), &cterm1);
// check term downgrade successfully
EXPECT_EQ(cterm1, cterm2);
le.clear_term();
paxos1->replicateLog(le);
sleep(1);
// check log replicate is normal
EXPECT_EQ(paxos4->getLastLogIndex(), paxos1->getLastLogIndex());
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
}
TEST(consensus, learner_heartbeat)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
std::string strTmp3("127.0.0.1:11004");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 5000;
std::shared_ptr<PaxosLog> rlog1, rlog4;
rlog1= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog1->append(le);
Paxos *paxos1= new Paxos(timeout, rlog1);
paxos1->init(strConfig, 1);
rlog4= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog4->append(le);
Paxos *paxos4= new Paxos(timeout, rlog4);
paxos4->initAsLearner(strTmp3);
le.set_optype(0);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
/* now we have 1 leader and 1 learner*/
const Paxos::StatsType &stats= paxos1->getStats();
uint64_t tmp = 0;
sleep(2);
EXPECT_EQ(stats.countHeartbeat, 0);
paxos1->changeMember(Paxos::CCAddNode, strConfig[0]);
while (rlog1->getLastLogIndex() != rlog4->getLastLogIndex()) {}
tmp = stats.countHeartbeat;
sleep(2);
EXPECT_GT(stats.countHeartbeat, tmp);
paxos1->downgradeMember(strConfig[0]);
while (rlog1->getLastLogIndex() != rlog4->getLastLogIndex()) {}
tmp = stats.countHeartbeat;
sleep(2);
EXPECT_EQ(stats.countHeartbeat, tmp);
paxos1->changeLearners(Paxos::CCDelNode, strConfig);
paxos1->setEnableLearnerHeartbeat(true);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
while (rlog1->getLastLogIndex() != rlog4->getLastLogIndex()) {}
tmp = stats.countHeartbeat;
sleep(2);
EXPECT_GT(stats.countHeartbeat, tmp);
paxos1->changeMember(Paxos::CCAddNode, strConfig[0]);
while (rlog1->getLastLogIndex() != rlog4->getLastLogIndex()) {}
tmp = stats.countHeartbeat;
sleep(2);
EXPECT_GT(stats.countHeartbeat, tmp);
paxos1->downgradeMember(strConfig[0]);
while (rlog1->getLastLogIndex() != rlog4->getLastLogIndex()) {}
tmp = stats.countHeartbeat;
sleep(2);
EXPECT_GT(stats.countHeartbeat, tmp);
delete paxos4;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir14");
}
/* change learnerSource use the address as argument */
TEST(consensus, learnersource_addr)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kMock);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog4;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
le.set_optype(0);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
// cannot configure follower
EXPECT_EQ(paxos1->configureLearner("127.0.0.1:11002", "127.0.0.1:11002"), PaxosErrorCode::PE_NOTFOUND);
// cannot configure not-exist node
EXPECT_EQ(paxos1->configureLearner("127.0.0.1:11005", "127.0.0.1:11002"), PaxosErrorCode::PE_NOTFOUND);
EXPECT_EQ(paxos1->configureLearner("127.0.0.1:11004", "127.0.0.1:11005"), PaxosErrorCode::PE_NOTFOUND);
// success case
EXPECT_EQ(paxos1->configureLearner("127.0.0.1:11004", "127.0.0.1:11003"), PaxosErrorCode::PE_NONE);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$03"));
// reset to 0
EXPECT_EQ(paxos1->configureLearner("127.0.0.1:11004", "127.0.0.1:11004"), PaxosErrorCode::PE_NONE);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
}
TEST(consensus, enable_learner_auto_reset_match_index)
{
std::vector<std::string> strConfig;
LogEntry le;
le.set_term(1);
le.set_index(1);
le.set_optype(kNormal);
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog2, rlog4;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
// We init log with mock log in index 1 to make error happend when we send log entry 1 to learner!
rlog->append(le);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog2= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
rlog->append(le);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
le.set_optype(0);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
EXPECT_EQ(paxos4->getLastLogIndex(), 3);
std::cout<< "shutdown learner"<< std::endl;
delete paxos4;
sleep(1);
paxos2->requestVote(true);
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
EXPECT_EQ(paxos2->getLastLogIndex(), 4);
// new leader's index 4 term should be 3
{
LogEntry le;
rlog2->getEntry(4, le);
EXPECT_EQ(le.term(), 3);
}
// learner's index 4 term should be 2
{
LogEntry le;
le.set_term(2);
le.set_optype(kNormal);
rlog4->append(le);
rlog4->getEntry(4, le);
EXPECT_EQ(le.term(), 2);
}
{
LogEntry le;
le.set_term(2);
le.set_optype(kNormal);
rlog4->append(le);
rlog4->getEntry(5, le);
EXPECT_EQ(le.term(), 2);
}
paxos2->changeLearners(Paxos::CCDelNode, strConfig);
sleep(1);
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string(""));
paxos2->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
EXPECT_EQ(paxos2->getLastLogIndex(), 6);
std::cout<< "restart learner"<< std::endl;
paxos4= new Paxos(timeout, rlog4);
paxos4->initAsLearner(strTmp3);
sleep(3);
EXPECT_EQ(paxos2->getEnableLearnerAutoResetMatchIndex(), false);
// log can not be replicated
{
LogEntry le;
rlog4->getEntry(4, le);
EXPECT_EQ(le.term(), 2);
}
paxos2->setEnableLearnerAutoResetMatchIndex(true);
EXPECT_EQ(paxos2->getEnableLearnerAutoResetMatchIndex(), true);
{
LogEntry le;
le.set_term(3);
le.set_optype(kNormal);
paxos2->replicateLog(le);
}
sleep(5);
// log can be replicated
EXPECT_EQ(paxos4->getLastLogIndex(), 7);
{
LogEntry le;
rlog4->getEntry(4, le);
EXPECT_EQ(le.term(), 3);
}
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
}
// test force single learner
// after a node becomes a single learner, it can be added back to a new cluster,
// and its log position will be automatically matched.
TEST(consensus, force_single_learner)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
std::string strTmp3("127.0.0.1:11004");
std::vector<Paxos::ClusterInfoType> cis;
const uint64_t timeout= 5000;
std::shared_ptr<PaxosLog> rlog, rlog1, rlog2, rlog4;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog2= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
auto localServer = std::make_shared<alisql::LocalServer>(0);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, 0, 4, 4, localServer);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
rlog4= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir14", true, 4 * 1024 * 1024);
Paxos *paxos4= new Paxos(timeout, rlog);
paxos4->initAsLearner(strTmp3);
paxos1->setEnableLearnerAutoResetMatchIndex(true);
paxos2->setEnableLearnerAutoResetMatchIndex(true);
paxos3->setEnableLearnerAutoResetMatchIndex(true);
paxos4->setEnableLearnerAutoResetMatchIndex(true);
// get standard learner meta info
EXPECT_EQ(paxos4->getConfig()->membersToString(), std::string("127.0.0.1:11004#5N"));
EXPECT_EQ(paxos4->getConfig()->learnersToString(), std::string(""));
sleep(1);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(3);
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11001")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@1"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
EXPECT_EQ(paxos2->getCommitIndex(), 2);
EXPECT_EQ(paxos3->getCommitIndex(), 2);
EXPECT_EQ(paxos4->getCommitIndex(), 2);
// does not work for learner
EXPECT_EQ(paxos4->forceSingleLearner(), 1);
EXPECT_EQ(paxos4->getConfig()->membersToString(), std::string("127.0.0.1:11004#5N"));
EXPECT_EQ(paxos4->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
// make learner single leader
paxos4->forceSingleLeader();
paxos4->requestVote();
LogEntry le;
le.set_optype(0);
le.set_ikey("a");
paxos4->replicateLog(le);
EXPECT_EQ(paxos4->getCommitIndex(), 4);
{
LogEntry le;
rlog4->getEntry(3, le);
EXPECT_EQ(le.term(), 3);
}
le.set_ikey("b");
paxos1->replicateLog(le);
sleep(3);
EXPECT_EQ(paxos1->getCommitIndex(), 3);
EXPECT_EQ(paxos2->getCommitIndex(), 3);
EXPECT_EQ(paxos3->getCommitIndex(), 3);
{
LogEntry le;
rlog1->getEntry(3, le);
EXPECT_EQ(le.term(), 2);
}
{
LogEntry le;
rlog2->getEntry(3, le);
EXPECT_EQ(le.term(), 2);
}
// generate bigger term log to test learner auto reset match index
paxos2->requestVote();
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
paxos1->requestVote();
sleep(3);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
EXPECT_EQ(paxos1->getCommitIndex(), 5);
EXPECT_EQ(paxos2->getCommitIndex(), 5);
EXPECT_EQ(paxos3->getCommitIndex(), 5);
// change follower to single learner
EXPECT_EQ(paxos2->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
EXPECT_EQ(paxos2->forceSingleLearner(), 0);
EXPECT_EQ(paxos2->getConfig()->membersToString(), std::string("0;127.0.0.1:11002#5N"));
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string(""));
// change leader to single learner
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11001")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@1"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
EXPECT_EQ(paxos1->forceSingleLearner(), 0);
EXPECT_EQ(paxos1->getConfig()->membersToString(), std::string("127.0.0.1:11001#5N"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string(""));
// add paxos 1 as learner
strConfig.clear();
strConfig.push_back("127.0.0.1:11001");
paxos4->changeLearners(Paxos::CCAddNode, strConfig);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 5);
EXPECT_EQ(paxos1->getCommitIndex(), 5);
EXPECT_EQ(paxos4->getConfig()->membersToString(("127.0.0.1:11004")), std::string("127.0.0.1:11004#5N@1"));
EXPECT_EQ(paxos4->getConfig()->learnersToString(), std::string("127.0.0.1:11001$00"));
EXPECT_EQ(paxos1->getConfig()->membersToString(), std::string("127.0.0.1:11001#5N"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11001$00"));
le.set_optype(0);
paxos4->replicateLog(le);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 6);
EXPECT_EQ(paxos1->getCommitIndex(), 6);
// add paxos 2 as learner
strConfig.clear();
strConfig.push_back("127.0.0.1:11002");
paxos4->changeLearners(Paxos::CCAddNode, strConfig);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 7);
EXPECT_EQ(paxos2->getCommitIndex(), 7);
EXPECT_EQ(paxos4->getConfig()->membersToString(("127.0.0.1:11004")), std::string("127.0.0.1:11004#5N@1"));
EXPECT_EQ(paxos4->getConfig()->learnersToString(), std::string("127.0.0.1:11001$00;127.0.0.1:11002$00"));
EXPECT_EQ(paxos2->getConfig()->membersToString(), std::string("0;127.0.0.1:11002#5N"));
EXPECT_EQ(paxos2->getConfig()->learnersToString(), std::string("127.0.0.1:11001$00;127.0.0.1:11002$00"));
le.set_optype(0);
paxos4->replicateLog(le);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 8);
EXPECT_EQ(paxos2->getCommitIndex(), 8);
{
// upgrade paxos 1 to follower
std::string s("127.0.0.1:11001");
paxos4->changeMember(Paxos::CCAddNode, s);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 9);
EXPECT_EQ(paxos1->getCommitIndex(), 9);
EXPECT_EQ(paxos4->getConfig()->membersToString(("127.0.0.1:11004")), std::string("127.0.0.1:11004#5N;127.0.0.1:11001#5N@1"));
EXPECT_EQ(paxos4->getConfig()->learnersToString(), std::string("0;127.0.0.1:11002$00"));
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11001")), std::string("127.0.0.1:11004#5N;127.0.0.1:11001#5N@2"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("0;127.0.0.1:11002$00"));
}
le.set_optype(0);
paxos4->replicateLog(le);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 10);
EXPECT_EQ(paxos2->getCommitIndex(), 10);
{
// upgrade paxos 2 to follower
std::string s("127.0.0.1:11002");
paxos4->changeMember(Paxos::CCAddNode, s);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 11);
EXPECT_EQ(paxos2->getCommitIndex(), 11);
EXPECT_EQ(paxos4->getConfig()->membersToString(("127.0.0.1:11004")), std::string("127.0.0.1:11004#5N;127.0.0.1:11001#5N;127.0.0.1:11002#5N@1"));
EXPECT_EQ(paxos4->getConfig()->learnersToString(), std::string(""));
EXPECT_EQ(paxos1->getConfig()->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11004#5N;127.0.0.1:11001#5N;127.0.0.1:11002#5N@3"));
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string(""));
}
le.set_optype(0);
paxos4->replicateLog(le);
sleep(3);
EXPECT_EQ(paxos4->getCommitIndex(), 12);
EXPECT_EQ(paxos2->getCommitIndex(), 12);
// since this function will break the protocol,
// logs might be different on every node
{
LogEntry le;
rlog1->getEntry(3, le);
EXPECT_TRUE(le.term() == 3 || le.term() == 2);
}
{
LogEntry le;
rlog2->getEntry(3, le);
EXPECT_TRUE(le.term() == 3 || le.term() == 2);
}
{
std::vector<Paxos::ClusterInfoType> cis;
paxos4->getClusterInfo(cis);
EXPECT_EQ(cis[2].matchIndex, 12);
}
{
paxos2->requestVote(true);
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
LogEntry le;
le.set_optype(0);
le.set_term(paxos2->getTerm());
localServer->writeLog(le);
sleep(2);
std::vector<Paxos::ClusterInfoType> cis;
paxos2->getClusterInfo(cis);
EXPECT_EQ(paxos2->getCommitIndex(), 14);
// if bug exists, match index will be 13
EXPECT_EQ(cis[2].matchIndex, 14);
}
delete paxos4;
delete paxos3;
delete paxos2;
delete paxos1;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
deleteDir("paxosLogTestDir14");
}
Reference in New Issue View Git Blame Copy Permalink