4493d40a-92aa-4162-9dfc-bfc4ec0996e6
/
polardbxengine
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
polardbxengine/extra/IS/consensus/unittest/consensus-t.cc

1654 lines
53 KiB
C++
Raw Blame History

/************************************************************************
*
* Copyright (c) 2016 Alibaba.com, Inc. All Rights Reserved
* $Id: consensus-t.cc,v 1.0 07/30/2016 02:41:21 PM yingqiang.zyq(yingqiang.zyq@alibaba-inc.com) $
*
************************************************************************/
/**
* @file consensus-t.cc
* @author yingqiang.zyq(yingqiang.zyq@alibaba-inc.com)
* @date 07/30/2016 02:41:21 PM
* @version 1.0
* @brief unit test for alisql::Consensus
*
**/
#include <atomic>
#include <thread>
#include <gtest/gtest.h>
#include "easyNet.h"
#include "service.h"
#include "paxos.h"
#include "paxos_log.h"
#include "paxos_server.h"
#include "paxos_configuration.h"
#include "paxos.pb.h"
#include "files.h"
#include "rd_paxos_log.h"
#include "file_paxos_log.h"
using namespace alisql;
TEST(consensus, helloworld)
{
EXPECT_TRUE(1);
}
TEST(consensus, StableConfiguration)
{
std::shared_ptr<StableConfiguration> config= std::shared_ptr<StableConfiguration>(new StableConfiguration());
EXPECT_TRUE(static_cast<bool>(config));
std::shared_ptr<RemoteServer> ptr;
std::shared_ptr<LocalServer> lptr;
config->servers.push_back(lptr= std::make_shared<LocalServer>(1));
lptr->strAddr= std::string("127.0.0.1:10001");
config->servers.push_back(ptr= std::make_shared<RemoteServer>(2));
ptr->matchIndex= 5;
ptr->strAddr= std::string("127.0.0.1:10002");
config->servers.push_back(ptr= std::make_shared<RemoteServer>(3));
ptr->strAddr= std::string("127.0.0.1:10003");
ptr->matchIndex= 6;
uint64_t qmin= config->quorumMin(&Server::getMatchIndex);
EXPECT_EQ(qmin, 5);
config->delMember(std::string("127.0.0.1:10002"), NULL);
EXPECT_EQ(config->getServerNum(), 2);
EXPECT_EQ(config->servers[0]->serverId, 1);
EXPECT_EQ(config->servers[0]->strAddr, std::string("127.0.0.1:10001"));
EXPECT_EQ(config->servers[2]->serverId, 3);
EXPECT_EQ(config->servers[2]->strAddr, std::string("127.0.0.1:10003"));
//EXPECT_EQ(config->membersToString(lptr->strAddr), std::string("127.0.0.1:10001#5N;127.0.0.1:10003#5N@1"));
EXPECT_EQ(config->membersToString(lptr->strAddr), std::string("127.0.0.1:10001#5N;0;127.0.0.1:10003#5N@1"));
config->configureMember(1, true, 9, NULL);
//EXPECT_EQ(config->membersToString(lptr->strAddr), std::string("127.0.0.1:10001#9F;127.0.0.1:10003#5N@1"));
EXPECT_EQ(config->membersToString(lptr->strAddr), std::string("127.0.0.1:10001#9F;0;127.0.0.1:10003#5N@1"));
std::string strConfig= "127.0.0.1:10001;127.0.0.1:10002;127.0.0.1:10003@1";
uint64_t index= 0;
std::vector<std::string> v= StableConfiguration::stringToVector(strConfig, index);
EXPECT_EQ(v[0], "127.0.0.1:10001");
EXPECT_EQ(v[1], "127.0.0.1:10002");
EXPECT_EQ(v[2], "127.0.0.1:10003");
EXPECT_EQ(index, 1);
//for Learner string
strConfig= "127.0.0.1:10001;127.0.0.1:10002";
v= StableConfiguration::stringToVector(strConfig, index);
EXPECT_EQ(v[0], "127.0.0.1:10001");
EXPECT_EQ(v[1], "127.0.0.1:10002");
EXPECT_EQ(std::string("127.0.0.1:10001"), StableConfiguration::getAddr(std::string("127.0.0.1:10001#5N")));
EXPECT_EQ(std::string("127.0.0.1:10001"), StableConfiguration::getAddr(std::string("127.0.0.1:10001$:2")));
strConfig= "127.0.0.1:11004$00;127.0.0.1:11005$00;0;127.0.0.1:11006$00";
v= StableConfiguration::stringToVector(strConfig, index);
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11005"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11005$02"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11005#5N"), v));
EXPECT_FALSE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11001"), v));
EXPECT_FALSE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11001#5N"), v));
// test learnerSource larger than 99 and 109 and 159
strConfig= "127.0.0.1:11004$:1;127.0.0.1:11005$;2;0;127.0.0.1:11006$00;127.0.0.1:11007$@3";
index = 0;
v= StableConfiguration::stringToVector(strConfig, index);
EXPECT_TRUE(index == 0);
EXPECT_TRUE(v.size() == 5);
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11005"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11005$;2"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11007$@3"), v));
strConfig= "127.0.0.1:11004$00;127.0.0.1:11005$00;0;127.0.0.1:11006$00;127.0.0.1:11007$;0";
index = 0;
v= StableConfiguration::stringToVector(strConfig, index);
EXPECT_TRUE(index == 0);
EXPECT_EQ(v.size(), 5);
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11007"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11007$;0"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11004"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11004$00"), v));
EXPECT_EQ(v[0], "127.0.0.1:11004$00");
EXPECT_EQ(v[4], "127.0.0.1:11007$;0");
strConfig= "0;127.0.0.1:11004$00;127.0.0.1:11005$00;0;127.0.0.1:11006$00;127.0.0.1:11007$;0";
index = 0;
v= StableConfiguration::stringToVector(strConfig, index);
EXPECT_TRUE(index == 0);
EXPECT_EQ(v.size(), 6);
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11007"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11007$;0"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11004"), v));
EXPECT_TRUE(StableConfiguration::isServerInVector(std::string("127.0.0.1:11004$00"), v));
EXPECT_EQ(v[0], "0");
EXPECT_EQ(v[5], "127.0.0.1:11007$;0");
}
TEST(consensus, StableConfiguration_uncontinue_serverid)
{
std::string strConfig;
uint64_t index= 0;
std::shared_ptr<StableConfiguration> config= std::shared_ptr<StableConfiguration>(new StableConfiguration());
EXPECT_TRUE(static_cast<bool>(config));
std::shared_ptr<RemoteServer> ptr;
std::shared_ptr<LocalServer> lptr;
config->servers.push_back(lptr= std::make_shared<LocalServer>(1));
lptr->strAddr= std::string("127.0.0.1:10001");
config->servers.push_back(nullptr);
config->servers.push_back(ptr= std::make_shared<RemoteServer>(3));
ptr->matchIndex= 5;
ptr->strAddr= std::string("127.0.0.1:10002");
config->servers.push_back(ptr= std::make_shared<RemoteServer>(4));
ptr->strAddr= std::string("127.0.0.1:10003");
ptr->matchIndex= 6;
uint64_t qmin= config->quorumMin(&Server::getMatchIndex);
EXPECT_EQ(qmin, 5);
config->delMember(std::string("127.0.0.1:10002"), NULL);
EXPECT_EQ(config->getServerNum(), 2);
EXPECT_EQ(config->servers[0]->serverId, 1);
EXPECT_EQ(config->servers[0]->strAddr, std::string("127.0.0.1:10001"));
EXPECT_EQ(config->servers[3]->serverId, 4);
EXPECT_EQ(config->servers[3]->strAddr, std::string("127.0.0.1:10003"));
//EXPECT_EQ(config->membersToString(lptr->strAddr), std::string("127.0.0.1:10001#5N;127.0.0.1:10003#5N@1"));
EXPECT_EQ(config->membersToString(lptr->strAddr), std::string("127.0.0.1:10001#5N;0;0;127.0.0.1:10003#5N@1"));
/*
config->configureMember(1, true, 9, NULL);
EXPECT_EQ(config->membersToString(lptr->strAddr), std::string("127.0.0.1:10001#9F;127.0.0.1:10003#5N@1"));
strConfig= "127.0.0.1:10001;127.0.0.1:10002;127.0.0.1:10003@1";
std::vector<std::string> v= StableConfiguration::stringToVector(strConfig, index);
EXPECT_EQ(v[0], "127.0.0.1:10001");
EXPECT_EQ(v[1], "127.0.0.1:10002");
EXPECT_EQ(v[2], "127.0.0.1:10003");
EXPECT_EQ(index, 1);
*/
//for Learner string
strConfig= "127.0.0.1:10001;0;127.0.0.1:10002";
auto v1= StableConfiguration::stringToVector(strConfig, index);
EXPECT_EQ(v1[0], "127.0.0.1:10001");
EXPECT_EQ(v1[1], "0");
EXPECT_EQ(v1[2], "127.0.0.1:10002");
}
TEST(consensus, PaxosLog)
{
// TODO need a factory.
//PaxosLog *rlog= new PaxosLog();
PaxosLog *rlog= (PaxosLog *) new RDPaxosLog("paxosLogTestDir", true, 4 * 1024 * 1024);
LogEntry entry;
EXPECT_EQ(rlog->getLastLogIndex(), 0);
rlog->getEntry(0, entry);
EXPECT_EQ(entry.term(), 0);
EXPECT_EQ(entry.index(), 0);
entry.set_term(1);
entry.set_index(1);
entry.set_optype(1);
entry.set_ikey("aa");
entry.set_value("bb");
rlog->append(entry);
entry.Clear();
rlog->getEntry(1, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 1);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 1);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa");
EXPECT_EQ(entry.value(), "bb");
entry.set_term(1);
entry.set_index(2);
entry.set_optype(1);
entry.set_ikey("aa1");
entry.set_value("bb1");
rlog->append(entry);
entry.set_term(1);
entry.set_index(3);
entry.set_optype(1);
entry.set_ikey("aa2");
entry.set_value("bb2");
rlog->append(entry);
entry.Clear();
rlog->getEntry(3, entry);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 3);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa2");
EXPECT_EQ(entry.value(), "bb2");
rlog->truncateBackward(2);
EXPECT_EQ(rlog->getLastLogIndex(), 1);
entry.Clear();
rlog->getEntry(1, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 1);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 1);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa");
EXPECT_EQ(entry.value(), "bb");
rlog->truncateBackward(1);
EXPECT_EQ(rlog->getLastLogIndex(), 0);
// Patch entries append API
::google::protobuf::RepeatedPtrField<LogEntry> entries;
entry.set_term(1);
entry.set_index(1);
entry.set_optype(1);
entry.set_ikey("aa");
entry.set_value("bb");
*(entries.Add())= entry;
entry.set_term(2);
entry.set_index(2);
entry.set_optype(1);
entry.set_ikey("bb");
entry.set_value("cc");
*(entries.Add())= entry;
EXPECT_EQ(2, rlog->append(entries));
rlog->getEntry(1, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 2);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 1);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa");
EXPECT_EQ(entry.value(), "bb");
rlog->getEntry(2, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 2);
EXPECT_EQ(entry.term(), 2);
EXPECT_EQ(entry.index(), 2);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "bb");
EXPECT_EQ(entry.value(), "cc");
delete rlog;
deleteDir("paxosLogTestDir");
}
TEST(consensus, FilePaxosLog)
{
//PaxosLog *rlog= new PaxosLog();
PaxosLog *rlog= (PaxosLog *) new FilePaxosLog("filelog", FilePaxosLog::LTSync);
LogEntry entry;
EXPECT_EQ(rlog->getLastLogIndex(), 0);
rlog->getEntry(0, entry);
EXPECT_EQ(entry.term(), 0);
EXPECT_EQ(entry.index(), 0);
entry.set_term(1);
entry.set_index(1);
entry.set_optype(1);
entry.set_ikey("aa");
entry.set_value("bb");
rlog->append(entry);
entry.Clear();
rlog->getEntry(1, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 1);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 1);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa");
EXPECT_EQ(entry.value(), "bb");
entry.set_term(1);
entry.set_index(2);
entry.set_optype(1);
entry.set_ikey("aa1");
entry.set_value("bb1");
rlog->append(entry);
entry.set_term(1);
entry.set_index(3);
entry.set_optype(1);
entry.set_ikey("aa2");
entry.set_value("bb2");
rlog->append(entry);
entry.Clear();
rlog->getEntry(3, entry);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 3);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa2");
EXPECT_EQ(entry.value(), "bb2");
rlog->truncateBackward(2);
EXPECT_EQ(rlog->getLastLogIndex(), 1);
entry.Clear();
rlog->getEntry(1, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 1);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 1);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa");
EXPECT_EQ(entry.value(), "bb");
rlog->truncateBackward(1);
EXPECT_EQ(rlog->getLastLogIndex(), 0);
// Patch entries append API
::google::protobuf::RepeatedPtrField<LogEntry> entries;
entry.set_term(1);
entry.set_index(1);
entry.set_optype(1);
entry.set_ikey("aa");
entry.set_value("bb");
*(entries.Add())= entry;
entry.set_term(2);
entry.set_index(2);
entry.set_optype(1);
entry.set_ikey("bb");
entry.set_value("cc");
*(entries.Add())= entry;
EXPECT_EQ(2, rlog->append(entries));
rlog->getEntry(1, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 2);
EXPECT_EQ(entry.term(), 1);
EXPECT_EQ(entry.index(), 1);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "aa");
EXPECT_EQ(entry.value(), "bb");
rlog->getEntry(2, entry);
EXPECT_EQ(rlog->getLastLogIndex(), 2);
EXPECT_EQ(entry.term(), 2);
EXPECT_EQ(entry.index(), 2);
EXPECT_EQ(entry.optype(), 1);
EXPECT_EQ(entry.ikey(), "bb");
EXPECT_EQ(entry.value(), "cc");
delete rlog;
deleteDir("filelog");
}
TEST(consensus, pbTest)
{
PaxosMsg msg;
msg.set_clusterid(0);
msg.set_msgid(0);
msg.set_serverid(0);
msg.set_msgtype(1);
msg.set_term(2);
msg.set_leaderid(3);
EXPECT_TRUE(msg.IsInitialized());
uint32_t length = uint32_t(msg.ByteSize());
length= length;
std::string buf;
msg.SerializeToString(&buf);
EXPECT_GT(buf.length(), 1);
PaxosMsg rcv;
rcv.ParseFromString(buf);
EXPECT_EQ(rcv.msgtype(), 1);
EXPECT_EQ(rcv.term(), 2);
EXPECT_TRUE(rcv.has_leaderid());
EXPECT_EQ(rcv.leaderid(), 3);
EXPECT_FALSE(rcv.has_prevlogindex());
}
TEST(consensus, RemoteServerSendMsg)
{
auto srv= std::shared_ptr<Service>(new Service(NULL));
EXPECT_TRUE(static_cast<bool>(srv));
srv->init();
srv->start(11001);
auto ptr= std::make_shared<RemoteServer>(1);
ptr->matchIndex= 5;
ptr->strAddr= std::string("127.0.0.1:11001");
ptr->srv= srv;
PaxosMsg msg;
msg.set_clusterid(0);
msg.set_msgid(0);
msg.set_serverid(0);
msg.set_msgtype(1);
msg.set_term(2);
msg.set_leaderid(3);
EXPECT_TRUE(msg.IsInitialized());
msg.clear_msgid();
ptr->connect(NULL);
sleep(0.5);
ptr->sendMsg(&msg);
sleep(0.5);
ptr.reset();
srv->shutdown();
}
TEST(consensus, Paxos_requestVote3)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog;
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
LogEntry le;
le.set_index(0);
le.set_optype(0);
le.set_ikey("bbb");
le.set_value("ccc");
paxos1->replicateLog(le);
// Test Learner
std::string strTmp("127.0.0.1:11004");
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir4", true, 4 * 1024 * 1024);
Paxos *learner1= new Paxos(timeout, rlog);
learner1->initAsLearner(strTmp);
sleep(1);
strConfig.clear();
strConfig.push_back(strTmp);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(2);
EXPECT_EQ(learner1->getLastLogIndex(), paxos1->getLastLogIndex());
le.clear_term();
paxos1->replicateLog(le);
sleep(2);
EXPECT_EQ(learner1->getLastLogIndex(), paxos1->getLastLogIndex());
//Transfer Leader to Server 2 success case
sleep(1);
paxos1->leaderTransfer(2);
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
//Test if the learners can still recieve msgs from new leader.
le.clear_term();
paxos2->replicateLog(le);
sleep(1);
EXPECT_EQ(learner1->getLastLogIndex(), paxos2->getLastLogIndex());
le.clear_term();
paxos2->replicateLog(le);
sleep(1);
EXPECT_EQ(learner1->getLastLogIndex(), paxos2->getLastLogIndex());
// Remove unexist Learner and add already exist.
strConfig.clear();
strConfig.emplace_back("127.0.0.1:11010");
EXPECT_EQ(paxos2->changeLearners(Paxos::CCDelNode, strConfig), PaxosErrorCode::PE_NOTFOUND);
strConfig.clear();
strConfig.push_back(strTmp);;
EXPECT_EQ(paxos2->changeLearners(Paxos::CCAddNode, strConfig), PaxosErrorCode::PE_EXISTS);
// Remove Learner
strConfig.clear();
strConfig.push_back(strTmp);
paxos2->changeLearners(Paxos::CCDelNode, strConfig);
sleep(2);
le.clear_term();
paxos2->replicateLog(le);
sleep(1);
EXPECT_LT(learner1->getLastLogIndex(), paxos2->getLastLogIndex());
// add member
// // create 2 learners
std::string strTmp2("127.0.0.1:11005");
std::string strTmp3("127.0.0.1:11006");
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir5", true, 4 * 1024 * 1024);
Paxos *learner2= new Paxos(timeout, rlog);
learner2->initAsLearner(strTmp2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir6", true, 4 * 1024 * 1024);
Paxos *learner3= new Paxos(timeout, rlog);
learner3->initAsLearner(strTmp3);
sleep(1);
// // add 2 learners
strConfig.clear();
strConfig.push_back(strTmp2);
strConfig.push_back(strTmp3);
paxos2->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
auto config= std::dynamic_pointer_cast<StableConfiguration>(paxos2->getConfig());
//auto& servers= config->getServers();
auto& learners= config->getLearners();
EXPECT_EQ(config->getServerNum(), 3);
EXPECT_EQ(learners.size(), 2);
EXPECT_EQ(learner2->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(learner3->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(config->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N@2"));
EXPECT_EQ(config->learnersToString(), std::string("127.0.0.1:11005$00;127.0.0.1:11006$00"));
// // change learners to followers one by one
paxos2->changeMember(Paxos::CCAddNode, strTmp2);
sleep(1);
le.clear_term();
paxos2->replicateLog(le);
sleep(1);
EXPECT_EQ(learner2->getState(), Paxos::FOLLOWER);
EXPECT_EQ(config->getServerNum(), 4);
EXPECT_EQ(learners.size(), 2);
EXPECT_EQ(config->getLearnerNum(), 1);
paxos2->changeMember(Paxos::CCAddNode, strTmp3);
sleep(1);
paxos2->configureMember(5, true, 9);
le.clear_term();
paxos2->replicateLog(le);
sleep(1);
EXPECT_EQ(learner3->getState(), Paxos::FOLLOWER);
EXPECT_EQ(config->getServerNum(), 5);
EXPECT_EQ(learners.size(), 0);
EXPECT_EQ(config->membersToString(("127.0.0.1:11002")), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N;127.0.0.1:11005#5N;127.0.0.1:11006#9F@2"));
EXPECT_EQ(config->learnersToString(), std::string(""));
EXPECT_EQ(learner3->getConfig()->membersToString(learner3->getLocalServer()->strAddr), std::string("127.0.0.1:11001#5N;127.0.0.1:11002#5N;127.0.0.1:11003#5N;127.0.0.1:11005#5N;127.0.0.1:11006#9F@5"));
// // check 2 new followers
EXPECT_EQ(learner2->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(learner3->getLastLogIndex(), paxos2->getLastLogIndex());
std::vector<Paxos::ClusterInfoType> cis;
paxos2->getClusterInfo(cis);
for (auto& ci : cis)
{
std::cout<< "serverId:"<< ci.serverId<< " ipPort:"<< ci.ipPort<< " matchIndex:"<< ci.matchIndex<< " nextIndex:"<< ci.nextIndex<< " role:"<< ci.role<< " hasVoted:"<< ci.hasVoted << " forceSync:" << ci.forceSync << " electionWeight: " << ci.electionWeight << std::endl<< std::flush;
}
EXPECT_EQ(cis[3].ipPort, strTmp2);
EXPECT_EQ(cis[3].forceSync, false);
EXPECT_EQ(cis[3].electionWeight, 5);
EXPECT_EQ(cis[4].ipPort, strTmp3);
EXPECT_EQ(cis[4].forceSync, true);
EXPECT_EQ(cis[4].electionWeight, 9);
// // delete one follower
paxos2->changeMember(Paxos::CCDelNode, strTmp2);
sleep(1);
le.clear_term();
paxos2->replicateLog(le);
sleep(1);
EXPECT_EQ(config->getServerNum(), 4);
EXPECT_EQ(learners.size(), 0);
EXPECT_EQ(config->learnersToString(), std::string(""));
EXPECT_LT(learner2->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(learner3->getLastLogIndex(), paxos2->getLastLogIndex());
paxos2->getClusterInfo(cis);
for (auto& ci : cis)
{
std::cout<< "serverId:"<< ci.serverId<< " ipPort:"<< ci.ipPort<< " matchIndex:"<< ci.matchIndex<< " nextIndex:"<< ci.nextIndex<< " role:"<< ci.role<< " hasVoted:"<< ci.hasVoted << " forceSync:" << ci.forceSync << " electionWeight: " << ci.electionWeight << std::endl<< std::flush;
}
EXPECT_EQ(cis[3].ipPort, strTmp3);
//Transfer Leader to Server 3 fail case
delete paxos3;
sleep(1);
paxos2->leaderTransfer(3);
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
EXPECT_EQ(paxos2->getSubState(), Paxos::SubLeaderTransfer);
sleep(2);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
EXPECT_EQ(paxos2->getSubState(), Paxos::SubNone);
delete paxos1;
delete learner2;
delete learner3;
// Leader stepdown.
sleep(3);
if (paxos2->getState() != Paxos::FOLLOWER)
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::FOLLOWER);
config.reset();
delete paxos2;
delete learner1;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
deleteDir("paxosLogTestDir4");
deleteDir("paxosLogTestDir5");
deleteDir("paxosLogTestDir6");
}
TEST(consensus, Paxos_requestVoteAuto)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11011");
strConfig.emplace_back("127.0.0.1:11012");
strConfig.emplace_back("127.0.0.1:11013");
std::shared_ptr<PaxosLog> rlog1= std::make_shared<RDPaxosLog>("paxosLogTestDir11", true, 4 * 1024 * 1024);
std::shared_ptr<PaxosLog> rlog2= std::make_shared<RDPaxosLog>("paxosLogTestDir12", true, 4 * 1024 * 1024);
std::shared_ptr<PaxosLog> rlog3= std::make_shared<RDPaxosLog>("paxosLogTestDir13", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(500, rlog1);
paxos1->init(strConfig, 1);
Paxos *paxos2= new Paxos(500, rlog2);
paxos2->init(strConfig, 2);
Paxos *paxos3= new Paxos(500, rlog3);
paxos3->init(strConfig, 3);
sleep(2.2);
EXPECT_TRUE(paxos1->getState() == Paxos::LEADER || paxos2->getState() == Paxos::LEADER || paxos3->getState() == Paxos::LEADER);
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir11");
deleteDir("paxosLogTestDir12");
deleteDir("paxosLogTestDir13");
}
void applyThread(Paxos *paxos)
{
paxos->waitCommitIndexUpdate(0);
EXPECT_GE(paxos->getCommitIndex(), 1);
uint64_t i= 0;
std::shared_ptr<PaxosLog> log= paxos->getLog();
for (i= 1; i <= paxos->getCommitIndex(); ++i)
{
LogEntry entry;
log->getEntry(i, entry);
std::cout<< "LogIndex "<<i <<": key:"<< entry.ikey()<<", value:"<< entry.value() << std::endl;
}
std::cout<< "ApplyThread: commitIndex has been updated to "<< paxos->getCommitIndex()<< " ."<<std::endl<< std::flush;
paxos->waitCommitIndexUpdate(10, 3);
EXPECT_LE(paxos->getCommitIndex(), 10);
EXPECT_NE(paxos->getTerm(), 3);
std::cout<< "ApplyThread: return because state changed."<<std::endl<< std::flush;
}
void stateChangeCb(Paxos::StateType state, uint64_t term, uint64_t commitIndex)
{
static uint64_t cnt= 0;
std::cout<< "State change to "<< state<< " in stateChangeCb term:"<< term<< " commitIndex:"<< commitIndex<< std::endl;
if (cnt == 0)
{
EXPECT_EQ(state, Paxos::CANDIDATE);
EXPECT_EQ(term, 2);
EXPECT_EQ(commitIndex, 0);
}
else if(cnt == 1)
{
EXPECT_EQ(state, Paxos::LEADER);
EXPECT_EQ(term, 2);
EXPECT_EQ(commitIndex, 0);
}
else if(cnt == 2)
{
EXPECT_EQ(state, Paxos::FOLLOWER);
EXPECT_EQ(term, 3);
EXPECT_EQ(commitIndex, 4);
}
else if(cnt == 3)
{
EXPECT_EQ(state, Paxos::FOLLOWER);
EXPECT_EQ(term, 4);
EXPECT_EQ(commitIndex, 4);
}
++ cnt;
}
void printPaxosStats(Paxos *paxos)
{
const Paxos::StatsType &stats= paxos->getStats();
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog<< " countMsgRequestVote:"<<stats.countMsgRequestVote<< " countOnMsgAppendLog:"<< stats.countOnMsgAppendLog<< " countHeartbeat:"<< stats.countHeartbeat << " countOnMsgRequestVote:"<<stats.countOnMsgRequestVote<< " countOnHeartbeat:"<<stats.countOnHeartbeat<< " countReplicateLog:"<<stats.countReplicateLog<< std::endl<< std::flush;
}
TEST(consensus, Paxos_replicateLog)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
uint64_t timeout= 12500;
std::shared_ptr<PaxosLog> rlog;
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir21", true, 4 * 1024 * 1024);
ClientService *cs1= new ClientService();
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1, cs1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir22", true, 4 * 1024 * 1024);
ClientService *cs2= new ClientService();
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, cs2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir23", true, 4 * 1024 * 1024);
ClientService *cs3= new ClientService();
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3, cs3);
// Need one reconnect, because the other Paxos is not inited.
sleep(5);
paxos1->setStateChangeCb(stateChangeCb);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_ikey("bbb");
//le.set_value("ccc");
const Paxos::StatsType &stats= paxos1->getStats();
//std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog << " countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
printPaxosStats(paxos1);
std::thread th1(applyThread, paxos1);
paxos1->replicateLog(le);
//std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog <<" countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
printPaxosStats(paxos1);
sleep(1);
paxos1->replicateLog(le);
//std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog <<" countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
printPaxosStats(paxos1);
sleep(1);
paxos1->replicateLog(le);
//std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog <<" countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
printPaxosStats(paxos1);
EXPECT_EQ(stats.countHeartbeat, 0);
sleep(3);
//std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog << " countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
printPaxosStats(paxos1);
EXPECT_EQ(stats.countHeartbeat, 2);
printPaxosStats(paxos2);
paxos2->requestVote();
sleep(1);
paxos3->requestVote();
sleep(1);
th1.join();
delete paxos1;
delete paxos2;
delete paxos3;
delete cs1;
delete cs2;
delete cs3;
deleteDir("paxosLogTestDir21");
deleteDir("paxosLogTestDir22");
deleteDir("paxosLogTestDir23");
}
std::atomic<uint64_t> failCnt(0);
class FailRDPaxosLog : public RDPaxosLog {
public:
FailRDPaxosLog(const std::string &dataDir, bool compress, size_t writeBufferSize, bool sync=true)
:RDPaxosLog(dataDir, compress, writeBufferSize, sync)
{};
virtual int getEntry(uint64_t logIndex, LogEntry &entry, bool fastfail= false) {
if (failCnt > 0) {
--failCnt;
std::cout<< "getEntry will return fail."<< std::endl<< std::flush;
return 1;
}
else {
return RDPaxosLog::getEntry(logIndex, entry);
}
}
};
TEST(consensus, Paxos_AliSQLServer)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
std::string strTmp3("127.0.0.1:11003");
uint64_t timeout= 12500;
std::shared_ptr<AliSQLServer> aserver;
std::shared_ptr<AliSQLServer> aserver1= std::make_shared<AliSQLServer>(1);
std::shared_ptr<PaxosLog> rlog, rlog1;
std::shared_ptr<FailRDPaxosLog> rdlog;
rlog1= rlog= rdlog= std::make_shared<FailRDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
rdlog->set_debug_async();
ClientService *cs1= new ClientService();
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1, cs1, 2, 2, aserver= std::make_shared<AliSQLServer>(1));
rlog= rdlog= std::make_shared<FailRDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
ClientService *cs2= new ClientService();
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, cs2);
rlog= rdlog= std::make_shared<FailRDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
rdlog->set_debug_async();
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->initAsLearner(strTmp3, nullptr, 2, 2, aserver1);
sleep(5);
paxos1->requestVote();
sleep(2);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_ikey("bbb");
le.set_value("ccc");
const Paxos::StatsType &stats= paxos1->getStats();
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog << " countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
paxos1->replicateLog(le);
// reject the old term replicateLog call.
le.set_term(1);
EXPECT_EQ(0, paxos1->replicateLog(le));
le.set_term(0);
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog <<" countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
sleep(1);
std::cout<< "commitIndex: "<< paxos1->getCommitIndex()<< std::endl<< std::flush;
EXPECT_EQ(paxos1->getCommitIndex(), 1);
std::cout<< "lastLogIndex: "<< rlog1->getLastLogIndex()<< std::endl<< std::flush;
EXPECT_GE(rlog1->getLastLogIndex(), 1);
aserver->writeLogDone(rlog1->getLastLogIndex());
sleep(1);
EXPECT_EQ(paxos1->getCommitIndex(), rlog1->getLastLogIndex());
std::cout<< "commitIndex: "<< paxos1->getCommitIndex()<< std::endl<< std::flush;
std::vector<Paxos::ClusterInfoType> cis;
paxos1->getClusterInfo(cis);
for (auto& ci : cis)
{
std::cout<< "serverId:"<< ci.serverId<< " ipPort:"<< ci.ipPort<< " matchIndex:"<< ci.matchIndex<< " nextIndex:"<< ci.nextIndex<< " role:"<< ci.role<< " hasVoted:"<< ci.hasVoted << " forceSync:" << ci.forceSync << " electionWeight: " << ci.electionWeight << std::endl<< std::flush;
}
Paxos::MemberInfoType mi;
paxos1->getMemberInfo(&mi);
std::cout
<< "serverId:"<< mi.serverId
<< " currentTerm:"<< mi.currentTerm
<< " currentLeader:"<< mi.currentLeader
<< " commitIndex:"<< mi.commitIndex
<< " lastLogTerm:"<< mi.lastLogTerm
<< " lastLogIndex:"<< mi.lastLogIndex
<< " role:"<< mi.role
<< " votedFor:"<< mi.votedFor
<< " lastAppliedIndex:"<< mi.lastAppliedIndex
<< " currentLeaderAddr:"<< mi.currentLeaderAddr
<< std::endl<< std::flush;
paxos2->getMemberInfo(&mi);
std::cout
<< "serverId:"<< mi.serverId
<< " currentTerm:"<< mi.currentTerm
<< " currentLeader:"<< mi.currentLeader
<< " commitIndex:"<< mi.commitIndex
<< " lastLogTerm:"<< mi.lastLogTerm
<< " lastLogIndex:"<< mi.lastLogIndex
<< " role:"<< mi.role
<< " votedFor:"<< mi.votedFor
<< " lastAppliedIndex:"<< mi.lastAppliedIndex
<< " currentLeaderAddr:"<< mi.currentLeaderAddr
<< std::endl<< std::flush;
strConfig.clear();
strConfig.push_back(strTmp3);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(1);
EXPECT_EQ(0, paxos1->changeMember(Paxos::CCAddNode, strTmp3));
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
EXPECT_EQ(0, paxos1->downgradeMember(3));
sleep(1);
cis.clear();
paxos1->getClusterInfo(cis);
Paxos::printClusterInfo(cis);
paxos3->forceSingleLeader();
sleep(1);
le.clear_term();
paxos3->replicateLog(le);
aserver1->writeLogDone(paxos3->getLastLogIndex());
sleep(1);
EXPECT_EQ(paxos3->getCommitIndex(), paxos3->getLastLogIndex());
sleep(1);
failCnt.store(1, std::memory_order_relaxed);
paxos1->replicateLog(le);
sleep(1);
/*
sleep(1);
paxos1->replicateLog(le);
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog <<" countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
sleep(1);
paxos1->replicateLog(le);
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog <<" countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
EXPECT_EQ(stats.countHeartbeat, 0);
sleep(3);
std::cout<< "countMsgAppendLog:"<<stats.countMsgAppendLog << " countHeartbeat:"<< stats.countHeartbeat<< std::endl<< std::flush;
EXPECT_EQ(stats.countHeartbeat, 2);
*/
delete paxos1;
delete paxos2;
delete paxos3;
delete cs1;
delete cs2;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
}
void stateChangeCb2_1(Paxos *p, Paxos::StateType state, uint64_t term, uint64_t commitIndex)
{
static uint64_t cnt= 0;
std::cout<< "Server(" << p->getLocalServer()->strAddr << ") state change to "<< state<< " in stateChangeCb term:"<< term<< " commitIndex:"<< commitIndex<< std::endl;
if (cnt == 0)
{
EXPECT_EQ(state, Paxos::CANDIDATE);
EXPECT_EQ(term, 2);
EXPECT_EQ(commitIndex, 0);
EXPECT_EQ(p->getLocalServer()->strAddr, "127.0.0.1:11001");
}
else if(cnt == 1)
{
EXPECT_EQ(state, Paxos::LEADER);
EXPECT_EQ(term, 2);
EXPECT_EQ(commitIndex, 0);
EXPECT_EQ(p->getLocalServer()->strAddr, "127.0.0.1:11001");
}
else if(cnt == 2)
{
EXPECT_EQ(state, Paxos::FOLLOWER);
EXPECT_EQ(term, 3);
EXPECT_EQ(commitIndex, 4);
EXPECT_EQ(p->getLocalServer()->strAddr, "127.0.0.1:11001");
}
else if(cnt == 3)
{
EXPECT_EQ(state, Paxos::FOLLOWER);
EXPECT_EQ(term, 4);
EXPECT_EQ(commitIndex, 4);
EXPECT_EQ(p->getLocalServer()->strAddr, "127.0.0.1:11001");
}
++ cnt;
}
void stateChangeCb2_2(Paxos *p, Paxos::StateType state, uint64_t term, uint64_t commitIndex)
{
std::cout << "Callback with 4 arguments" << std::endl;
EXPECT_EQ(p->getLocalServer()->strAddr, "127.0.0.1:11002");
}
void stateChangeCb2_3_lessarg(Paxos::StateType state, uint64_t term, uint64_t commitIndex)
{
std::cout << "Callback with 3 arguments" << std::endl;
}
// callback1: Paxos_replicateLog
TEST(consensus, callback2)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
uint64_t timeout= 12500;
std::shared_ptr<PaxosLog> rlog;
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir21", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir22", true, 4 * 1024 * 1024);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir23", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
// Need one reconnect, because the other Paxos is not inited.
sleep(5);
paxos1->setStateChangeCb(stateChangeCb2_1);
paxos2->setStateChangeCb(stateChangeCb2_2);
paxos3->setStateChangeCb(stateChangeCb2_3_lessarg);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_ikey("bbb");
//le.set_value("ccc");
le.set_value("0123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 12345");
const Paxos::StatsType &stats= paxos1->getStats();
std::thread th1(applyThread, paxos1);
paxos1->replicateLog(le);
sleep(1);
paxos1->replicateLog(le);
sleep(1);
paxos1->replicateLog(le);
EXPECT_EQ(stats.countHeartbeat, 0);
sleep(3);
EXPECT_EQ(stats.countHeartbeat, 2);
printPaxosStats(paxos2);
paxos2->requestVote();
sleep(1);
paxos3->requestVote();
sleep(1);
th1.join();
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir21");
deleteDir("paxosLogTestDir22");
deleteDir("paxosLogTestDir23");
}
TEST(consensus, timeout_set)
{
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
const uint64_t timeout= 2000;
std::shared_ptr<PaxosLog> rlog, rlog1;
rlog1= rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog);
paxos1->init(strConfig, 1);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2= new Paxos(timeout, rlog);
paxos2->init(strConfig, 2);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog);
paxos3->init(strConfig, 3);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
std::string strTmp("127.0.0.1:11004");
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir4", true, 4 * 1024 * 1024);
Paxos *learner1= new Paxos(timeout, rlog);
learner1->initAsLearner(strTmp);
sleep(1);
strConfig.clear();
strConfig.push_back(strTmp);
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
sleep(2);
LogEntry le;
for (int i=0; i<10; ++i)
{
le.Clear();
le.set_optype(1);
paxos1->replicateLog(le);
}
paxos1->setSendPacketTimeout(10000);
for (int i=0; i<10; ++i)
{
le.Clear();
le.set_optype(1);
paxos1->replicateLog(le);
}
paxos1->setLearnerConnTimeout(1000);
for (int i=0; i<10; ++i)
{
le.Clear();
le.set_optype(1);
paxos1->replicateLog(le);
}
sleep(2);
EXPECT_EQ(learner1->getLastLogIndex(), paxos1->getLastLogIndex());
EXPECT_EQ(paxos2->getLastLogIndex(), paxos1->getLastLogIndex());
delete learner1;
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
deleteDir("paxosLogTestDir4");
}
TEST(consensus, fix_match_index)
{
std::vector<std::string> strConfig = { "127.0.0.1:11001", "127.0.0.1:11002",
"127.0.0.1:11003" };
uint64_t timeout = 10000;
std::shared_ptr<PaxosLog> rlog;
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1 = new Paxos(timeout, rlog);
paxos1->init(strConfig, 1, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2 = new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3 = new Paxos(timeout, rlog);
paxos3->init(strConfig, 3, NULL);
sleep(1);
paxos1->requestVote();
sleep(2);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
/* replicate a log */
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_value("12345");
paxos1->replicateLog(le);
std::atomic<bool> shutdown(false);
auto worker_main = [&shutdown](Paxos* p, uint64_t time) {
while(shutdown.load() == false)
{
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_value("12345");
if (p->getState() == Paxos::LEADER)
p->replicateLog(le);
usleep(time * 1000);
}
};
std::thread writer1(worker_main, paxos1, 100);
std::thread writer2(worker_main, paxos2, 100);
std::thread writer3(worker_main, paxos3, 100);
paxos2->forceFixMatchIndex(2, paxos3->getLastLogIndex() - 2);
std::thread op1([paxos1, paxos2, paxos3, &shutdown](){
sleep(1);
while(shutdown.load() == false)
{
paxos1->forceFixMatchIndex(2, paxos2->getLastLogIndex() - 2);
sleep(2);
paxos1->forceFixMatchIndex(3, paxos3->getLastLogIndex() - 2);
sleep(2);
}
});
/* how long to test */
sleep(10);
shutdown.store(true);
writer1.join();
writer2.join();
writer3.join();
op1.join();
paxos1->replicateLog(le);
paxos2->replicateLog(le);
paxos3->replicateLog(le);
sleep(2);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos2->getLastLogIndex(), paxos3->getLastLogIndex());
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
}
TEST(consensus, checksum)
{
std::vector<std::string> strConfig = { "127.0.0.1:11001", "127.0.0.1:11002",
"127.0.0.1:11003" };
uint64_t timeout = 10000;
std::shared_ptr<PaxosLog> rlog;
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1 = new Paxos(timeout, rlog);
paxos1->init(strConfig, 1, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2 = new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3 = new Paxos(timeout, rlog);
paxos3->init(strConfig, 3, NULL);
sleep(1);
paxos1->requestVote();
sleep(2);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
/* replicate a log */
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_value("12345");
paxos1->replicateLog(le);
sleep(1);
std::function<uint32_t(uint32_t, const unsigned char*, size_t)> checksumFunc =
[](uint32_t base, const unsigned char* buf, size_t len) -> uint32_t
{
return base + len;
};
paxos1->setChecksumCb(checksumFunc);
paxos2->setChecksumCb(checksumFunc);
paxos3->setChecksumCb(checksumFunc);
paxos1->setChecksumMode(true);
paxos2->setChecksumMode(true);
paxos3->setChecksumMode(true);
std::cout << "enable checksum..." << std::endl;
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos2->getLastLogIndex(), paxos3->getLastLogIndex());
le.set_index(0);
le.set_value("12345");
le.set_checksum(0);
paxos1->replicateLog(le);
sleep(1);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos2->getLastLogIndex(), paxos3->getLastLogIndex());
le.set_index(0);
le.set_value("12345");
le.set_checksum(6); // 5 is correct
paxos1->replicateLog(le);
for (int i=0; i<10; ++i)
{
le.set_index(0);
le.set_value("12345");
le.set_checksum(0);
paxos1->replicateLog(le);
}
sleep(1);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex() + 11);
EXPECT_EQ(paxos2->getLastLogIndex(), paxos3->getLastLogIndex());
std::cout << "disable checksum..." << std::endl;
paxos1->setChecksumMode(false);
paxos2->setChecksumMode(false);
paxos3->setChecksumMode(false);
sleep(5);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos2->getLastLogIndex(), paxos3->getLastLogIndex());
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
}
TEST(consensus, update_commitindex_in_leader_transfer)
{
std::vector<std::string> strConfig = { "127.0.0.1:11001", "127.0.0.1:11002",
"127.0.0.1:11003" };
uint64_t timeout = 10000;
std::shared_ptr<PaxosLog> rlog;
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1 = new Paxos(timeout, rlog);
paxos1->init(strConfig, 1, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2 = new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3 = new Paxos(timeout, rlog);
paxos3->init(strConfig, 3, NULL);
sleep(1);
paxos1->requestVote();
sleep(2);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
/* replicate a log */
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_value("12345");
paxos1->replicateLog(le);
for (int i=0; i<10; ++i)
paxos1->replicateLog(le);
while (paxos1->getLastLogIndex() != paxos2->getLastLogIndex())
sleep(1);
paxos2->debugSkipUpdateCommitIndex = true;
strConfig.clear();
strConfig.push_back("127.0.0.1:11004");
paxos1->changeLearners(Paxos::CCAddNode, strConfig);
for (int i=0; i<10; ++i)
paxos1->replicateLog(le);
while (paxos1->getLastLogIndex() != paxos2->getLastLogIndex())
sleep(1);
EXPECT_EQ(paxos1->getCommitIndex(), paxos2->getLastLogIndex());
EXPECT_NE(paxos1->getCommitIndex(), paxos2->getCommitIndex());
paxos1->leaderTransfer(2);
sleep(2);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
paxos2->debugSkipUpdateCommitIndex = false;
while (paxos1->getLastLogIndex() != paxos2->getLastLogIndex())
sleep(1);
sleep(5);
EXPECT_EQ(paxos1->getCommitIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos1->getCommitIndex(), paxos2->getCommitIndex());
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
}
TEST(consensus, flow_control)
{
std::vector<std::string> strConfig = { "127.0.0.1:11001", "127.0.0.1:11002",
"127.0.0.1:11003" };
uint64_t timeout = 5000; // 1s heartbeat
std::shared_ptr<PaxosLog> rlog;
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1 = new Paxos(timeout, rlog);
paxos1->init(strConfig, 1, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2 = new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, NULL);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3 = new Paxos(timeout, rlog);
paxos3->init(strConfig, 3, NULL);
sleep(1);
paxos1->requestVote();
sleep(2);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
/* replicate a log */
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_value("12345");
paxos1->replicateLog(le);
for (int i=0; i<4; ++i)
paxos1->replicateLog(le);
sleep(2);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
uint64_t mps = paxos1->getMaxPacketSize();
paxos1->setMaxPacketSize(1);
paxos1->set_flow_control(2, -1);
paxos1->set_flow_control(3, -2);
for (int i=0; i<4; ++i)
paxos1->replicateLog(le);
sleep(2);
EXPECT_NE(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_NE(paxos1->getLastLogIndex(), paxos3->getLastLogIndex());
sleep(6);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_NE(paxos1->getLastLogIndex(), paxos3->getLastLogIndex());
paxos1->setMaxPacketSize(mps);
paxos1->set_flow_control(3, 0);
for (int i=0; i<4; ++i)
paxos1->replicateLog(le);
sleep(2);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos1->getLastLogIndex(), paxos3->getLastLogIndex());
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
}
TEST(consensus, memory_usage_count)
{
std::vector<std::string> strConfig = { "127.0.0.1:11001", "127.0.0.1:11002",
"127.0.0.1:11003" };
uint64_t timeout = 5000; // 1s heartbeat
std::shared_ptr<PaxosLog> rlog, rlog1;
rlog1 = rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1 = new Paxos(timeout, rlog1);
paxos1->init(strConfig, 1, NULL, 4, 4, NULL, true);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2 = new Paxos(timeout, rlog);
paxos2->init(strConfig, 2, NULL, 4, 4, NULL, true);
rlog = std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3 = new Paxos(timeout, rlog);
paxos3->init(strConfig, 3, NULL, 4, 4, NULL, true);
sleep(1);
paxos1->requestVote();
sleep(2);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
/* replicate a log */
LogEntry le;
le.set_index(0);
le.set_optype(1);
le.set_value("12345");
paxos1->replicateLog(le);
for (int i=0; i<4; ++i)
paxos1->replicateLog(le);
sleep(2);
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
EXPECT_EQ(paxos1->getLastLogIndex(), paxos2->getLastLogIndex());
// memory usage not equal to 0
EXPECT_NE(easy_pool_alloc_byte, 0);
std::cout << "easy_pool_alloc_byte: " << easy_pool_alloc_byte << std::endl;
delete paxos1;
delete paxos2;
delete paxos3;
// memory usage go back to 0
EXPECT_EQ(easy_pool_alloc_byte, 0);
std::cout << "easy_pool_alloc_byte: " << easy_pool_alloc_byte << std::endl;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
}
TEST(consensus, paxos_force_promote) {
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
uint64_t timeout= 10000;
std::shared_ptr<PaxosLog> rlog;
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog, 3000);
paxos1->init(strConfig, 1, NULL);
paxos1->initAutoPurgeLog(false, false, NULL); // disable autoPurge
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2= new Paxos(timeout, rlog, 3000);
paxos2->init(strConfig, 2, NULL);
paxos2->initAutoPurgeLog(false, false, NULL); // disable autoPurge
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog, 3000);
paxos3->init(strConfig, 3, NULL);
std::string learnerAddr = "127.0.0.1:11004";
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir4", true, 4 * 1024 * 1024);
Paxos *paxos4= new Paxos(timeout, rlog, 3000);
paxos4->initAsLearner(learnerAddr);
sleep(3);
paxos1->requestVote();
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
LogEntry le;
le.set_index(0);
le.set_optype(0);
le.set_value("aaa");
paxos1->replicateLog(le);
sleep(1);
std::vector<std::string> vec;
vec.push_back(learnerAddr);
paxos1->changeLearners(Paxos::CCAddNode, vec);
sleep(1);
EXPECT_EQ(paxos1->getConfig()->learnersToString(), std::string("127.0.0.1:11004$00"));
/* learner forcePromote: do nothing */
paxos4->forcePromote();
sleep(1);
EXPECT_EQ(paxos4->getState(), Paxos::LEARNER);
/* follower forcePromote: become new leader */
paxos2->forcePromote();
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
EXPECT_EQ(paxos1->getState(), Paxos::FOLLOWER);
delete paxos1;
delete paxos2;
delete paxos3;
delete paxos4;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
deleteDir("paxosLogTestDir4");
}
TEST(consensus, extra_info) {
std::vector<std::string> strConfig;
strConfig.emplace_back("127.0.0.1:11001");
strConfig.emplace_back("127.0.0.1:11002");
strConfig.emplace_back("127.0.0.1:11003");
uint64_t timeout= 5000;
std::shared_ptr<PaxosLog> rlog;
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir1", true, 4 * 1024 * 1024);
Paxos *paxos1= new Paxos(timeout, rlog);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir2", true, 4 * 1024 * 1024);
Paxos *paxos2= new Paxos(timeout, rlog);
rlog= std::make_shared<RDPaxosLog>("paxosLogTestDir3", true, 4 * 1024 * 1024);
Paxos *paxos3= new Paxos(timeout, rlog);
class MockExtraStore: public ExtraStore {
public:
MockExtraStore(uint p) : port(p), isLeader(0) {}
virtual std::string getRemote() { return data; }
virtual void setRemote(const std::string &d) { data = d; }
/* local info to send to others */
virtual std::string getLocal() { return isLeader? std::to_string(port): ""; }
uint getRemotePort() { return data != ""? stoul(data): 0; }
void setIsLeader(bool b) { isLeader = b; }
private:
uint port;
std::string data;
bool isLeader; /* maintained by rds server level */
};
paxos1->init(strConfig, 1);
paxos2->init(strConfig, 2);
paxos3->init(strConfig, 3);
paxos1->setExtraStore(std::make_shared<MockExtraStore>(paxos1->getPort()-8000));
paxos2->setExtraStore(std::make_shared<MockExtraStore>(paxos2->getPort()-8000));
paxos3->setExtraStore(std::make_shared<MockExtraStore>(paxos3->getPort()-8000));
sleep(3);
paxos1->requestVote(1);
sleep(1);
EXPECT_EQ(paxos1->getState(), Paxos::LEADER);
std::dynamic_pointer_cast<MockExtraStore>(paxos1->getExtraStore())->setIsLeader(1);
std::dynamic_pointer_cast<MockExtraStore>(paxos2->getExtraStore())->setIsLeader(0);
std::dynamic_pointer_cast<MockExtraStore>(paxos3->getExtraStore())->setIsLeader(0);
LogEntry le;
le.set_index(0);
le.set_optype(0);
le.set_value("aaa");
paxos1->replicateLog(le);
sleep(1);
/* every follower will have port-8000 from leader paxos1 */
EXPECT_EQ(std::dynamic_pointer_cast<MockExtraStore>(paxos1->getExtraStore())->getRemotePort(), 0);
EXPECT_EQ(std::dynamic_pointer_cast<MockExtraStore>(paxos2->getExtraStore())->getRemotePort(), paxos1->getPort()-8000);
EXPECT_EQ(std::dynamic_pointer_cast<MockExtraStore>(paxos3->getExtraStore())->getRemotePort(), paxos1->getPort()-8000);
paxos2->requestVote(1);
sleep(1);
EXPECT_EQ(paxos2->getState(), Paxos::LEADER);
std::dynamic_pointer_cast<MockExtraStore>(paxos1->getExtraStore())->setIsLeader(0);
std::dynamic_pointer_cast<MockExtraStore>(paxos2->getExtraStore())->setIsLeader(1);
std::dynamic_pointer_cast<MockExtraStore>(paxos3->getExtraStore())->setIsLeader(0);
paxos2->replicateLog(le);
sleep(1);
/* every follower will have port-8000 from new leader paxos2 */
EXPECT_EQ(std::dynamic_pointer_cast<MockExtraStore>(paxos1->getExtraStore())->getRemotePort(), paxos2->getPort()-8000);
EXPECT_EQ(std::dynamic_pointer_cast<MockExtraStore>(paxos2->getExtraStore())->getRemotePort(), 0);
EXPECT_EQ(std::dynamic_pointer_cast<MockExtraStore>(paxos3->getExtraStore())->getRemotePort(), paxos2->getPort()-8000);
delete paxos1;
delete paxos2;
delete paxos3;
deleteDir("paxosLogTestDir1");
deleteDir("paxosLogTestDir2");
deleteDir("paxosLogTestDir3");
}
Reference in New Issue View Git Blame Copy Permalink