/************************************************************************ * * Copyright (c) 2016 Alibaba.com, Inc. All Rights Reserved * $Id: easyNet.cc,v 1.0 07/31/2016 04:53:21 PM yingqiang.zyq(yingqiang.zyq@alibaba-inc.com) $ * ************************************************************************/ /** * @file easyNet.cc * @author yingqiang.zyq(yingqiang.zyq@alibaba-inc.com) * @date 07/31/2016 04:53:21 PM * @version 1.0 * @brief * **/ #include "easyNet.h" #include "service.h" #include "paxos_server.h" //TODO no driect include #include "paxos.h" //TODO no driect include #include "paxos.pb.h" namespace alisql { easy_atomic_t easy_pool_alloc_byte = 0; static void *easy_count_realloc(void *ptr, size_t size) { char *p1, *p = (char *)ptr; if (p) { p -= 8; easy_atomic_add(&easy_pool_alloc_byte, -(*((int *)p))); } if (size) { easy_atomic_add(&easy_pool_alloc_byte, size); p1 = (char *)easy_realloc(p, size + 8); if (p1) { *((int *)p1) = size; p1 += 8; } return p1; } else if (p) easy_free(p); return NULL; } EasyNet::EasyNet(uint64_t num, const uint64_t sessionTimeout, bool memory_usage_count) :reciveCnt_(0) ,isShutdown_(false) ,sessionTimeout_(sessionTimeout) { if (unlikely(memory_usage_count)) { // set allocator for memory usage count easy_pool_set_allocator(easy_count_realloc); } eio_= easy_eio_create(NULL, num); eio_->do_signal = 0; } int EasyNet::init(void *ptr) { memset(&clientHandler_, 0, sizeof(clientHandler_)); clientHandler_.decode= EasyNet::paxosDecode; clientHandler_.encode= EasyNet::paxosEncode; clientHandler_.process= EasyNet::reciveProcess; clientHandler_.on_connect= EasyNet::onConnected; clientHandler_.on_disconnect= EasyNet::onDisconnected; clientHandler_.cleanup= EasyNet::onClientCleanup; clientHandler_.get_packet_id= EasyNet::getPacketId; clientHandler_.user_data= (void *)workPool_; clientHandler_.user_data2= ptr; memset(&serverHandler_, 0, sizeof(serverHandler_)); serverHandler_.decode= EasyNet::paxosDecode; serverHandler_.encode= EasyNet::paxosEncode; serverHandler_.process= EasyNet::reciveProcess; serverHandler_.user_data= (void *)workPool_; serverHandler_.user_data2= ptr; return 0; }; int EasyNet::start(int port) { if (eio_ == NULL) return -3; if (easy_connection_add_listen(eio_, NULL, port, &serverHandler_) == NULL) return -1; if (easy_eio_start(eio_)) return -2; return 0; } int EasyNet::shutdown() { lock_.lock(); isShutdown_= true; lock_.unlock(); /* * The follow function should not be called by hold the lock. * Because some callback function which try to hold lock, will be called. */ easy_eio_shutdown(eio_); easy_eio_stop(eio_); easy_eio_wait(eio_); easy_eio_destroy(eio_); return 0; } int EasyNet::stop() { easy_eio_stop(eio_); return 0; } easy_addr_t EasyNet::createConnection(const std::string &addr, NetServerRef server, uint64_t timeout, uint64_t index) { easy_addr_t iaddr; if (isShutdown_) { iaddr.port= 0; return iaddr; } iaddr= easy_inet_str_to_addr(addr.c_str(), 0); iaddr.cidx= index; if (server != nullptr) setConnData(iaddr, server); if (easy_connection_connect(eio_, iaddr, &clientHandler_, timeout, NULL, EASY_CONNECT_AUTOCONN) != EASY_OK) //if (easy_connection_connect(eio_, iaddr, &clientHandler_, timeout, NULL, 0) != EASY_OK) iaddr.port= 0; /* easy_session_t *s= easy_connection_connect_init(NULL, &clientHandler_, 200, NULL, 0, NULL); if (iaddr.family == 0 || s == NULL) iaddr.port= 0; if (NULL == easy_client_send(eio_, iaddr, s)) iaddr.port= 0; */ return iaddr; } void EasyNet::disableConnection(easy_addr_t addr) { easy_connection_disconnect(eio_, addr); } int EasyNet::sendPacket(easy_addr_t addr, const char *buf, uint64_t len, uint64_t id) { easy_session_t *s; NetPacket *np; if ((np= easy_session_packet_create(NetPacket, s, len)) == NULL) { return -1; } auto server= std::dynamic_pointer_cast(getConnData(addr)); if (!server || !server->isLearner || !server->paxos || server->paxos->getLocalServer()->learnerConnTimeout == 0) easy_session_set_timeout(s, sessionTimeout_); //ms else easy_session_set_timeout(s, server->paxos->getLocalServer()->learnerConnTimeout * 4); np->data= &np->buffer[0]; np->len= len; np->packetId= id; memcpy(np->data, buf, len); if (easy_client_dispatch(eio_, addr, s) == EASY_ERROR) { easy_session_destroy(s); return -2; } /* TODO: onFail callback function need to be added*/ return 0; } int EasyNet::sendPacket(easy_addr_t addr, const std::string& buf, uint64_t id) { return sendPacket(addr, buf.c_str(), buf.length(), id); } int EasyNet::resendPacket(easy_addr_t addr, void *ptr, uint64_t id) { NetPacket *np= (NetPacket *)ptr; return sendPacket(addr, np->data, np->len, id); } int EasyNet::setRecvPacketCallback(void *handler) { setWorkPool((easy_thread_pool_t *)handler); return 0; } NetServerRef EasyNet::getConnData(easy_addr_t addr, bool locked) { std::unique_lock lg(lock_, std::defer_lock); if (!locked) lg.lock(); auto it= connStatus_.find(getAddrKey(addr)); if (it != connStatus_.end()) return it->second; else return nullptr; } NetServerRef EasyNet::getConnDataAndSetFail(easy_connection_t *c, bool isFail) { auto addr= c->addr; std::lock_guard lg(lock_); auto s= getConnData(addr, true); auto server= std::dynamic_pointer_cast(s); if (server == nullptr) return s; if (isFail) { if (!server->netError.load()) { if (server->c == c || server->c == nullptr) easy_error_log("EasyNet::onDisconnected server %ld\n", server->serverId); else easy_error_log("EasyNet::onDisconnected server %ld, which is the old one\n", server->serverId); } /* disconnect */ if (server->c != nullptr && server->c != c) return s; server->waitForReply= 0; server->netError.store(isFail); } else { if (server->netError.load()) easy_error_log("EasyNet::onConnected server %ld\n", server->serverId); /* connect */ server->c = c; server->waitForReply= 0; server->netError.store(isFail); } return s; } void EasyNet::setConnData(easy_addr_t addr, NetServerRef server) { std::lock_guard lg(lock_); connStatus_.insert(std::make_pair(getAddrKey(addr), server)); } void EasyNet::delConnDataById(uint64_t id) { /* Protect the connStatus_ map */ std::lock_guard lg(lock_); for (auto it= connStatus_.begin(); it != connStatus_.end();) { if (std::dynamic_pointer_cast(it->second)->serverId == id) connStatus_.erase(it++); else ++it; if (connStatus_.size() == 0) break; } } uint64_t EasyNet::getAddrKey(easy_addr_t addr) { uint64_t ret= addr.u.addr; ret <<= 32; ret |= addr.port; return ret; } void EasyNet::tryFreeMsg(NetPacket *np) { if (np->msg) { delete static_cast(np->msg); np->msg= nullptr; } } int EasyNet::reciveProcess(easy_request_t *r) { if (r == NULL) return EASY_ERROR; /* TODO means send callback process here? called by easy_session_process */ if (r->ms->c == NULL) return EASY_ERROR; auto tp= (easy_thread_pool_t *) (r->ms->c->handler->user_data); auto srv= (Service *) (r->ms->c->handler->user_data2); if (srv == NULL) return EASY_ERROR; assert(r->user_data == NULL); r->user_data = (void *)srv; auto en= srv->getEasyNet(); en->incRecived(); if (tp == NULL || !srv->workPoolIsRunning()) return EASY_ERROR; int ret= EASY_AGAIN; if (r->ipacket == NULL) { /* * timeout case. * we push the session to thread pool. * Service::process will handle this case. */ auto server= std::dynamic_pointer_cast(en->getConnData(r->ms->c->addr)); /* May already be remove. */ if (server != nullptr) { NetPacket *np= (NetPacket *)(r->opacket); easy_warn_log("EasyNet::reciveProcess sendMsg timeout for serverid:%ld packet_id:0x%llx(%llu)\n", server->serverId, np->packetId, np->packetId); } //server->waitForReply= 0; /* Here we should not return EASY_ERROR. */ //ret= EASY_AGAIN; ret= EASY_ERROR; } if (r->ms->c->type == EASY_TYPE_SERVER) { auto tp2 = srv->getHeartbeatPool(); NetPacket *np = (NetPacket *)r->ipacket; assert(np); PaxosMsg *msg = new PaxosMsg; if (!msg->ParseFromArray(np->data, np->len)) { // receive a wrong msg. easy_error_log("A msg have %ld entries!! droped!!\n", msg->entries_size()); r->opacket= (void *)NULL; return EASY_OK; } assert(r->args == 0); r->args = (void *)msg; if (tp2 == NULL || msg->msgtype() != Consensus::AppendLog || msg->entries_size() != 0 || Service::running < Service::workThreadCnt || !((Paxos *)(srv->getConsensus()))->getOptimisticHeartbeat()) { easy_thread_pool_push(tp, r, easy_hash_key((uint64_t)(long)r)); } else { // set msg type to OptimisticHeartbeat to let service know it is in heartbeat pool msg->set_msgtype(Consensus::OptimisticHeartbeat); // msg is heartbeat and work pool is full, put in heartbeat pool easy_thread_pool_push(tp2, r, easy_hash_key((uint64_t)(long)r)); } } else { easy_session_t *s= (easy_session_t *)r->ms; easy_thread_pool_push_session(tp, s, easy_hash_key((uint64_t)(long)r)); } return ret; } void *EasyNet::paxosDecode(easy_message_t *m) { NetPacket *np; uint64_t len, datalen; if ((len= m->input->last - m->input->pos) < NetPacketHeaderSize) return NULL; datalen = *((uint64_t *)m->input->pos); if (datalen > 0x4000000) { easy_error_log("data_len is invalid: %llu\n", datalen); m->status = EASY_ERROR; return NULL; } len -= NetPacketHeaderSize; if (len < datalen) { m->next_read_len = datalen - len; easy_debug_log("Decode a net packet fail, data len expect:%llu got:%llu", datalen, len); return NULL; } if ((np= (NetPacket *)easy_pool_calloc(m->pool, sizeof(NetPacket))) == NULL) { m->status = EASY_ERROR; return NULL; } m->input->pos += NetPacketHeaderSize; np->type= NetPacketTypeNet; np->data= (char *)m->input->pos; np->len= datalen; if (m->c->type == EASY_TYPE_CLIENT) { auto msg= new PaxosMsg(); msg->ParseFromArray(np->data, np->len); np->packetId= msg->msgid(); np->msg= (void *)msg; } m->input->pos += datalen; easy_debug_log("Decode a net packet success, total lens:%llu", datalen + NetPacketHeaderSize); return (void *)np; } int EasyNet::paxosEncode(easy_request_t *r, void *data) { easy_buf_t *b; NetPacket *np= (NetPacket *)data; if ((b= easy_buf_create(r->ms->pool, NetPacketHeaderSize + np->len)) == NULL) return EASY_ERROR; *((uint64_t *)b->last)= np->len; memcpy(b->last + NetPacketHeaderSize, np->data, np->len); b->last += (NetPacketHeaderSize + np->len); easy_request_addbuf(r, b); return EASY_OK; } int EasyNet::onConnected(easy_connection_t *c) { auto srv= (Service *) (c->handler->user_data2); if (srv == NULL) return 0; auto en= srv->getEasyNet(); auto server= std::dynamic_pointer_cast(en->getConnDataAndSetFail(c, false)); if (server != nullptr) { /* We don't start heartbeatTimer for learner, so we send heartbeat here. */ server->onConnectCb(); } return 0; } int EasyNet::onDisconnected(easy_connection_t *c) { auto srv= (Service *) (c->handler->user_data2); if (srv == NULL) return 0; auto en= srv->getEasyNet(); auto server= std::dynamic_pointer_cast(en->getConnDataAndSetFail(c, true)); if (server != nullptr) { // onDisconnectCB } return 0; } int EasyNet::onClientCleanup(easy_request_t *r, void *apacket) { NetPacket *np= nullptr; if (r == NULL) { np= static_cast(apacket); if (np && np->msg) { PaxosMsg *msg= static_cast(np->msg); easy_warn_log("EasyNet::onClientCleanup: msgId(%llu) packet_id:%llu responce from server %llu which already be deleted for timeout.", msg->msgid(), np->packetId, msg->serverid()); delete msg; np->msg= NULL; } return 0; } np= static_cast(r->ipacket); if (np && np->msg) { PaxosMsg *msg= static_cast(np->msg); easy_warn_log("EasyNet::onClientCleanup: msgId(%llu) packet_id:%llu responce from server %llu.", msg->msgid(), np->packetId, msg->serverid()); delete msg; np->msg= NULL; } return 0; } uint64_t EasyNet::getPacketId(easy_connection_t *c, void *data) { NetPacket *np= static_cast(data); return np->packetId; } } //namespace alisql