4493d40a-92aa-4162-9dfc-bfc4ec0996e6
/
polardbxengine
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
polardbxengine/plugin/polarx_rpc/server/epoll.h

1212 lines
40 KiB
C++
Raw Blame History

//
// Created by zzy on 2022/7/5.
//
#pragma once
#include <algorithm>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <deque>
#include <mutex>
#include <sstream>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
#include <arpa/inet.h>
#include <fcntl.h>
#include <ifaddrs.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <pthread.h>
#include <sched.h>
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <sys/poll.h>
#include <sys/socket.h>
#include <sys/sysinfo.h>
#include <unistd.h>
#include "../common_define.h"
#include "../session/session.h"
#include "../utility/array_queue.h"
#include "../utility/atomicex.h"
#include "../utility/cpuinfo.h"
#include "../utility/perf.h"
#include "../utility/time.h"
#include "epoll_group_ctx.h"
#include "server_variables.h"
#include "timer_heap.h"
/** Note: for linux only */
#ifndef SO_REUSEPORT
#define SO_REUSEPORT 15
#endif
namespace polarx_rpc {
static constexpr uint32_t MAX_EPOLL_GROUPS = 128;
static constexpr uint32_t MAX_EPOLL_EXTRA_GROUPS = 32;
static constexpr uint32_t MAX_EPOLL_THREADS_PER_GROUP = 128;
static constexpr uint32_t MIN_EPOLL_WAIT_TOTAL_THREADS = 4;
static constexpr uint32_t MAX_EPOLL_WAIT_TOTAL_THREADS = 128;
static constexpr uint32_t MAX_EPOLL_EVENTS_PER_THREAD = 16;
static constexpr uint32_t MAX_EPOLL_TIMEOUT = 60 * 1000; /// 60s
static constexpr uint32_t MAX_TCP_KEEP_ALIVE = 7200;
static constexpr uint32_t MIN_TCP_LISTEN_QUEUE = 1;
static constexpr uint32_t MAX_TCP_LISTEN_QUEUE = 4096;
static constexpr uint32_t MIN_WORK_QUEUE_CAPACITY = 128;
static constexpr uint32_t MAX_WORK_QUEUE_CAPACITY = 4096;
class CmtEpoll;
/**
* General interface for epoll callback.
*/
class CepollCallback {
public:
virtual ~CepollCallback() = default;
virtual void set_fd(int fd) = 0;
/// for reclaim in epoll callback
virtual void fd_pre_register() {}
/// for rollback register if add epoll fail
virtual bool fd_rollback_register() { return true; }
/// notify for adding reference
virtual void pre_events() {}
/// destruct the context when return false
virtual bool events(uint32_t events, int index, int total) = 0;
virtual bool send(const void *data, size_t length) { return false; }
};
/**
* Timer/worker task.
*/
struct task_t final {
private:
void *run_ctx_;
void (*run_)(void *);
void *del_ctx_;
void (*del_)(void *);
public:
task_t()
: run_ctx_(nullptr), run_(nullptr), del_ctx_(nullptr), del_(nullptr) {}
task_t(void *run_ctx, void (*run)(void *), void *del_ctx, void (*del)(void *))
: run_ctx_(run_ctx), run_(run), del_ctx_(del_ctx), del_(del) {}
task_t(const task_t &another) = default;
task_t(task_t &&another) noexcept
: run_ctx_(another.run_ctx_), run_(another.run_),
del_ctx_(another.del_ctx_), del_(another.del_) {
another.run_ctx_ = nullptr;
another.run_ = nullptr;
another.del_ctx_ = nullptr;
another.del_ = nullptr;
}
~task_t() = default;
task_t &operator=(const task_t &another) = default;
task_t &operator=(task_t &&another) noexcept {
run_ctx_ = another.run_ctx_;
run_ = another.run_;
del_ctx_ = another.del_ctx_;
del_ = another.del_;
another.run_ctx_ = nullptr;
another.run_ = nullptr;
another.del_ctx_ = nullptr;
another.del_ = nullptr;
return *this;
}
explicit operator bool() const { return run_ != nullptr; }
void call() const {
if (run_ != nullptr)
run_(run_ctx_);
}
void fin() const {
if (del_ != nullptr)
del_(del_ctx_);
}
};
/// The inherited class should has private destructor to prevent alloc on stack.
template <class T> class Ctask {
NO_COPY_MOVE(Ctask);
protected:
Ctask() = default;
virtual ~Ctask() = default;
private:
static void run_routine(void *ctx) {
auto task = reinterpret_cast<T *>(ctx);
task->run();
}
static void del_routine(void *ctx) {
auto task = reinterpret_cast<T *>(ctx);
delete task;
}
public:
// Caution: Must call this function with object by new.
task_t gen_task() {
return {this, Ctask::run_routine, this, Ctask::del_routine};
}
};
class CmtEpoll final {
NO_COPY_MOVE(CmtEpoll);
private:
/// group info
const uint32_t group_id_;
/// base epoll object
int epfd_;
/// timer task
CmcsSpinLock timer_lock_;
CtimerHeap<task_t> timer_heap_;
/// work queue
int eventfd_;
CarrayQueue<task_t> work_queue_;
/// worker wait counter
std::atomic<intptr_t> wait_cnt_;
std::atomic<intptr_t> loop_cnt_;
/// extra data for epoll group
epoll_group_ctx_t extra_ctx_;
std::atomic<int64_t> last_cleanup_;
/// affinity for dynamic threads
bool with_affinity_;
cpu_set_t cpus_{{}};
std::string cores_str_;
/// dynamic threads scale
int base_thread_count_;
std::atomic<int> stall_count_;
std::atomic<int> worker_count_; /// work with epoll
std::atomic<int> tasker_count_; /// work without epoll
std::atomic<int64_t> last_scale_time_;
std::atomic<int64_t> last_tasker_time_;
std::mutex scale_lock_;
std::atomic<int> session_count_; /// all session under this epoll
/// watch dog deadlock check
size_t last_head_;
intptr_t last_loop_;
static inline int nonblock(int fd, int set) {
int flags;
int r;
do {
r = ::fcntl(fd, F_GETFL);
} while (UNLIKELY(r == -1 && errno == EINTR));
if (UNLIKELY(r == -1))
return -errno;
/** Bail out now if already set/clear. */
if (!!(r & O_NONBLOCK) == !!set)
return 0;
if (set != 0)
flags = r | O_NONBLOCK;
else
flags = r & ~O_NONBLOCK;
do {
r = ::fcntl(fd, F_SETFL, flags);
} while (UNLIKELY(r == -1 && errno == EINTR));
if (UNLIKELY(r != 0))
return -errno;
return 0;
}
static inline int nodelay(int fd, int on) {
if (UNLIKELY(::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &on, sizeof(on)) !=
0))
return -errno;
return 0;
}
static inline int keepalive(int fd, int on, unsigned int delay) {
if (UNLIKELY(::setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &on, sizeof(on)) !=
0))
return -errno;
#ifdef TCP_KEEPIDLE
if (on &&
::setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &delay, sizeof(delay)) != 0)
return -errno;
#endif
/** Solaris/SmartOS, if you don't support keep-alive,
* then don't advertise it in your system headers...
*/
#if defined(TCP_KEEPALIVE) && !defined(__sun)
if (on && ::setsockopt(fd, IPPROTO_TCP, TCP_KEEPALIVE, &delay,
sizeof(delay)) != 0)
return -errno;
#endif
return 0;
}
void loop(uint32_t group_id, uint32_t thread_id, bool base_thread,
int affinity, bool epoll_wait, bool is_worker) {
plugin_info.threads.fetch_add(1, std::memory_order_release);
if (affinity >= 0 && !multi_affinity_in_group) {
auto thread = pthread_self();
cpu_set_t cpu;
CPU_ZERO(&cpu);
auto iret = pthread_getaffinity_np(thread, sizeof(cpu), &cpu);
if ((0 == iret && CPU_ISSET(affinity, &cpu)) || force_all_cores) {
/// only set when this thread is allowed to run on it
CPU_ZERO(&cpu);
CPU_SET(affinity, &cpu);
iret = pthread_setaffinity_np(thread, sizeof(cpu), &cpu);
{
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr) {
if (0 == iret)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll start worker thread %u:%u(%u,%u) bind to core %d.",
group_id, thread_id, base_thread, epoll_wait, affinity);
else
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll start worker thread %u:%u(%u,%u) bind "
"to core %d failed. %s",
group_id, thread_id, base_thread, epoll_wait, affinity,
std::strerror(errno));
}
}
}
} else if ((!base_thread || multi_affinity_in_group) && with_affinity_) {
/// auto bind for dynamic thread or multi bind base thread
auto thread = pthread_self();
auto iret = pthread_setaffinity_np(thread, sizeof(cpus_), &cpus_);
{
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr) {
if (0 == iret)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll start%s worker thread %u:%u(%u,%u) bind to cores %s.",
base_thread ? "" : " dynamic", group_id, thread_id, base_thread,
epoll_wait, cores_str_.c_str());
else
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll start%s worker thread %u:%u(%u,%u) bind "
"to cores %s failed. %s",
base_thread ? "" : " dynamic", group_id, thread_id, base_thread,
epoll_wait, cores_str_.c_str(), std::strerror(errno));
}
}
}
std::vector<task_t> timer_tasks;
CmcsSpinLock::mcs_spin_node_t timer_lock_node;
Csession::init_thread_for_session();
epoll_event events[MAX_EPOLL_EVENTS_PER_THREAD];
while (true) {
/// try pop and run task first
while (true) {
task_t t; /// pop one task at a time(more efficient with multi-thread)
int64_t start_time = 0;
if (enable_perf_hist)
start_time = Ctime::steady_ns();
/// get one from queue.
work_queue_.pop(t);
if (start_time != 0) {
auto task_end_time = Ctime::steady_ns();
auto work_queue_time = task_end_time - start_time;
g_work_queue_hist.update(static_cast<double>(work_queue_time) / 1e9);
}
if (!t)
break;
t.call();
t.fin();
}
if (!base_thread) {
if (UNLIKELY(shrink_thread_pool(is_worker)))
break;
}
/// limits the events
auto max_events = epoll_events_per_thread;
if (UNLIKELY(max_events <= 0))
max_events = 1;
else if (UNLIKELY(max_events > MAX_EPOLL_EVENTS_PER_THREAD))
max_events = MAX_EPOLL_EVENTS_PER_THREAD;
auto timeout = epoll_timeout;
if (UNLIKELY(timeout <= 0))
timeout = 1; /// busy waiting not allowed
else if (UNLIKELY(timeout > MAX_EPOLL_TIMEOUT))
timeout = MAX_EPOLL_TIMEOUT;
/// only one thread with correct timeout to trigger timer is ok
if (timer_lock_.try_lock(timer_lock_node)) {
int64_t next_trigger;
auto has_next = timer_heap_.peak(next_trigger);
if (has_next) {
/// adjust timeout
auto now_time = Ctime::steady_ms();
if (LIKELY(next_trigger - now_time > 0)) {
timeout =
std::min(timeout, static_cast<uint32>(next_trigger - now_time));
DBG_LOG(
("polarx_rpc thread %u:%u enter epoll with timer timeout %ums",
group_id, thread_id, timeout));
} else {
timeout = 0;
DBG_LOG(
("polarx_rpc thread %u:%u enter epoll with expired timer task",
group_id, thread_id));
}
} else {
DBG_LOG(("polarx_rpc thread %u:%u enter epoll with no timer task",
group_id, thread_id));
}
timer_lock_.unlock(timer_lock_node);
} else {
DBG_LOG(
("polarx_rpc thread %u:%u enter epoll with failed timer lock race",
group_id, thread_id));
}
wait_cnt_.fetch_add(1, std::memory_order_release);
if (!work_queue_.empty()) {
wait_cnt_.fetch_sub(1, std::memory_order_release);
continue; /// dealing task first
}
int n;
if (epoll_wait)
n = ::epoll_wait(epfd_, events, static_cast<int>(max_events),
static_cast<int>(timeout));
else {
::pollfd fds{eventfd_, POLLIN, 0};
n = poll(&fds, 1, static_cast<int>(timeout));
if (n > 0) {
/// fake one
assert(1 == n);
events[0].data.fd = eventfd_;
events[0].events = EPOLLIN;
}
}
loop_cnt_.fetch_add(1, std::memory_order_relaxed);
wait_cnt_.fetch_sub(1, std::memory_order_release);
if (0 == n) {
DBG_LOG(("polarx_rpc thread %u:%u leave epoll timeout, timeout %ums",
group_id, thread_id, timeout));
} else {
DBG_LOG(("polarx_rpc thread %u:%u leave epoll with %d events", group_id,
thread_id, n));
}
auto total = 0;
for (auto i = 0; i < n; ++i) {
if (events[i].data.fd == eventfd_) {
/// consume the event fd as soon as possible
/// which makes more threads notified if many tasks inserted
uint64_t dummy;
::read(eventfd_, &dummy, sizeof(dummy));
DBG_LOG(
("polarx_rpc thread %u:%u notified work", group_id, thread_id));
} else {
auto cb = reinterpret_cast<CepollCallback *>(events[i].data.ptr);
assert(cb != nullptr);
cb->pre_events();
++total;
}
}
/// Note: move timer callback here before events dealing
/// timer task only one thread is ok
if (timer_lock_.try_lock(timer_lock_node)) {
timer_tasks.clear();
auto now_time = Ctime::steady_ms();
task_t task;
int32_t id;
uint32_t type;
while (timer_heap_.pop(now_time, task, id, type))
timer_tasks.emplace_back(std::move(task));
timer_lock_.unlock(timer_lock_node);
/// run outside the lock
for (const auto &t : timer_tasks) {
t.call();
t.fin();
}
}
auto index = 0;
for (auto i = 0; i < n; ++i) {
if (events[i].data.fd == eventfd_)
continue; /// ignore it
auto cb = reinterpret_cast<CepollCallback *>(events[i].data.ptr);
assert(cb != nullptr);
auto bret = cb->events(events[i].events, index, total);
if (!bret)
delete cb;
++index;
}
/// do clean up on extra context
auto last_time = last_cleanup_.load(std::memory_order_relaxed);
auto now_time = Ctime::steady_ms();
if (UNLIKELY(now_time - last_time > epoll_group_ctx_refresh_time)) {
/// every 10s
if (last_cleanup_.compare_exchange_strong(last_time, now_time)) {
/// only one thread do this
uintptr_t first = 0;
for (auto i = 0; i < extra_ctx_.BUFFERED_REUSABLE_SESSION_COUNT;
++i) {
std::unique_ptr<reusable_session_t> s;
auto bret = extra_ctx_.reusable_sessions.pop(s);
if (!bret)
break;
/// 10 min lifetime
if (now_time - s->start_time_ms > shared_session_lifetime)
s.reset(); /// release
else {
auto ptr_val = reinterpret_cast<uintptr_t>(s.get());
extra_ctx_.reusable_sessions.push(std::move(s)); /// put it back
if (0 == first)
first = ptr_val;
else if (ptr_val == first)
break; /// all checked
}
}
}
}
}
Csession::deinit_thread_for_session();
plugin_info.threads.fetch_sub(1, std::memory_order_release);
}
explicit CmtEpoll(uint32_t group_id, size_t work_queue_depth)
: group_id_(group_id), work_queue_(work_queue_depth), wait_cnt_(0),
loop_cnt_(0), last_cleanup_(0), with_affinity_(true),
base_thread_count_(0), stall_count_(0), worker_count_(0),
tasker_count_(0), last_scale_time_(0), last_tasker_time_(0),
session_count_(0), last_head_(0), last_loop_(0) {
/// clear cpu set
CPU_ZERO(&cpus_);
/// init epoll
epfd_ = ::epoll_create(0xFFFF); // 65535
if (UNLIKELY(epfd_ < 0))
throw std::runtime_error(std::strerror(errno));
/// init eventfd
eventfd_ = ::eventfd(0, EFD_NONBLOCK);
if (UNLIKELY(eventfd_ < 0)) {
::close(epfd_);
throw std::runtime_error(std::strerror(errno));
}
/// register it
::epoll_event event;
event.data.fd = eventfd_;
event.events = EPOLLIN | EPOLLET; /// only notify one
auto iret = ::epoll_ctl(epfd_, EPOLL_CTL_ADD, eventfd_, &event);
if (UNLIKELY(iret != 0)) {
::close(eventfd_);
::close(epfd_);
throw std::runtime_error(std::strerror(errno));
}
}
~CmtEpoll() {
/// never exit
::abort();
}
inline void init_thread(uint32_t group_id, uint32_t threads,
const std::vector<CcpuInfo::cpu_info_t> &affinities,
int base_idx, int epoll_wait_threads,
int epoll_wait_gap) {
/// record threads count first
base_thread_count_ = worker_count_ = static_cast<int>(threads);
global_thread_count() += static_cast<int>(threads);
/// build group affinities first
std::ostringstream oss;
oss << '[';
for (uint32_t thread_id = 0; thread_id < threads; ++thread_id) {
auto affinity = base_idx + thread_id < affinities.size()
? affinities[base_idx + thread_id].processor
: -1;
/// record affinities
if (affinity < 0)
with_affinity_ = false;
else if (!CPU_ISSET(affinity, &cpus_)) {
CPU_SET(affinity, &cpus_); /// add to group set
if (thread_id != 0)
oss << ',';
oss << affinity;
}
}
if (with_affinity_) {
oss << ']';
cores_str_ = oss.str();
}
/// now cores_str_, with_affinity_ and cpus_ are valid
/// create threads
for (uint32_t thread_id = 0; thread_id < threads; ++thread_id) {
auto affinity = base_idx + thread_id < affinities.size()
? affinities[base_idx + thread_id].processor
: -1;
auto is_epoll_wait =
0 == thread_id % epoll_wait_gap && --epoll_wait_threads >= 0;
/// all thread is base thread when init
std::thread thread(&CmtEpoll::loop, this, group_id, thread_id, true,
affinity, is_epoll_wait, true);
thread.detach();
}
}
static inline int get_core_number() {
#if defined(_WIN32)
SYSTEM_INFO info;
GetSystemInfo(&info);
return (int)info.dwNumberOfProcessors;
#elif defined(__APPLE__)
auto ncpu = 1;
auto len = sizeof(ncpu);
::sysctlbyname("hw.activecpu", &ncpu, &len, nullptr, 0);
return ncpu;
#else
return ::get_nprocs();
#endif
}
public:
static inline std::atomic<int> &global_thread_count() {
static std::atomic<int> g_cnt(0);
return g_cnt;
}
static inline CmtEpoll **get_instance(size_t &instance_count) {
static std::once_flag once;
static CmtEpoll **inst = nullptr;
static size_t inst_cnt = 0;
if (UNLIKELY(nullptr == inst || 0 == inst_cnt)) {
std::call_once(once, []() {
/// recheck all variables to prevent part read when modifying
auto threads = epoll_threads_per_group;
if (UNLIKELY(threads <= 0))
threads = 1;
else if (UNLIKELY(threads > MAX_EPOLL_THREADS_PER_GROUP))
threads = MAX_EPOLL_THREADS_PER_GROUP;
auto groups = epoll_groups;
auto base_groups = groups;
if (groups <= 0) {
auto cores = get_core_number();
if (auto_cpu_affinity) {
cpu_set_t cpu;
CPU_ZERO(&cpu);
auto iret =
pthread_getaffinity_np(pthread_self(), sizeof(cpu), &cpu);
if (0 == iret) {
auto cpus = 0;
for (auto i = 0; i < CPU_SETSIZE; ++i) {
if (CPU_ISSET(i, &cpu))
++cpus;
}
cores = cpus; /// at most cpus can run
}
}
groups = cores / threads + (0 == cores % threads ? 0 : 1);
if (groups < min_auto_epoll_groups)
groups = (min_auto_epoll_groups / groups +
(0 == min_auto_epoll_groups % groups ? 0 : 1)) *
groups;
base_groups = groups;
/// dealing extra group
auto extra = epoll_extra_groups;
if (extra > MAX_EPOLL_EXTRA_GROUPS)
extra = MAX_EPOLL_EXTRA_GROUPS;
groups += extra;
}
if (UNLIKELY(base_groups > MAX_EPOLL_GROUPS))
base_groups = MAX_EPOLL_GROUPS;
if (UNLIKELY(groups > MAX_EPOLL_GROUPS))
groups = MAX_EPOLL_GROUPS;
std::vector<CcpuInfo::cpu_info_t> affinities;
if (auto_cpu_affinity) {
auto info_map = CcpuInfo::get_cpu_info();
cpu_set_t cpu;
CPU_ZERO(&cpu);
auto iret = pthread_getaffinity_np(pthread_self(), sizeof(cpu), &cpu);
if (0 == iret) {
for (auto i = 0; i < CPU_SETSIZE; ++i) {
auto it = info_map.find(i);
if (CPU_ISSET(i, &cpu) ||
(force_all_cores && it != info_map.end())) {
if (it == info_map.end())
/// no cpu info, just set to 0
affinities.emplace_back(CcpuInfo::cpu_info_t{i, 0, 0});
else
affinities.emplace_back(it->second);
}
}
/// sort before duplicate
/// result: 2314 -> 1234
std::sort(affinities.begin(), affinities.end());
/// if affinities not enough for base groups, just duplicate it
if (base_groups * threads > affinities.size()) {
auto duplicates = base_groups * threads / affinities.size();
if (duplicates > 1) {
/// result: 1234 -> 12341234
std::vector<CcpuInfo::cpu_info_t> final_affinities;
final_affinities.reserve(duplicates * affinities.size());
for (size_t i = 0; i < duplicates; ++i) {
for (const auto &item : affinities)
final_affinities.emplace_back(item);
}
affinities = final_affinities;
}
}
}
}
auto total_epoll_wait_threads = max_epoll_wait_total_threads;
if (0 == total_epoll_wait_threads)
total_epoll_wait_threads = groups * threads;
else if (UNLIKELY(total_epoll_wait_threads <
MIN_EPOLL_WAIT_TOTAL_THREADS))
total_epoll_wait_threads = MIN_EPOLL_WAIT_TOTAL_THREADS;
else if (UNLIKELY(total_epoll_wait_threads >
MAX_EPOLL_WAIT_TOTAL_THREADS))
total_epoll_wait_threads = MAX_EPOLL_WAIT_TOTAL_THREADS;
if (total_epoll_wait_threads < groups)
/// at least one thread wait on epoll
total_epoll_wait_threads = groups;
auto epoll_wait_threads_per_group = 1;
/// select some thread in epoll to do epoll_wait
while (epoll_wait_threads_per_group < static_cast<int>(threads) &&
(epoll_wait_threads_per_group + 1) * groups <=
total_epoll_wait_threads)
++epoll_wait_threads_per_group;
auto epoll_wait_threads_gap = threads / epoll_wait_threads_per_group;
auto work_queue_capacity = epoll_work_queue_capacity;
if (UNLIKELY(work_queue_capacity < MIN_WORK_QUEUE_CAPACITY))
work_queue_capacity = MIN_WORK_QUEUE_CAPACITY;
else if (UNLIKELY(work_queue_capacity > MAX_WORK_QUEUE_CAPACITY))
work_queue_capacity = MAX_WORK_QUEUE_CAPACITY;
auto tmp = new CmtEpoll *[groups];
for (uint32_t group_id = 0; group_id < groups; ++group_id) {
tmp[group_id] = new CmtEpoll(group_id, work_queue_capacity);
tmp[group_id]->init_thread(
group_id, threads, affinities, group_id * threads,
epoll_wait_threads_per_group, epoll_wait_threads_gap);
}
{
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll start with %u groups with each group %u "
"threads. With %u thread bind to fixed CPU core",
groups, threads, affinities.size());
}
inst_cnt = groups;
inst = tmp;
});
}
instance_count = inst_cnt;
return inst;
}
inline const uint32_t &group_id() const { return group_id_; }
/// 0 if success else -errno
inline int add_fd(int fd, uint32_t events, CepollCallback *cb,
bool tcp = true) const {
auto iret = nonblock(fd, 1);
if (UNLIKELY(iret != 0))
return iret;
if (tcp && UNLIKELY((iret = nodelay(fd, 1)) != 0))
return iret;
auto tmp = tcp_keep_alive;
if (UNLIKELY(tmp > MAX_TCP_KEEP_ALIVE))
tmp = MAX_TCP_KEEP_ALIVE;
if (tcp && tmp > 0 && UNLIKELY((iret = keepalive(fd, 1, tmp)) != 0))
return iret;
::epoll_event event;
event.data.ptr = cb;
event.events = events;
cb->set_fd(fd);
cb->fd_pre_register(); /// pre register before epoll add
DBG_LOG(("polarx_rpc epoll add fd %d", fd));
iret = ::epoll_ctl(epfd_, EPOLL_CTL_ADD, fd, &event);
DBG_LOG(("polarx_rpc epoll add fd %d done ret %d", fd, iret));
if (UNLIKELY(iret != 0)) {
auto bret = cb->fd_rollback_register();
if (!bret)
delete cb;
return -errno;
}
return 0;
}
/// 0 if success else -errno
inline int reset_fd(int fd, uint32_t events, CepollCallback *cb) const {
::epoll_event event;
event.data.ptr = cb;
event.events = events;
DBG_LOG(("polarx_rpc epoll mod fd %d", fd));
auto iret = ::epoll_ctl(epfd_, EPOLL_CTL_MOD, fd, &event);
DBG_LOG(("polarx_rpc epoll mod fd %d done ret %d", fd, iret));
return LIKELY(0 == iret) ? 0 : -errno;
}
/// 0 if success else -errno
inline int del_fd(int fd) const {
epoll_event dummy;
::memset(&dummy, 0, sizeof(dummy));
DBG_LOG(("polarx_rpc epoll del fd %d", fd));
auto iret = ::epoll_ctl(epfd_, EPOLL_CTL_DEL, fd, &dummy);
DBG_LOG(("polarx_rpc epoll del fd %d done ret %d", fd, iret));
return LIKELY(0 == iret) ? 0 : -errno;
}
static inline int check_port(uint16_t port) {
auto fd = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (UNLIKELY(fd <= 0))
return -errno;
sockaddr_in address;
::memset(&address, 0, sizeof(address));
if (::inet_pton(AF_INET, "127.0.0.1", &address.sin_addr.s_addr) != 1) {
auto err = errno;
::close(fd);
return -err;
}
address.sin_family = AF_INET;
address.sin_port = htons(port);
if (0 == ::connect(fd, reinterpret_cast<struct sockaddr *>(&address),
sizeof(address))) {
::close(fd);
return -EADDRINUSE;
}
auto err = errno;
::close(fd);
return ECONNREFUSED == err ? 0 : -err;
}
/// 0 if success else -errno
inline int listen_port(uint16_t port, CepollCallback *cb,
bool reuse = false) const {
sockaddr_in address;
::memset(&address, 0, sizeof(address));
address.sin_addr.s_addr = htonl(INADDR_ANY);
address.sin_port = htons(port);
auto fd = ::socket(AF_INET, SOCK_STREAM, 0);
if (UNLIKELY(fd <= 0))
return -errno;
int sock_op = 1;
::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &sock_op, sizeof(sock_op));
if (reuse)
::setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &sock_op, sizeof(sock_op));
if (UNLIKELY(::bind(fd, (struct sockaddr *)&address, sizeof(address)) !=
0)) {
auto err = errno;
::close(fd);
return -err;
}
auto depth = tcp_listen_queue;
if (UNLIKELY(depth < MIN_TCP_LISTEN_QUEUE))
depth = MIN_TCP_LISTEN_QUEUE;
else if (UNLIKELY(depth > MAX_TCP_LISTEN_QUEUE))
depth = MAX_TCP_LISTEN_QUEUE;
if (UNLIKELY(::listen(fd, depth) != 0)) {
auto err = errno;
::close(fd);
return -err;
}
int iret;
if (UNLIKELY((iret = add_fd(fd, EPOLLIN | EPOLLET, cb)) != 0)) {
::close(fd);
return iret;
}
return 0;
}
inline void push_trigger(task_t &&task, int64_t trigger_time) {
int32_t id;
int64_t last_time;
/// reuse work queue spin conf
CautoMcsSpinLock lck(timer_lock_, mcs_spin_cnt);
if (UNLIKELY(!timer_heap_.peak(last_time)))
last_time = trigger_time + 1;
timer_heap_.push(std::forward<task_t>(task), trigger_time, id);
lck.unlock();
if (UNLIKELY(last_time - trigger_time >= 0)) {
/// need notify to restart new thread to wait with smaller timeout
uint64_t dummy = 1;
::write(eventfd_, &dummy, sizeof(dummy));
}
}
inline bool push_work(task_t &&task) {
auto bret = work_queue_.push(std::forward<task_t>(task));
if (!bret)
return false;
/// read with write barrier
auto waiting = wait_cnt_.fetch_add(0, std::memory_order_acq_rel);
if (waiting > 0) {
/// notify if some one is in epoll
uint64_t dummy = 1;
::write(eventfd_, &dummy, sizeof(dummy));
}
return true;
}
inline epoll_group_ctx_t &get_extra_ctx() { return extra_ctx_; }
inline void add_stall_count() { ++stall_count_; }
inline void sub_stall_count() { --stall_count_; }
/// thread pool auto scale and shrink
inline bool worker_stall_since_last_check() {
auto head = work_queue_.head();
if (LIKELY(head != last_head_)) {
last_head_ = head;
return false;
}
/// consumer not moved
auto tail = work_queue_.tail();
if (head != tail) /// not empty
return true;
/// check epoll wait exists
auto loop = loop_cnt_.load(std::memory_order_acquire);
auto waits = wait_cnt_.load(std::memory_order_acquire);
if (likely(waits > 0)) {
last_loop_ = loop;
return false;
}
if (LIKELY(loop != last_loop_)) {
last_loop_ = loop;
return false;
}
return true; /// empty task but no thread wait on epoll
}
inline void force_scale_thread_pool() {
last_scale_time_.store(Ctime::steady_ms(), std::memory_order_release);
std::lock_guard<std::mutex> lck(scale_lock_);
if (worker_count_.load(std::memory_order_acquire) >=
session_count_.load(std::memory_order_acquire) + base_thread_count_) {
if (enable_thread_pool_log) {
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll %u thread pool force scale over limit, worker %d tasker "
"%d, session %d. Total threads %d.",
group_id_, worker_count_.load(std::memory_order_acquire),
tasker_count_.load(std::memory_order_acquire),
session_count_.load(std::memory_order_acquire),
global_thread_count().load(std::memory_order_acquire));
}
return; /// ignore if worker more than session
}
/// force scale one thread
++worker_count_;
++global_thread_count();
std::thread thread(&CmtEpoll::loop, this, group_id_, 999, false, -1, true,
true);
thread.detach();
if (enable_thread_pool_log) {
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll %u thread pool force scale to worker %d tasker %d. Total "
"threads %d.",
group_id_, worker_count_.load(std::memory_order_acquire),
tasker_count_.load(std::memory_order_acquire),
global_thread_count().load(std::memory_order_acquire));
}
}
inline const std::atomic<int> &session_count() const {
return session_count_;
}
inline std::atomic<int> &session_count() { return session_count_; }
inline void balance_tasker() {
auto pending = work_queue_.length();
auto workers = worker_count_.load(std::memory_order_acquire);
assert(workers >= 0);
auto taskers = tasker_count_.load(std::memory_order_acquire);
assert(taskers >= 0);
auto multiply = epoll_group_tasker_multiply;
auto multiply_low = multiply / 2;
if (multiply_low < 1)
multiply_low = 1;
if (pending * 2 > work_queue_.capacity() ||
pending > multiply_low * (workers + taskers)) {
last_tasker_time_.store(Ctime::steady_ms(), std::memory_order_release);
if (pending * 2 <= work_queue_.capacity() &&
pending <= multiply * (workers + taskers))
return; /// still under thresh
/// need balance
std::lock_guard<std::mutex> lck(scale_lock_);
workers = worker_count_.load(std::memory_order_acquire);
assert(workers >= 0);
taskers = tasker_count_.load(std::memory_order_acquire);
assert(taskers >= 0);
auto sessions = session_count_.load(std::memory_order_acquire);
assert(taskers >= 0);
if (workers + taskers < sessions &&
workers + taskers < static_cast<int>(pending)) {
auto extend = (pending - workers - taskers) / multiply;
if (0 == extend)
extend = 1;
if (extend > epoll_group_tasker_extend_step)
extend = epoll_group_tasker_extend_step;
tasker_count_ += extend;
global_thread_count() += extend;
for (size_t i = 0; i < extend; ++i) {
std::thread thread(&CmtEpoll::loop, this, group_id_, 999, false, -1,
enable_epoll_in_tasker, false);
thread.detach();
}
if (enable_thread_pool_log) {
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll %u thread pool tasker scale to %d, worker %d. Total "
"threads %d.",
group_id_, tasker_count_.load(std::memory_order_acquire),
worker_count_.load(std::memory_order_acquire),
global_thread_count().load(std::memory_order_acquire));
}
}
}
}
inline void try_scale_thread_pool(int wait_type) {
auto thresh = static_cast<int>(epoll_group_thread_scale_thresh);
if (UNLIKELY(thresh < 0))
thresh = 0;
else if (UNLIKELY(thresh >= base_thread_count_)) {
thresh = base_thread_count_ - 1;
assert(thresh >= 0);
}
auto stalled = stall_count_.load(std::memory_order_acquire);
assert(stalled >= 0);
auto workers = worker_count_.load(std::memory_order_acquire);
assert(workers >= 0);
auto prefer_thread_count =
static_cast<int>(base_thread_count_ + epoll_group_dynamic_threads);
/// refresh the last time if needed
if (stalled > workers - base_thread_count_ + thresh)
last_scale_time_.store(Ctime::steady_ms(), std::memory_order_release);
else if (workers >= prefer_thread_count) {
if (stalled > workers / 4)
last_scale_time_.store(Ctime::steady_ms(), std::memory_order_release);
return; /// do nothing
}
/// do scale if needed(recheck in lock)
std::lock_guard<std::mutex> lck(scale_lock_);
stalled = stall_count_.load(std::memory_order_acquire);
assert(stalled >= 0);
workers = worker_count_.load(std::memory_order_acquire);
assert(workers >= 0);
if (workers >=
session_count_.load(std::memory_order_acquire) + base_thread_count_) {
if (enable_thread_pool_log) {
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll %u thread pool scale over limit, worker %d tasker %d, "
"session %d. Total threads %d.",
group_id_, worker_count_.load(std::memory_order_acquire),
tasker_count_.load(std::memory_order_acquire),
session_count_.load(std::memory_order_acquire),
global_thread_count().load(std::memory_order_acquire));
}
return; /// ignore if worker more than session
}
auto scaled = false;
if (stalled > workers - base_thread_count_ + thresh) {
/// need extra thread to handle new request
++worker_count_;
++global_thread_count();
std::thread thread(&CmtEpoll::loop, this, group_id_, 999, false, -1, true,
true);
thread.detach();
scaled = true;
} else if (workers < prefer_thread_count) {
do {
++worker_count_;
++global_thread_count();
std::thread thread(&CmtEpoll::loop, this, group_id_, 999, false, -1,
true, true);
thread.detach();
} while (worker_count_.load(std::memory_order_acquire) <
prefer_thread_count);
scaled = true;
}
if (scaled && enable_thread_pool_log) {
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll %u thread pool scale to worker %d tasker %d. Total "
"threads %d. wait_type %d",
group_id_, worker_count_.load(std::memory_order_acquire),
tasker_count_.load(std::memory_order_acquire),
global_thread_count().load(std::memory_order_acquire), wait_type);
}
}
inline bool shrink_thread_pool(bool is_worker) {
if (!is_worker) {
/// tasker thread
if (Ctime::steady_ms() -
last_tasker_time_.load(std::memory_order_acquire) <=
epoll_group_dynamic_threads_shrink_time)
return false;
/// free it
--tasker_count_;
--global_thread_count();
if (enable_thread_pool_log) {
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll %u thread pool shrink to worker %d tasker %d. Total "
"thread %d.",
group_id_, worker_count_.load(std::memory_order_acquire),
tasker_count_.load(std::memory_order_acquire),
global_thread_count().load(std::memory_order_acquire));
}
return true;
}
auto bret = false;
auto prefer_thread_count =
static_cast<int>(base_thread_count_ + epoll_group_dynamic_threads);
auto thresh = static_cast<int>(epoll_group_thread_scale_thresh);
if (UNLIKELY(thresh < 0))
thresh = 0;
else if (UNLIKELY(thresh >= base_thread_count_)) {
thresh = base_thread_count_ - 1;
assert(thresh >= 0);
}
auto stalled = stall_count_.load(std::memory_order_acquire);
assert(stalled >= 0);
auto workers = worker_count_.load(std::memory_order_acquire);
assert(workers >= 0);
/// enter mutex only when we need to do
if (stalled < workers - base_thread_count_ + thresh &&
Ctime::steady_ms() - last_scale_time_.load(std::memory_order_acquire) >
epoll_group_dynamic_threads_shrink_time &&
workers > prefer_thread_count) {
/// shrink only when no waiting exists and no multiple stall for a while
std::lock_guard<std::mutex> lck(scale_lock_);
/// recheck waiting
stalled = stall_count_.load(std::memory_order_acquire);
if (worker_count_.load(std::memory_order_acquire) > prefer_thread_count &&
stalled < prefer_thread_count - 1) {
--worker_count_;
--global_thread_count();
bret = true;
if (enable_thread_pool_log) {
std::lock_guard<std::mutex> plugin_lck(plugin_info.mutex);
if (plugin_info.plugin_info != nullptr)
my_plugin_log_message(
&plugin_info.plugin_info, MY_WARNING_LEVEL,
"MtEpoll %u thread pool shrink to worker %d tasker %d. Total "
"threads %d.",
group_id_, worker_count_.load(std::memory_order_acquire),
tasker_count_.load(std::memory_order_acquire),
global_thread_count().load(std::memory_order_acquire));
}
}
}
return bret;
}
};
} // namespace polarx_rpc
Reference in New Issue View Git Blame Copy Permalink