本篇实现Linux网络库epoll+时间堆实现高性能定时器,采用C语言实现
一、初衷
如何用epoll实现多个定时器的操作是本设计的关键,我们知道,epoll_wait的最后一个参数是阻塞等待的时候,单位是毫秒。可以这样设计:
1、当时间堆中没有定时器时,epoll_wait的超时时间T设为-1,表示一直阻塞等待新用户的到来;
2、当时间堆中有定时器时,epoll_wait的超时时间T设为最小堆堆顶的超时值,这样可以保证让最近触发的定时器能得以执行;
3、在epoll_wait阻塞等待期间,若有其它的用户到来,则epoll_wait返回n>0,进行常规的处理,随后应重新设置epoll_wait为小顶堆堆顶的超时时间。
二、源码
1、CMakeLists.txt
cmake_minimum_required(VERSION 2.8) PROJECT(min_heap_libevent_epoll) AUX_SOURCE_DIRECTORY(. SRC_LIST) ADD_EXECUTABLE(${PROJECT_NAME} ${SRC_LIST})
2、minheap-event-firecat.h
#ifndef MINHEAPEVENTFIRECAT_H #define MINHEAPEVENTFIRECAT_H #include <sys/time.h> #include <time.h> #include <stdio.h> #include <stdlib.h> //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/mm-internal.h #define mm_malloc(sz) malloc(sz) #define mm_calloc(n, sz) calloc((n), (sz)) #define mm_strdup(s) strdup(s) #define mm_realloc(p, sz) realloc((p), (sz)) #define mm_free(p) free(p) //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/include/event2/util.h #define evutil_timercmp(tvp, uvp, cmp) \ (((tvp)->tv_sec == (uvp)->tv_sec) ? \ ((tvp)->tv_usec cmp (uvp)->tv_usec) : \ ((tvp)->tv_sec cmp (uvp)->tv_sec)) #define evutil_timersub(tvp, uvp, vvp) \ do { \ (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ if ((vvp)->tv_usec < 0) { \ (vvp)->tv_sec--; \ (vvp)->tv_usec += 1000000; \ } \ } while (0) #define evutil_timeradd(tvp, uvp, vvp) \ do { \ (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \ (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec; \ if ((vvp)->tv_usec >= 1000000) { \ (vvp)->tv_sec++; \ (vvp)->tv_usec -= 1000000; \ } \ } while (0) //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/include/event2/event_struct.h struct event { /* for managing timeouts */ union { //TAILQ_ENTRY(event) ev_next_with_common_timeout; int min_heap_idx; } ev_timeout_pos; unsigned int timer_id; struct timeval ev_interval; struct timeval ev_timeout; int ev_exe_num; void (*ev_callback)(void *arg); int ev_arg; int ev_res; /* result passed to event callback */ int ev_flags; }; static inline void gettime(struct timeval *tm); void gettime(struct timeval *tm) { gettimeofday(tm, NULL); } #endif // MINHEAPEVENTFIRECAT_H
3、minheap-internal.h
/* * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson * * Copyright (c) 2006 Maxim Yegorushkin <maxim.yegorushkin@gmail.com> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef MINHEAP_INTERNAL_H_INCLUDED_ #define MINHEAP_INTERNAL_H_INCLUDED_ //come from https://github.com/libevent/libevent/blob/release-2.1.8-stable/minheap-internal.h /*firecat del #include "event2/event-config.h" #include "evconfig-private.h" #include "event2/event.h" #include "event2/event_struct.h" #include "event2/util.h" #include "util-internal.h" #include "mm-internal.h" */ #include "minheap-event-firecat.h" //firecat add typedef struct min_heap { struct event** p; unsigned n, a; } min_heap_t; static inline void min_heap_ctor_(min_heap_t* s); static inline void min_heap_dtor_(min_heap_t* s); static inline void min_heap_elem_init_(struct event* e); static inline int min_heap_elt_is_top_(const struct event *e); static inline int min_heap_empty_(min_heap_t* s); static inline unsigned min_heap_size_(min_heap_t* s); static inline struct event* min_heap_top_(min_heap_t* s); static inline int min_heap_reserve_(min_heap_t* s, unsigned n); static inline int min_heap_push_(min_heap_t* s, struct event* e); static inline struct event* min_heap_pop_(min_heap_t* s); static inline int min_heap_adjust_(min_heap_t *s, struct event* e); static inline int min_heap_erase_(min_heap_t* s, struct event* e); static inline void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e); static inline void min_heap_shift_up_unconditional_(min_heap_t* s, unsigned hole_index, struct event* e); static inline void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e); #define min_heap_elem_greater(a, b) \ (evutil_timercmp(&(a)->ev_timeout, &(b)->ev_timeout, >)) void min_heap_ctor_(min_heap_t* s) { s->p = 0; s->n = 0; s->a = 0; } void min_heap_dtor_(min_heap_t* s) { if (s->p) mm_free(s->p); } void min_heap_elem_init_(struct event* e) { e->ev_timeout_pos.min_heap_idx = -1; } int min_heap_empty_(min_heap_t* s) { return 0u == s->n; } unsigned min_heap_size_(min_heap_t* s) { return s->n; } struct event* min_heap_top_(min_heap_t* s) { return s->n ? *s->p : 0; } int min_heap_push_(min_heap_t* s, struct event* e) { if (min_heap_reserve_(s, s->n + 1)) return -1; min_heap_shift_up_(s, s->n++, e); return 0; } struct event* min_heap_pop_(min_heap_t* s) { if (s->n) { struct event* e = *s->p; min_heap_shift_down_(s, 0u, s->p[--s->n]); e->ev_timeout_pos.min_heap_idx = -1; return e; } return 0; } int min_heap_elt_is_top_(const struct event *e) { return e->ev_timeout_pos.min_heap_idx == 0; } int min_heap_erase_(min_heap_t* s, struct event* e) { if (-1 != e->ev_timeout_pos.min_heap_idx) { struct event *last = s->p[--s->n]; unsigned parent = (e->ev_timeout_pos.min_heap_idx - 1) / 2; /* we replace e with the last element in the heap. We might need to shift it upward if it is less than its parent, or downward if it is greater than one or both its children. Since the children are known to be less than the parent, it can't need to shift both up and down. */ if (e->ev_timeout_pos.min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], last)) min_heap_shift_up_unconditional_(s, e->ev_timeout_pos.min_heap_idx, last); else min_heap_shift_down_(s, e->ev_timeout_pos.min_heap_idx, last); e->ev_timeout_pos.min_heap_idx = -1; return 0; } return -1; } int min_heap_adjust_(min_heap_t *s, struct event *e) { if (-1 == e->ev_timeout_pos.min_heap_idx) { return min_heap_push_(s, e); } else { unsigned parent = (e->ev_timeout_pos.min_heap_idx - 1) / 2; /* The position of e has changed; we shift it up or down * as needed. We can't need to do both. */ if (e->ev_timeout_pos.min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], e)) min_heap_shift_up_unconditional_(s, e->ev_timeout_pos.min_heap_idx, e); else min_heap_shift_down_(s, e->ev_timeout_pos.min_heap_idx, e); return 0; } } int min_heap_reserve_(min_heap_t* s, unsigned n) { if (s->a < n) { struct event** p; unsigned a = s->a ? s->a * 2 : 8; if (a < n) a = n; if (!(p = (struct event**)mm_realloc(s->p, a * sizeof *p))) return -1; s->p = p; s->a = a; } return 0; } void min_heap_shift_up_unconditional_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned parent = (hole_index - 1) / 2; do { (s->p[hole_index] = s->p[parent])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = parent; parent = (hole_index - 1) / 2; } while (hole_index && min_heap_elem_greater(s->p[parent], e)); (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned parent = (hole_index - 1) / 2; while (hole_index && min_heap_elem_greater(s->p[parent], e)) { (s->p[hole_index] = s->p[parent])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = parent; parent = (hole_index - 1) / 2; } (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct event* e) { unsigned min_child = 2 * (hole_index + 1); while (min_child <= s->n) { min_child -= min_child == s->n || min_heap_elem_greater(s->p[min_child], s->p[min_child - 1]); if (!(min_heap_elem_greater(e, s->p[min_child]))) break; (s->p[hole_index] = s->p[min_child])->ev_timeout_pos.min_heap_idx = hole_index; hole_index = min_child; min_child = 2 * (hole_index + 1); } (s->p[hole_index] = e)->ev_timeout_pos.min_heap_idx = hole_index; } #endif /* MINHEAP_INTERNAL_H_INCLUDED_ */
4、mytimer.h -- 与第1篇不同,本篇采用C语言实现。timer_add的函数参数也有调整,新增了int timer_id,int arg等参数。
#ifndef MYTIMER_H #define MYTIMER_H #include "minheap-internal.h" #define LIMIT_TIMER 1 //有限次数定时器 #define CYCLE_TIMER 2 //循环定时器 //typedef enum {false, true} bool; extern struct min_heap _min_heap; extern void timer_init(); extern void timer_destroy(); extern unsigned int timer_add(unsigned int timer_id, int interval, void (*fun)(void*), int arg, int flag, int exe_num); extern int timer_remove(unsigned int timer_id); extern int timer_process(); #endif // MYTIMER_H
5、mytimer.c
#include "mytimer.h" struct min_heap _min_heap; void timer_init() { min_heap_ctor_(&_min_heap); } void timer_destroy() { int i = 0; for (i = 0; i < _min_heap.n; i++) { free(_min_heap.p[i]); } min_heap_dtor_(&_min_heap); } unsigned int timer_add(unsigned int timer_id, int interval, void(*fun)(void*), int arg, int flag /* = CYCLE_TIMER */, int exe_num /* = 0 */) { struct event * ev = (struct event*) malloc(sizeof(struct event)); min_heap_elem_init_(ev); if (NULL == ev) return NULL; struct timeval now; gettime(&now); ev->ev_interval.tv_sec = interval / 1000; ev->ev_interval.tv_usec = (interval % 1000) * 1000; evutil_timeradd(&now, &(ev->ev_interval), &(ev->ev_timeout)); ev->ev_flags = flag; ev->ev_callback = fun; ev->ev_arg = arg; ev->ev_exe_num = exe_num; ev->timer_id = timer_id; min_heap_push_(&_min_heap, ev); return ev->timer_id; } int timer_remove(unsigned int timer_id) { int i = 0; for (i = 0; i < _min_heap.n; i++) { if (timer_id == _min_heap.p[i]->timer_id) { struct event * e = _min_heap.p[i]; min_heap_erase_(&_min_heap, _min_heap.p[i]); free(e); return 1; } } return 0; } int timer_process() { struct event *event; struct timeval now; while ((event = min_heap_top_(&_min_heap)) != NULL) { gettime(&now); if (evutil_timercmp(&now, &(event->ev_timeout), < )) break; min_heap_pop_(&_min_heap); event->ev_callback(event->ev_arg); if (event->ev_flags == CYCLE_TIMER || (event->ev_flags == LIMIT_TIMER && --event->ev_exe_num > 0)) { evutil_timeradd(&(event->ev_timeout), &(event->ev_interval), &(event->ev_timeout)); min_heap_push_(&_min_heap, event); } else { free(event); } } return 0; }
6、main.c -- 重点关注一下函数static void do_epoll(int listenfd)
#include <stdio.h> #include <stdlib.h> #include <string.h> #include <errno.h> #include <sys/socket.h> #include <sys/epoll.h> #include <sys/types.h> #include <sys/resource.h> /*setrlimit */ #include <unistd.h> #include <string.h> #include <fcntl.h> #include <netinet/in.h> #include <arpa/inet.h> #include <assert.h> #include <signal.h> #include <pthread.h> #include "mytimer.h" #define IPADDRESS "127.0.0.1" #define PORT 8011 #define MAXSIZE 1024 #define LISTENQ 5 #define FDSIZE 50000 #define EPOLLEVENTS 100 #define CONFIG_MIN_RESERVED_FDS 32 //come from redis src #define CONFIG_FDSET_INCR (CONFIG_MIN_RESERVED_FDS+96) int stop_server = 0; //函数声明 //创建套接字并进行绑定 static int socket_bind(const char* ip,int port); //IO多路复用epoll static void do_epoll(int listenfd); //事件处理函数 static void handle_events(int epollfd,struct epoll_event *events,int num,int listenfd,char *buf); //处理接收到的连接 static void handle_accpet(int epollfd,int listenfd); //读处理 static void do_read(int epollfd,int fd,char *buf); //写处理 static void do_write(int epollfd,int fd,char *buf); //添加事件 static void add_event(int epollfd,int fd,int state); //修改事件 static void modify_event(int epollfd,int fd,int state); //删除事件 static void delete_event(int epollfd,int fd,int state); //other static int do_error(int fd, int *error); static int setnonblocking(int fd); static void daemonize(void); static int set_fdlimit(); static void signal_exit_handler(); static void signal_exit_func(int signo); static void handle_timer(); static void timerfun_callback(int arg); int main(int argc,char *argv[]) { //设置每个进程允许打开的最大文件数,socket if (set_fdlimit() < 0) { return -1; } int background = 0; if (background) { daemonize(); } //设置信号处理,SIG_IGN表示忽略信号,SIG_DFL表示使用信号的默认处理方式 //signal(SIGHUP, SIG_IGN); //开启的话,就捕获不到终端窗口关闭的信号了。即窗口关闭,进程仍然进行。 signal(SIGPIPE, SIG_IGN); /* if (argc != 2) { fprintf(stderr, "Usage: %s port\n", argv[0]); return 1; } int port = atoi(argv[1]);*/ int listenfd; listenfd = socket_bind(IPADDRESS,PORT); listen(listenfd,LISTENQ); printf("start listening...\n"); signal_exit_handler(); timer_init(); do_epoll(listenfd); timer_destroy(); return 0; } static int socket_bind(const char* ip,int port) { int listenfd; struct sockaddr_in servaddr; listenfd = socket(AF_INET,SOCK_STREAM,0); if (listenfd == -1) { perror("socket error:"); exit(1); } bzero(&servaddr,sizeof(servaddr)); servaddr.sin_family = AF_INET; //inet_pton(AF_INET,ip,&servaddr.sin_addr); servaddr.sin_port = htons(port); servaddr.sin_addr.s_addr = htonl(INADDR_ANY); int error; int reuse = 1; int ret = setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); if (ret == -1) { return do_error(listenfd, &error); } if (bind(listenfd,(struct sockaddr*)&servaddr,sizeof(servaddr)) == -1) { perror("bind error: "); exit(1); } return listenfd; } static void do_epoll(int listenfd) { int epollfd; struct epoll_event events[EPOLLEVENTS]; int ret; char buf[MAXSIZE]; memset(buf,0,MAXSIZE); //创建一个描述符 int error; epollfd = epoll_create(1024);//1024 is just a hint for the kernel if (epollfd == -1) { return do_error(epollfd, &error); } //添加监听描述符事件 add_event(epollfd,listenfd,EPOLLIN); struct event *event; struct timeval now; struct timeval tv; struct timeval *tvp = NULL; while ( stop_server == 0 ) { if ((event = min_heap_top_(&_min_heap)) != NULL) { gettime(&now); tv.tv_sec = event->ev_timeout.tv_sec - now.tv_sec;; tv.tv_usec = event->ev_timeout.tv_usec - now.tv_usec; if ( tv.tv_usec < 0 ) { tv.tv_usec += 1000000; tv.tv_sec--; } tvp = &tv; } else { tvp = NULL; } //获取已经准备好的描述符事件 if (tvp == NULL) { ret = epoll_wait(epollfd,events,EPOLLEVENTS,-1); } else { ret = epoll_wait(epollfd, events, EPOLLEVENTS, tvp->tv_sec*1000 + tvp->tv_usec/1000); } handle_events(epollfd,events,ret,listenfd,buf); handle_timer(); } close(epollfd); } static void handle_events(int epollfd,struct epoll_event *events,int num,int listenfd,char *buf) { int i; int fd; //进行选好遍历 for (i = 0;i < num;i++) { fd = events[i].data.fd; //根据描述符的类型和事件类型进行处理 if ((fd == listenfd) &&(events[i].events & EPOLLIN)) handle_accpet(epollfd,listenfd); else if (events[i].events & EPOLLIN) do_read(epollfd,fd,buf); else if (events[i].events & EPOLLOUT) do_write(epollfd,fd,buf); } } static void handle_timer() { timer_process(); } static void handle_accpet(int epollfd,int listenfd) { int clifd; struct sockaddr_in cliaddr; socklen_t cliaddrlen = sizeof(cliaddr); clifd = accept(listenfd,(struct sockaddr*)&cliaddr,&cliaddrlen); if (clifd == -1) perror("accpet error:"); else { printf("accept a new client: %s:%d\n",inet_ntoa(cliaddr.sin_addr),cliaddr.sin_port); //添加一个客户描述符和事件 add_event(epollfd,clifd,EPOLLIN); //add timer timer_add(clifd, 1000, timerfun_callback, clifd, CYCLE_TIMER, 0);//ms } } static void do_read(int epollfd,int fd,char *buf) { int nread; nread = read(fd,buf,MAXSIZE); if (nread == -1) { perror("read error:"); close(fd); delete_event(epollfd,fd,EPOLLIN); } else if (nread == 0) { fprintf(stderr,"client close,fd=%d\n",fd); close(fd); delete_event(epollfd,fd,EPOLLIN); timer_remove(fd); } else { printf("read message is: %s,fd=%d\n",buf,fd); //修改描述符对应的事件,由读改为写 modify_event(epollfd,fd,EPOLLOUT); } } static void do_write(int epollfd,int fd,char *buf) { int nwrite; nwrite = write(fd,buf,strlen(buf)); if (nwrite == -1) { perror("write error:"); close(fd); delete_event(epollfd,fd,EPOLLOUT); } else modify_event(epollfd,fd,EPOLLIN); memset(buf,0,MAXSIZE); } static void add_event(int epollfd,int fd,int state) { struct epoll_event ev; ev.events = state; ev.data.fd = fd; epoll_ctl(epollfd,EPOLL_CTL_ADD,fd,&ev); setnonblocking(fd); } static void delete_event(int epollfd,int fd,int state) { struct epoll_event ev; ev.events = state; ev.data.fd = fd; epoll_ctl(epollfd,EPOLL_CTL_DEL,fd,&ev); } static void modify_event(int epollfd,int fd,int state) { struct epoll_event ev; ev.events = state; ev.data.fd = fd; epoll_ctl(epollfd,EPOLL_CTL_MOD,fd,&ev); } static int do_error(int fd, int *error) { fprintf(stderr, "error: %s\n", strerror(errno)); *error = errno; while ((close(fd) == -1) && (errno == EINTR)); errno = *error; return 1; } static int setnonblocking(int fd) { int old_option = fcntl(fd, F_GETFL); int new_option = old_option | O_NONBLOCK; fcntl(fd, F_SETFL, new_option); return old_option; } static void daemonize(void) { //come from /redis/server.c/daemonize() int fd; if (fork() != 0) exit(0); /* parent exits */ setsid(); /* create a new session */ /* Every output goes to /dev/null. If Redis is daemonized but * the 'logfile' is set to 'stdout' in the configuration file * it will not log at all. */ if ((fd = open("/dev/null", O_RDWR, 0)) != -1) { dup2(fd, STDIN_FILENO); dup2(fd, STDOUT_FILENO); dup2(fd, STDERR_FILENO); if (fd > STDERR_FILENO) close(fd); } } static int set_fdlimit() { //设置每个进程允许打开的最大文件数 //这项功能等价于linux终端命令 "ulimit -n 102400" struct rlimit rt; rt.rlim_max = rt.rlim_cur = FDSIZE + CONFIG_FDSET_INCR; if (setrlimit(RLIMIT_NOFILE, &rt) == -1) { perror("setrlimit error"); return -1; } return 0; } static void signal_exit_handler() { struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_handler = signal_exit_func; sigaction(SIGINT, &sa, NULL);//当按下ctrl+c时,它的效果就是发送SIGINT信号 sigaction(SIGTERM, &sa, NULL);//kill pid sigaction(SIGQUIT, &sa, NULL);//ctrl+\代表退出SIGQUIT //SIGSTOP和SIGKILL信号是不可捕获的,所以下面两句话写了等于没有写 sigaction(SIGKILL, &sa, NULL);//kill -9 pid sigaction(SIGSTOP, &sa, NULL);//ctrl+z代表停止 //#define SIGTERM 15 //#define SIGKILL 9 //kill和kill -9,两个命令在linux中都有杀死进程的效果,然而两命令的执行过程却大有不同,在程序中如果用错了,可能会造成莫名其妙的现象。 //执行kill pid命令,系统会发送一个SIGTERM信号给对应的程序。 //执行kill -9 pid命令,系统给对应程序发送的信号是SIGKILL,即exit。exit信号不会被系统阻塞,所以kill -9能顺利杀掉进程。 } static void signal_exit_func(int signo) { printf("exit signo is %d\n", signo); stop_server = 1; } static void timerfun_callback(int arg) { printf("id is %d\n", arg); }
---
参考文章:
我个人的Linux TCP server和client测试源码,C语言(2)(★firecat推荐★)
linux网络编程二十三:高性能定时器之时间堆 --- 《Linux高性能服务器编程(作者游双) 11.4.2》
IO多路复用之epoll总结 --- 注意源码有一处错误,需要修正为:
static void handle_accpet(int epollfd,int listenfd)
{
int clifd;
struct sockaddr_in cliaddr;
socklen_t cliaddrlen = sizeof(cliaddr);