linux 内核poll/select/epoll实现剖析

static int ep_remove(struct eventpoll *ep, struct epitem *epi)  
{  
    unsigned long flags;  
    struct file *file = epi->ffd.file;  
  
    /* 
     * Removes poll wait queue hooks. We _have_ to do this without holding 
     * the "ep->lock" otherwise a deadlock might occur. This because of the 
     * sequence of the lock acquisition. Here we do "ep->lock" then the wait 
     * queue head lock when unregistering the wait queue. The wakeup callback 
     * will run by holding the wait queue head lock and will call our callback 
     * that will try to get "ep->lock". 
     */  
    ep_unregister_pollwait(ep, epi);  
  
    /* Remove the current item from the list of epoll hooks */  
    spin_lock(&file->f_lock);  
    if (ep_is_linked(&epi->fllink))  
        list_del_init(&epi->fllink);  
    spin_unlock(&file->f_lock);  
  
    rb_erase(&epi->rbn, &ep->rbr);  
  
    spin_lock_irqsave(&ep->lock, flags);  
    if (ep_is_linked(&epi->rdllink))  
        list_del_init(&epi->rdllink);  
    spin_unlock_irqrestore(&ep->lock, flags);  
  
    /* At this point it is safe to free the eventpoll item */  
    kmem_cache_free(epi_cache, epi);  
  
    atomic_long_dec(&ep->user->epoll_watches);  
  
    return 0;  
}

/* 
 * Modify the interest event mask by dropping an event if the new mask 
 * has a match in the current file status. Must be called with "mtx" held. 
 */  
static int ep_modify(struct eventpoll *ep, struct epitem *epi, struct epoll_event *event)  
{  
    int pwake = 0;  
    unsigned int revents;  
    poll_table pt;  
  
    init_poll_funcptr(&pt, NULL);  
  
    /* 
     * Set the new event interest mask before calling f_op->poll(); 
     * otherwise we might miss an event that happens between the 
     * f_op->poll() call and the new event set registering. 
     */  
    epi->event.events = event->events;  
    pt._key = event->events;  
    epi->event.data = event->data; /* protected by mtx */  
  
    /* 
     * Get current event bits. We can safely use the file* here because 
     * its usage count has been increased by the caller of this function. 
     */  
    revents = epi->ffd.file->f_op->poll(epi->ffd.file, &pt);  
  
    /* 
     * If the item is "hot" and it is not registered inside the ready 
     * list, push it inside. 
     */  
    if (revents & event->events) {  
        spin_lock_irq(&ep->lock);  
        if (!ep_is_linked(&epi->rdllink)) {  
            list_add_tail(&epi->rdllink, &ep->rdllist);  
  
            /* Notify waiting tasks that events are available */  
            if (waitqueue_active(&ep->wq))  
                wake_up_locked(&ep->wq);  
            if (waitqueue_active(&ep->poll_wait))  
                pwake++;  
        }  
        spin_unlock_irq(&ep->lock);  
    }  
  
    /* We have to call this outside the lock */  
    if (pwake)  
        ep_poll_safewake(&ep->poll_wait);  
  
    return 0;  
}

/* 
epoll_wait实现 
*/  
  
SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,  
                int, maxevents, int, timeout)  
{  
    int error;  
    struct file *file;  
    struct eventpoll *ep;  
  
    // 检查输入数据有效性  
    if (maxevents <= 0 || maxevents > EP_MAX_EVENTS) {  
        return -EINVAL;  
    }  
  
    if (!access_ok(VERIFY_WRITE, events, maxevents * sizeof(struct epoll_event))) {  
        error = -EFAULT;  
        goto error_return;  
    }  
  
    /* Get the "struct file *" for the eventpoll file */  
    error = -EBADF;  
    file = fget(epfd);  
    if (!file) {  
        goto error_return;  
    }  
  
    error = -EINVAL;  
    if (!is_file_epoll(file)) {  
        goto error_fput;  
    }  
    // 取得ep 结构  
    ep = file->private_data;  
  
    // 等待事件  
    error = ep_poll(ep, events, maxevents, timeout);  
  
error_fput:  
    fput(file);  
error_return:  
  
    return error;  
}  
  
static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,  
                   int maxevents, long timeout)  
{  
    int res = 0, eavail, timed_out = 0;  
    unsigned long flags;  
    long slack = 0;  
    wait_queue_t wait;  
    ktime_t expires, *to = NULL;  
  
    if (timeout > 0) {  
        // 转换为内核时间  
        struct timespec end_time = ep_set_mstimeout(timeout);  
  
        slack = select_estimate_accuracy(&end_time);  
        to = &expires;  
        *to = timespec_to_ktime(end_time);  
    } else if (timeout == 0) {  
        // 已经超时直接检查readylist  
        timed_out = 1;  
        spin_lock_irqsave(&ep->lock, flags);  
        goto check_events;  
    }  
  
fetch_events:  
    spin_lock_irqsave(&ep->lock, flags);  
  
    // 没有可用的事件，ready list 和ovflist 都为空  
    if (!ep_events_available(ep)) {  
  
        // 添加当前进程的唤醒函数  
        init_waitqueue_entry(&wait, current);  
        __add_wait_queue_exclusive(&ep->wq, &wait);  
  
        for (;;) {  
            /* 
             * We don't want to sleep if the ep_poll_callback() sends us 
             * a wakeup in between. That's why we set the task state 
             * to TASK_INTERRUPTIBLE before doing the checks. 
             */  
            set_current_state(TASK_INTERRUPTIBLE);  
            if (ep_events_available(ep) || timed_out) {  
                break;  
            }  
            if (signal_pending(current)) {  
                res = -EINTR;  
                break;  
            }  
  
            spin_unlock_irqrestore(&ep->lock, flags);  
            // 挂起当前进程，等待唤醒或超时  
            if (!schedule_hrtimeout_range(to, slack, HRTIMER_MODE_ABS)) {  
                timed_out = 1;  
            }  
  
            spin_lock_irqsave(&ep->lock, flags);  
        }  
      
        __remove_wait_queue(&ep->wq, &wait);  
  
        set_current_state(TASK_RUNNING);  
    }  
check_events:  
    // 再次检查是否有可用事件  
    eavail = ep_events_available(ep);  
  
    spin_unlock_irqrestore(&ep->lock, flags);  
  
    /* 
     * Try to transfer events to user space. In case we get 0 events and 
     * there's still timeout left over, we go trying again in search of 
     * more luck. 
     */  
    if (!res && eavail   
            && !(res = ep_send_events(ep, events, maxevents)) // 复制事件到用户空间  
            && !timed_out) // 复制事件失败并且没有超时，重新等待。  
            {  
        goto fetch_events;  
    }  
  
    return res;  
}  
  
  
static inline int ep_events_available(struct eventpoll *ep)  
{  
    return !list_empty(&ep->rdllist) || ep->ovflist != EP_UNACTIVE_PTR;  
}  
  
struct ep_send_events_data {  
    int maxevents;  
    struct epoll_event __user *events;  
};  
  
static int ep_send_events(struct eventpoll *ep,  
                          struct epoll_event __user *events, int maxevents)  
{  
    struct ep_send_events_data esed;  
  
    esed.maxevents = maxevents;  
    esed.events = events;  
  
    return ep_scan_ready_list(ep, ep_send_events_proc, &esed, 0);  
}  
  
static int ep_send_events_proc(struct eventpoll *ep, struct list_head *head,  
                               void *priv)  
{  
    struct ep_send_events_data *esed = priv;  
    int eventcnt;  
    unsigned int revents;  
    struct epitem *epi;  
    struct epoll_event __user *uevent;  
  
    // 遍历已就绪链表  
    for (eventcnt = 0, uevent = esed->events;  
            !list_empty(head) && eventcnt < esed->maxevents;) {  
        epi = list_first_entry(head, struct epitem, rdllink);  
  
        list_del_init(&epi->rdllink);  
        // 获取ready 事件掩码  
        revents = epi->ffd.file->f_op->poll(epi->ffd.file, NULL) &  
                  epi->event.events;  
  
        /* 
         * If the event mask intersect the caller-requested one, 
         * deliver the event to userspace. Again, ep_scan_ready_list() 
         * is holding "mtx", so no operations coming from userspace 
         * can change the item. 
         */  
        if (revents) {  
            // 事件就绪, 复制到用户空间  
            if (__put_user(revents, &uevent->events) ||  
                    __put_user(epi->event.data, &uevent->data)) {  
                list_add(&epi->rdllink, head);  
                return eventcnt ? eventcnt : -EFAULT;  
            }  
            eventcnt++;  
            uevent++;  
            if (epi->event.events & EPOLLONESHOT) {  
                epi->event.events &= EP_PRIVATE_BITS;  
            } else if (!(epi->event.events & EPOLLET)) {  
                // 不是边缘模式, 再次添加到ready list,  
                // 下次epoll_wait 时直接进入此函数检查ready list是否仍然继续  
                list_add_tail(&epi->rdllink, &ep->rdllist);  
            }  
            // 如果是边缘模式, 只有当文件状态发生改变时,  
            // 才文件会再次触发wait_address 上wait_queue的回调函数,  
        }  
    }  
  
    return eventcnt;  
}

static const struct file_operations eventpoll_fops = {  
    .release = ep_eventpoll_release,  
    .poll    = ep_eventpoll_poll,  
    .llseek  = noop_llseek,  
};  
  
static unsigned int ep_eventpoll_poll(struct file *file, poll_table *wait)  
{  
    int pollflags;  
    struct eventpoll *ep = file->private_data;  
    // 插入到wait_queue  
    poll_wait(file, &ep->poll_wait, wait);  
    // 扫描就绪的文件列表, 调用每个文件上的poll 检测是否真的就绪,  
    // 然后复制到用户空间  
    // 文件列表中有可能有epoll文件, 调用poll的时候有可能会产生递归,  
    // 调用所以用ep_call_nested 包装一下, 防止死循环和过深的调用  
    pollflags = ep_call_nested(&poll_readywalk_ncalls, EP_MAX_NESTS,  
                               ep_poll_readyevents_proc, ep, ep, current);  
    // static struct nested_calls poll_readywalk_ncalls;  
    return pollflags != -1 ? pollflags : 0;  
}  
  
static int ep_poll_readyevents_proc(void *priv, void *cookie, int call_nests)  
{  
    return ep_scan_ready_list(priv, ep_read_events_proc, NULL, call_nests + 1);  
}  
  
static int ep_scan_ready_list(struct eventpoll *ep,  
                              int (*sproc)(struct eventpoll *,  
                                      struct list_head *, void *),  
                              void *priv,  
                              int depth)  
{  
    int error, pwake = 0;  
    unsigned long flags;  
    struct epitem *epi, *nepi;  
    LIST_HEAD(txlist);  
  
    /* 
     * We need to lock this because we could be hit by 
     * eventpoll_release_file() and epoll_ctl(). 
     */  
    mutex_lock_nested(&ep->mtx, depth);  
  
    spin_lock_irqsave(&ep->lock, flags);  
    // 移动rdllist 到新的链表txlist  
    list_splice_init(&ep->rdllist, &txlist);  
    // 改变ovflist 的状态, 如果ep->ovflist != EP_UNACTIVE_PTR,  
    // 当文件激活wait_queue时，就会将对应的epitem加入到ep->ovflist  
    // 否则将文件直接加入到ep->rdllist，  
    // 这样做的目的是避免丢失事件  
    // 这里不需要检查ep->ovflist 的状态，因为ep->mtx的存在保证此处的ep->ovflist  
    // 一定是EP_UNACTIVE_PTR  
    ep->ovflist = NULL;  
    spin_unlock_irqrestore(&ep->lock, flags);  
  
    // 调用扫描函数处理txlist  
    error = (*sproc)(ep, &txlist, priv);  
  
    spin_lock_irqsave(&ep->lock, flags);  
  
    // 调用 sproc 时可能有新的事件，遍历这些新的事件将其插入到ready list  
    for (nepi = ep->ovflist; (epi = nepi) != NULL;  
            nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {  
        // #define EP_UNACTIVE_PTR (void *) -1  
        // epi 不在rdllist, 插入  
        if (!ep_is_linked(&epi->rdllink)) {  
            list_add_tail(&epi->rdllink, &ep->rdllist);  
        }  
    }  
    // 还原ep->ovflist的状态  
    ep->ovflist = EP_UNACTIVE_PTR;  
  
    // 将处理后的 txlist 链接到 rdllist  
    list_splice(&txlist, &ep->rdllist);  
  
    if (!list_empty(&ep->rdllist)) {  
        // 唤醒epoll_wait  
        if (waitqueue_active(&ep->wq)) {  
            wake_up_locked(&ep->wq);  
        }  
        // 当前的ep有其他的事件通知机制监控  
        if (waitqueue_active(&ep->poll_wait)) {  
            pwake++;  
        }  
    }  
    spin_unlock_irqrestore(&ep->lock, flags);  
  
    mutex_unlock(&ep->mtx);  
  
    if (pwake) {  
        // 安全唤醒外部的事件通知机制  
        ep_poll_safewake(&ep->poll_wait);  
    }  
  
    return error;  
}  
  
static int ep_read_events_proc(struct eventpoll *ep, struct list_head *head,  
                               void *priv)  
{  
    struct epitem *epi, *tmp;  
    poll_table pt;  
    init_poll_funcptr(&pt, NULL);  
    list_for_each_entry_safe(epi, tmp, head, rdllink) {  
        pt._key = epi->event.events;  
        if (epi->ffd.file->f_op->poll(epi->ffd.file, &pt) &  
                epi->event.events) {  
            return POLLIN | POLLRDNORM;  
        } else {  
             // 这个事件虽然在就绪列表中,  
             // 但是实际上并没有就绪, 将他移除  
         // 这有可能是水平触发模式中没有将文件从就绪列表中移除  
         // 也可能是事件插入到就绪列表后有其他的线程对文件进行了操作  
            list_del_init(&epi->rdllink);  
        }  
    }  
    return 0;  
}