Linux内核-协议栈-从BSD Socket接口层到传输层1

本文接上一篇Linux内核协议栈-初始化流程分析，在上一篇中主要分析了了Linux内核协议栈涉及到的关键初始化函数，在这一篇文章中将分析协议栈的BSD socket和到传输层的流程。采取的方式是分析socket相关的主要系统调用。针对不同的系统调用，其到达的协议层深度可能不同，有的基本只到sock层就够了，但是有些可能需要会涉及到比如tcp的具体细节和更底层的细节。本文基本追溯到传输层的开始，再深入的细节后续文章分析。

---

1.准备
协议的基本分层：
(A代表socket的某个系统调用)
BSD socket system calls A => proto_ops->A => sock->A => tcp_prot => A

• BSD socket层和具体协议族某个类型的联系是通过struct proto_ops，在include/linux/net.h中定义了不同协议族如af_inet，af_unix等的通用操作函数指针的结构体struct proto_ops，具体的定义有各个协议族的某个类型的子模块自己完成。比如ipv4/af_inet.c中定义的af_inet family的tcp/udp等相应的struct proto_ops。
• 由于对于每个family的不同类型，其针对socket的某些需求可能不同，所以抽了一层struct sock出来，sock->sk_prot挂接到具体tcp/udp等传输层的struct proto上(具体定义在ipv4/tcp_ipv4.c,ipv4/udp.c)
• 另外，由于内容比较多，这一篇主要分析socket，bind，listen，accept几个系统调用，下一篇会涉及connect，send，recv等的分析

//不同协议族的通用函数hooks
//比如af_inet相关的定义在ipv4/af_inet.c中
//除了创建socket为系统调用外，基本针对socket层的操作函数都在这里面
struct proto_ops {
int     family;
    struct module   *owner;
int     (*release)   (struct socket *sock);
int     (*bind)      (struct socket *sock,
                      struct sockaddr *myaddr,
int sockaddr_len);
int     (*connect)   (struct socket *sock,
                      struct sockaddr *vaddr,
int sockaddr_len, int flags);
int     (*socketpair)(struct socket *sock1,
                      struct socket *sock2);
int     (*accept)    (struct socket *sock,
                      struct socket *newsock, int flags);
int     (*getname)   (struct socket *sock,
                      struct sockaddr *addr,
int *sockaddr_len, int peer);
    unsigned int    (*poll)      (struct file *file, struct socket *sock,
                      struct poll_table_struct *wait);
int     (*ioctl)     (struct socket *sock, unsigned int cmd,
                      unsigned long arg);
#ifdef CONFIG_COMPAT
int     (*compat_ioctl) (struct socket *sock, unsigned int cmd,
                      unsigned long arg);
#endif
int     (*listen)    (struct socket *sock, int len);
int     (*shutdown)  (struct socket *sock, int flags);
int     (*setsockopt)(struct socket *sock, int level,
int optname, char __user *optval, unsigned int optlen);
/*省略部分*/
};

//传输层的proto 
//作为sock->sk_prot与具体传输层的hooks
struct proto {
    void            (*close)(struct sock *sk,
                    long timeout);
int         (*connect)(struct sock *sk,
                    struct sockaddr *uaddr,
int addr_len);
int         (*disconnect)(struct sock *sk, int flags);

    struct sock *       (*accept)(struct sock *sk, int flags, int *err);

int         (*ioctl)(struct sock *sk, int cmd,
                     unsigned long arg);
int         (*init)(struct sock *sk);
    void            (*destroy)(struct sock *sk);
    void            (*shutdown)(struct sock *sk, int how);
int         (*setsockopt)(struct sock *sk, int level,
int optname, char __user *optval,
                    unsigned int optlen);
int         (*getsockopt)(struct sock *sk, int level,
int optname, char __user *optval,
int __user *option);
#ifdef CONFIG_COMPAT
int         (*compat_setsockopt)(struct sock *sk,
int level,
int optname, char __user *optval,
                    unsigned int optlen);
int         (*compat_getsockopt)(struct sock *sk,
int level,
int optname, char __user *optval,
int __user *option);
int         (*compat_ioctl)(struct sock *sk,
                    unsigned int cmd, unsigned long arg);
#endif
int         (*sendmsg)(struct kiocb *iocb, struct sock *sk,
                       struct msghdr *msg, size_t len);
int         (*recvmsg)(struct kiocb *iocb, struct sock *sk,
                       struct msghdr *msg,
                       size_t len, int noblock, int flags,
int *addr_len);
int         (*sendpage)(struct sock *sk, struct page *page,
int offset, size_t size, int flags);
int         (*bind)(struct sock *sk,
                    struct sockaddr *uaddr, int addr_len);

/*省略部分*/
};

同时附上其他几个关键结构体：

//bsd socket层
//include/linux/net.h
struct socket {
    socket_state        state;
    kmemcheck_bitfield_begin(type);
    short           type;
    kmemcheck_bitfield_end(type);
    unsigned long       flags;
struct socket_wq __rcu  *wq;
struct file     *file;
struct sock     *sk;
    const struct proto_ops  *ops;
};

//sock层
struct sock {
 sock_common    __sk_common;
#define sk_node __sk_common.skc_node
#define sk_nulls_node __sk_common.skc_nulls_node
#define sk_refcnt __sk_common.skc_refcnt
#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
#define sk_dontcopy_end __sk_common.skc_dontcopy_end
#define sk_hash __sk_common.skc_hash
#define sk_portpair __sk_common.skc_portpair
#define sk_num __sk_common.skc_num
#define sk_dport __sk_common.skc_dport
#define sk_addrpair __sk_common.skc_addrpair
#define sk_daddr __sk_common.skc_daddr
#define sk_rcv_saddr __sk_common.skc_rcv_saddr
#define sk_family __sk_common.skc_family
#define sk_state __sk_common.skc_state
#define sk_reuse __sk_common.skc_reuse
#define sk_reuseport __sk_common.skc_reuseport
#define sk_ipv6only __sk_common.skc_ipv6only
#define sk_bound_dev_if __sk_common.skc_bound_dev_if
#define sk_bind_node __sk_common.skc_bind_node
#define sk_prot __sk_common.skc_prot
#define sk_net __sk_common.skc_net
#define sk_v6_daddr __sk_common.skc_v6_daddr
#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr

    unsigned long       sk_flags;
struct dst_entry    *sk_rx_dst;
struct dst_entry __rcu  *sk_dst_cache;
    spinlock_t      sk_dst_lock;
    atomic_t        sk_wmem_alloc;
    atomic_t        sk_omem_alloc;
int         sk_sndbuf;
struct sk_buff_head sk_write_queue;
/*省略部分*/
struct pid      *sk_peer_pid;
const struct cred   *sk_peer_cred;
long            sk_rcvtimeo;
long            sk_sndtimeo;
void            *sk_protinfo;
struct timer_list   sk_timer;
    ktime_t         sk_stamp;
    u16         sk_tsflags;
    u32         sk_tskey;
struct socket       *sk_socket;
void            *sk_user_data;
struct page_frag    sk_frag;
struct sk_buff      *sk_send_head;
/*省略部分*/
};

---

2.开始
主要追溯几个典型的socket相关的系统调用，如socket,bind,listen,accept等等

• socket

//创建socket的系统调用
SYSCALL_DEFINE3(socket, int, family, int, type, int, protocol)
{
int retval;
struct socket *sock;
int flags;

/* Check the SOCK_* constants for consistency. */
    BUILD_BUG_ON(SOCK_CLOEXEC != O_CLOEXEC);
    BUILD_BUG_ON((SOCK_MAX | SOCK_TYPE_MASK) != SOCK_TYPE_MASK);
    BUILD_BUG_ON(SOCK_CLOEXEC & SOCK_TYPE_MASK);
    BUILD_BUG_ON(SOCK_NONBLOCK & SOCK_TYPE_MASK);

    flags = type & ~SOCK_TYPE_MASK;
if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
return -EINVAL;
type &= SOCK_TYPE_MASK;

if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK))
        flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK;

//分配inode，返回inode中的一个成员作为sock
    retval = sock_create(family, type, protocol, &sock);
if (retval < 0)
goto out;

//找个fd映射sock
//得到空fd
//分配伪dentry和file，并将socket file的operations与file挂接 
    retval = sock_map_fd(sock, flags & (O_CLOEXEC | O_NONBLOCK));
/*省略部分*/
}

• socketpair

//创建socketpair，注意af_inet协议族下没有pair，af_unix下有
SYSCALL_DEFINE4(socketpair, int, family, int, type, int, protocol,
int __user *, usockvec)
{
struct socket *sock1, *sock2;
int fd1, fd2, err;
struct file *newfile1, *newfile2;
int flags;

    flags = type & ~SOCK_TYPE_MASK;
if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
return -EINVAL;
type &= SOCK_TYPE_MASK;

if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK))
        flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK;

//创建socket1 
    err = sock_create(family, type, protocol, &sock1);
if (err < 0)
goto out;

//创建socket2
    err = sock_create(family, type, protocol, &sock2);
if (err < 0)
goto out_release_1;

//调用socket operations的socketpair 
//关于不同协议层的函数hook，公共结构体是struct proto_ops 
//对于不同的family，比如af_inet协议族的定义在ipv4/af_inet.c
//
//对于af_inet没有socketpair 
//对于af_unix有socketpair
    err = sock1->ops->socketpair(sock1, sock2);
if (err < 0)
goto out_release_both;

//后面部分就很类似了，找到空fd，分配file，绑定到socket，将file
    安装到当前进程
    fd1 = get_unused_fd_flags(flags);
if (unlikely(fd1 < 0)) {
        err = fd1;
goto out_release_both;
    }

    fd2 = get_unused_fd_flags(flags);
if (unlikely(fd2 < 0)) {
        err = fd2;
goto out_put_unused_1;
    }

    newfile1 = sock_alloc_file(sock1, flags, NULL);
if (unlikely(IS_ERR(newfile1))) {
        err = PTR_ERR(newfile1);
goto out_put_unused_both;
    }

    newfile2 = sock_alloc_file(sock2, flags, NULL);
if (IS_ERR(newfile2)) {
        err = PTR_ERR(newfile2);
goto out_fput_1;
    }

    err = put_user(fd1, &usockvec[0]);
if (err)
goto out_fput_both;

    err = put_user(fd2, &usockvec[1]);
if (err)
goto out_fput_both;

    audit_fd_pair(fd1, fd2);

    fd_install(fd1, newfile1);
    fd_install(fd2, newfile2);
/* fd1 and fd2 may be already another descriptors.
 * Not kernel problem.
 */
return 0;

• bind

SYSCALL_DEFINE3(bind, int, fd, struct sockaddr __user *, umyaddr, int, addrlen)
{
struct socket *sock;
struct sockaddr_storage address;
int err, fput_needed;
//根据fd查找file，进而查找socket指针sock
    sock = sockfd_lookup_light(fd, &err, &fput_needed);
if (sock) {
//把用户态地址数据移到内核态
//调用copy_from_user 
        err = move_addr_to_kernel(umyaddr, addrlen, &address);
if (err >= 0) {
//security hook
            err = security_socket_bind(sock,
                           (struct sockaddr *)&address,
                           addrlen);
if (!err)
//ok, 到具体family定义的proto_ops中的bind 
//比如对af_inet,主要是设置socket->sock->inet_sock的一些参数，比如接收地址，端口什么的
                err = sock->ops->bind(sock,
                              (struct sockaddr *)
                              &address, addrlen);
        }
        fput_light(sock->file, fput_needed);
    }
return err;
}

• listen
  listen所做的事情也比较简单，从系统调用的listen(fd, backlog)到proto_ops 的inet_listen与前面类似，这里分析下inet_listen中的核心函数inet_csk_listen_start(位于ipv4/inet_connection_sock.c中)。

int inet_csk_listen_start(struct sock *sk, const int nr_table_entries)
{
//获得网络层inte_sock 
    struct inet_sock *inet = inet_sk(sk);
//管理request connection的结构体 
    struct inet_connection_sock *icsk = inet_csk(sk);
//分配backlog个长度的accpet_queue的结构连接请求的队列
    int rc = reqsk_queue_alloc(&icsk->icsk_accept_queue, nr_table_entries);

if (rc != 0)
return rc;

    sk->sk_max_ack_backlog = 0;
    sk->sk_ack_backlog = 0;
    inet_csk_delack_init(sk);

/* There is race window here: we announce ourselves listening,
 * but this transition is still not validated by get_port().
 * It is OK, because this socket enters to hash table only
 * after validation is complete.
 */
//切换状态到listening 
    sk->sk_state = TCP_LISTEN;
if (!sk->sk_prot->get_port(sk, inet->inet_num)) {
        inet->inet_sport = htons(inet->inet_num);
//更新dst_entry表
        sk_dst_reset(sk);
        sk->sk_prot->hash(sk);

return 0;
    }
    sk->sk_state = TCP_CLOSE;
    __reqsk_queue_destroy(&icsk->icsk_accept_queue);
return -EADDRINUSE;
}

• accept
  上面socket, socketpair, bind基本只涉及到BSD socket, sock层相关的，过程比较简单，而accept层在sock层和tcp层交互稍微复杂，下面详细分析

//socket.c
//accept系统调用
SYSCALL_DEFINE4(accept4, int, fd, struct sockaddr __user *, upeer_sockaddr,
        int __user *, upeer_addrlen, int, flags)
{
/*省略部分*/
    err = -ENFILE;
//for client socket 
    newsock = sock_alloc();
if (!newsock)
        goto out_put;

    newsock->type = sock->type;
    newsock->ops = sock->ops;

/*
 * We don't need try_module_get here, as the listening socket (sock)
 * has the protocol module (sock->ops->owner) held.
 */
    __module_get(newsock->ops->owner);

//得到当前进程空fd，分给newsock file
    newfd = get_unused_fd_flags(flags);
if (unlikely(newfd < 0)) {
        err = newfd;
        sock_release(newsock);
        goto out_put;
    }
//从flab分配空file结构
    newfile = sock_alloc_file(newsock, flags, sock->sk->sk_prot_creator->name);
if (unlikely(IS_ERR(newfile))) {
        err = PTR_ERR(newfile);
        put_unused_fd(newfd);
        sock_release(newsock);
        goto out_put;
    }

    err = security_socket_accept(sock, newsock);
if (err)
        goto out_fd;

//proto_ops中的accept 
//accept从系统调用到具体协议族的某个type的struct proto_ops的accept如af_inet tcp的的accept，再到sock层的accept，然后sock层的accept实际上对应的是具体传输层的struct proto中的accpet，如tcp/udp的struct proto tcp_prot/udp_prot，然后放入newsock 
    err = sock->ops->accept(sock, newsock, sock->file->f_flags);
if (err < 0)
        goto out_fd;

if (upeer_sockaddr) {
if (newsock->ops->getname(newsock, (struct sockaddr *)&address,
&len, 2) < 0) {
            err = -ECONNABORTED;
            goto out_fd;
        }
//拷贝client socket addr storage到userspace
        err = move_addr_to_user(&address,
                    len, upeer_sockaddr, upeer_addrlen);
if (err < 0)
            goto out_fd;
    }
    fd_install(newfd, newfile);
    err = newfd;
/*省略部分*/

}

//ipv4/af_inet.c
//inet family的tcp相关的proto_ops
int inet_accept(struct socket *sock, struct socket *newsock, int flags)
{
    struct sock *sk1 = sock->sk;
int err = -EINVAL;
//进入(网络)sock层，accept新sock 
    struct sock *sk2 = sk1->sk_prot->accept(sk1, flags, &err);
if (!sk2)
goto do_err;

//锁住sock，因为需要操作sock内的request_socket请求队列头
    wait_queue_head_t等数据
    lock_sock(sk2);
    sock_rps_record_flow(sk2);
    WARN_ON(!((1 << sk2->sk_state) &
          (TCPF_ESTABLISHED | TCPF_SYN_RECV |
          TCPF_CLOSE_WAIT | TCPF_CLOSE)));
    sock_graft(sk2, newsock);
//设置client socket状态 
    newsock->state = SS_CONNECTED;
    err = 0;
    release_sock(sk2);
do_err:
return err;
}

//ipv4/tcp_ipv4.c
//这里进入struct proto tcp_prot中的accept
struct sock *inet_csk_accept(struct sock *sk, int flags, int *err)
{
    struct inet_connection_sock *icsk = inet_csk(sk);
//icsk : inet_connection_sock 面向连接的客户端连接处理相关的信息
//接收队列
    struct request_sock_queue *queue = &icsk->icsk_accept_queue;
    struct sock *newsk;
    struct request_sock *req;
    int error;
//lock sock
    lock_sock(sk);
//如果不是ACCPET状态转换过来，出错
    error = -EINVAL;
if (sk->sk_state != TCP_LISTEN)
        goto out_err;

//如果request_sock队列是空的， 利用等待队列挂起当前进程到等待队列，并且将等待队列放入sock中的请求队列头
if (reqsk_queue_empty(queue)) { 
//如果非阻塞，0，否则为sk的接收时间
        long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
        error = -EAGAIN;
if (!timeo)   //如果非阻塞而且接收队列是空，直接返回-EAGAIN
            goto out_err;
//阻塞情况下，等待timeo时间的超时
//利用了等待队列，下面会详细注解 
        error = inet_csk_wait_for_connect(sk, timeo);
if (error)
            goto out_err;
    }
//不是空，移出一个连接请求 
    req = reqsk_queue_remove(queue);
//连接请求的sock
    newsk = req->sk;
//减少backlog 
    sk_acceptq_removed(sk);
//fastopenq?
if (sk->sk_protocol == IPPROTO_TCP && queue->fastopenq != NULL) {
        spin_lock_bh(&queue->fastopenq->lock);
if (tcp_rsk(req)->listener) {
/* We are still waiting for the final ACK from 3WHS
 * so can't free req now. Instead, we set req->sk to
 * NULL to signify that the child socket is taken
 * so reqsk_fastopen_remove() will free the req
 * when 3WHS finishes (or is aborted).
 */
            req->sk = NULL;
            req = NULL;
        }
        spin_unlock_bh(&queue->fastopenq->lock);
    }
//ok,清理，返回newsk
/*省略部分*/

//ipv4/inet_connection_sock.c
//accept连接请求的核心函数
static int inet_csk_wait_for_connect(struct sock *sk, long timeo)
{
struct inet_connection_sock *icsk = inet_csk(sk);
//定义一个等待队列wait_queue_t wait 进程是当前进程
    DEFINE_WAIT(wait);
int err;
for (;;) {
//sk_leep(sk) : sock的wait_queue_head_t
//wait : wait_queue_t
//这里将current进程的wait_queue_t加入sk的wait_queue_head_t中，spin锁定 
//wait_queue_head_t，设置current状态，然后spin解锁时可能重新schedule 
        prepare_to_wait_exclusive(sk_sleep(sk), &wait,
                      TASK_INTERRUPTIBLE);

//被唤醒，解锁sock 
        release_sock(sk);
//如果请求队列为空,说明timeout了
if (reqsk_queue_empty(&icsk->icsk_accept_queue))
//schedule timeout
            timeo = schedule_timeout(timeo);

//再锁住进行下次循环，准备再次进入TASK_INTERRUPTIBLE
        lock_sock(sk);
        err = 0;

//检查是否有连接到达, 如果有，break,唤醒等待队列 
if (!reqsk_queue_empty(&icsk->icsk_accept_queue))
break;
        err = -EINVAL;
//如果不是listening 状态转过来的, 除错-EINVAL 
if (sk->sk_state != TCP_LISTEN)
break;

//检查interrupt错误
        err = sock_intr_errno(timeo);

//如果当前进程收到信号了，break 
if (signal_pending(current))
break;

//如果传入的timeo为0，则回到nonblock的状态, break 
        err = -EAGAIN;
if (!timeo)
break;
    }

//ok, 有连接到达，设置state为running, 唤醒wait queue的第一个进程，移除wait_queue_t和wait_queue_head_t 
    finish_wait(sk_sleep(sk), &wait);
return err;
}