DPDK学习（rte_eal_init）

lcore_config

struct lcore_config {
   unsigned detected;         /**< true if lcore was detected */
   pthread_t thread_id;       /**< pthread identifier */
   int pipe_master2slave[2];  /**< communication pipe with master */
   int pipe_slave2master[2];  /**< communication pipe with master */
   lcore_function_t * volatile f;         /**< function to call */
   void * volatile arg;       /**< argument of function */
   volatile int ret;          /**< return value of function */
   volatile enum rte_lcore_state_t state; /**< lcore state */
   unsigned socket_id;        /**< physical socket id for this lcore */
   unsigned core_id;          /**< core number on socket for this lcore */
   int core_index;            /**< relative index, starting from 0 */
   rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
   uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
};

extern struct lcore_config lcore_config[RTE_MAX_LCORE];

rte_eal_init()

rte_eal_init() {
    ...
    // 读取/sys/devices/system/cpu/cpuX和/sys/devices/system/node/nodeX中的信息，填充到lcore_config
    rte_eal_cpu_init()
    ...
    /* 解析-c、--master_lcore、--lcores等参数
       确认可用的logical CPU
       调用eal_adjust_config()设置cfg->master_lcore = 0 (设置第一个lcore为MASTER lcore) */
    eal_parse_args() {
        ...
        eal_parse_common_option() {
            ...
            eal_parse_coremask()
            ...
            eal_parse_master_lcore()
            ...
            eal_parse_lcores()
            ...
        }
        ...
        eal_adjust_config()
        ...
    }
    ...
    // 设置当前线程为MASTER lcore，调用eal_thread_set_affinity()绑定MASTER lcore到logical CPU
    eal_thread_init_master() {
        ...
        eal_thread_set_affinity()
        ...
    }
    ...
    /* 为每个SLAVE lcore创建线程，调用eal_thread_loop()->eal_thread_set_affinity()
       绑定SLAVE lcore到logical CPU */
    RTE_LCORE_FOREACH_SLAVE(i) {

        /*
         * create communication pipes between master thread
         * and children
         */
        // MASTER lcore创建pipes用于MASTER和SLAVE lcore间通信（父子线程间通信）
        if (pipe(lcore_config[i].pipe_master2slave) < 0)
            rte_panic("Cannot create pipe\n");
        if (pipe(lcore_config[i].pipe_slave2master) < 0)
            rte_panic("Cannot create pipe\n");

        lcore_config[i].state = WAIT;

        /* create a thread for each lcore */
        ret = pthread_create(&lcore_config[i].thread_id, NULL, eal_thread_loop, NULL);
        ...
    }

    /*
     * Launch a dummy function on all slave lcores, so that master lcore
     * knows they are all ready when this function returns.
     */
    rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
    rte_eal_mp_wait_lcore();
    ...
}

rte_eal_remote_launch()

调用rte_eal_remote_launch()将回调函数和参数注册到lcore_config[].f和lcore_config[].arg中，并通知SLAVE lcore执行该回调函数

/* * Send a message to a slave lcore identified by slave_id to call a * function f with argument arg. Once the execution is done, the * remote lcore switch in FINISHED state. */
int
rte_eal_remote_launch(int (*f)(void *), void *arg, unsigned slave_id)
{
   int n;
   char c = 0;
   int m2s = lcore_config[slave_id].pipe_master2slave[1];
   int s2m = lcore_config[slave_id].pipe_slave2master[0];

   if (lcore_config[slave_id].state != WAIT)
      return -EBUSY;

   // 注册回调函数和参数
   lcore_config[slave_id].f = f;
   lcore_config[slave_id].arg = arg;

   /* send message */
   // 发送信号给SLAVE lcore
   n = 0;
   while (n == 0 || (n < 0 && errno == EINTR))
      n = write(m2s, &c, 1);
   if (n < 0)
      rte_panic("cannot write on configuration pipe\n");

   /* wait ack */
   do {
      n = read(s2m, &c, 1);
   } while (n < 0 && errno == EINTR);

   if (n <= 0)
      rte_panic("cannot read on configuration pipe\n");

   return 0;
}

rte_eal_mp_remote_launch()

/*
 * Check that every SLAVE lcores are in WAIT state, then call
 * rte_eal_remote_launch() for all of them. If call_master is true
 * (set to CALL_MASTER), also call the function on the master lcore.
 */
int
rte_eal_mp_remote_launch(int (*f)(void *), void *arg,
          enum rte_rmt_call_master_t call_master)
{
   int lcore_id;
   int master = rte_get_master_lcore();

   /* check state of lcores */
   // 检查每个SLAVE lcore的状态是否为WAIT
   RTE_LCORE_FOREACH_SLAVE(lcore_id) {
      if (lcore_config[lcore_id].state != WAIT)
         return -EBUSY;
   }

   /* send messages to cores */
   RTE_LCORE_FOREACH_SLAVE(lcore_id) {
      rte_eal_remote_launch(f, arg, lcore_id);
   }

   // 在MASTER lcore上调用该回调函数
   if (call_master == CALL_MASTER) {
      lcore_config[master].ret = f(arg);
      lcore_config[master].state = FINISHED;
   }

   return 0;
}

rte_eal_wait_lcore()

/*
 * Wait until a lcore finished its job.
 */
int
rte_eal_wait_lcore(unsigned slave_id)
{
   if (lcore_config[slave_id].state == WAIT)
      return 0;

   // 等待各个SLAVE lcore执行完毕 
   while (lcore_config[slave_id].state != WAIT &&
          lcore_config[slave_id].state != FINISHED)
      rte_pause();

   rte_rmb();

   /* we are in finished state, go to wait state */
   // 修改SLAVE lcore的状态为WAIT
   lcore_config[slave_id].state = WAIT;
   return lcore_config[slave_id].ret;
}

rte_eal_mp_wait_lcore()

/*
 * Do a rte_eal_wait_lcore() for every lcore. The return values are
 * ignored.
 */
void
rte_eal_mp_wait_lcore(void)
{
   unsigned lcore_id;

   RTE_LCORE_FOREACH_SLAVE(lcore_id) {
      rte_eal_wait_lcore(lcore_id);
   }
}