如何编写Hello World iWarp应用程序?

I am trying to write a small program demonstrating Remote DMA over iWarp. I have the softiwarp Linux kernel module loaded and the userspace library compiled.

我正在写一个演示远程DMA在iWarp上的小程序。我已经加载了softiwarp Linux内核模块，并编译了用户空间库。

I am looking for documentation or sample code that explains setting up a connection and, e.g., sending a simple data block to the remote end ("Hello World!") to get me started, yet all I can find is the OpenFabrics' training web site, which is not helpful at all.

我正在寻找解释建立连接的文档或示例代码，例如，将一个简单的数据块发送到远程终端(“Hello World!”)来启动，但我所能找到的只是open面料的培训网站，这一点用处都没有。

2 个解决方案

#1

#include <arpa/inet.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include <getopt.h>
#include <stdlib.h>
#include <unistd.h>
#include <rdma/rdma_cma.h>

struct Args {
    char *remote_addr;
    int is_server;
    int port;
};

struct Args args;

int main(int argc, char *argv[])
{
    struct rdma_event_channel *evch;
    struct rdma_cm_id *server_id;
    struct rdma_cm_id *client_id;
    struct rdma_cm_event *event;
    struct rdma_conn_param conn_param;
    struct ibv_pd *pd;
    struct ibv_cq *cq;
    struct ibv_mr *mr;
    struct ibv_send_wr snd_wr;
    struct ibv_recv_wr rcv_wr;
    struct ibv_sge sge;
    struct ibv_wc wc;
    struct ibv_qp_init_attr attr = {
        .cap = {
            .max_send_wr = 32,
            .max_recv_wr = 32,
            .max_send_sge = 1,
            .max_recv_sge = 1,
            .max_inline_data = 64
        },
        .qp_type = IBV_QPT_RC
    };

    char msg[256] = "Hello World";
    int msg_len = strlen(msg) + 1;
    struct sockaddr_in sin;

    args.remote_addr = "0.0.0.0";
    args.is_server = 1;
    args.port = 21234;

    /* Parameter parsing. */
    while (1) {
        int c;

        c = getopt(argc, argv, "c:p:");
        if (c == -1)
            break;

        switch (c) {
        case 'c':
            args.is_server = 0;
            args.remote_addr = optarg;
            break;
        case 'p':
            args.port = strtol(optarg, NULL, 0);
            break;
        default:
            perror("Invalid option");
            exit(-1);
        };
    }

    if (args.is_server) {
        if (!(evch = rdma_create_event_channel())) {
            perror("rdma_create_event_channel");
            exit(-1);
        }

        if (rdma_create_id(evch, &server_id, NULL, RDMA_PS_TCP)) {
            perror("rdma_create_id");
            exit(-1);
        }

        sin.sin_family = AF_INET;
        sin.sin_port = htons(args.port);
        sin.sin_addr.s_addr = htonl(INADDR_ANY);

        if (rdma_bind_addr(server_id, (struct sockaddr *)&sin)) {
            perror("rdma_bind_addr");
            exit(-1);
        }

        if (rdma_listen(server_id, 6)) {
            perror("rdma_listen");
            exit(-1);
        }

        if (rdma_get_cm_event(evch, &event)
            || event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
            perror("rdma_get_cm_event");
            exit(-1);
        }

        client_id = (struct rdma_cm_id *)event->id;

        if (!(pd = ibv_alloc_pd(client_id->verbs))) {
            perror("ibv_alloc_pd");
            exit(-1);
        }

        if (!(mr = ibv_reg_mr(pd, msg, 256,
                IBV_ACCESS_REMOTE_WRITE |
                IBV_ACCESS_LOCAL_WRITE |
                IBV_ACCESS_REMOTE_READ))) {
            perror("ibv_reg_mr");
            exit(-1);
        }

        if (!(cq = ibv_create_cq(client_id->verbs, 32, 0, 0, 0))) {
            perror("ibv_create_cq");
            exit(-1);
        }

        attr.send_cq = attr.recv_cq = cq;
        if (rdma_create_qp(client_id, pd, &attr)) {
            perror("rdma_create_qp");
            exit(-1);
        }

        memset(&conn_param, 0, sizeof conn_param);
        if (rdma_accept(client_id, &conn_param)) {
            perror("rdma_accept");
            exit(-1);
        }

        rdma_ack_cm_event(event);
        if (rdma_get_cm_event(evch, &event)
            || event->event != RDMA_CM_EVENT_ESTABLISHED) {
            perror("rdma_get_cm_event");
            exit(-1);
        }
        rdma_ack_cm_event(event);

        sge.addr = (uint64_t)msg;
        sge.length = msg_len;
        sge.lkey = mr->lkey;
        snd_wr.sg_list = &sge;
        snd_wr.num_sge = 1;
        snd_wr.opcode = IBV_WR_SEND;
        snd_wr.send_flags = IBV_SEND_SIGNALED;
        snd_wr.next = NULL;
        if (ibv_post_send(client_id->qp, &snd_wr, NULL)) {
            perror("ibv_post_send");
            exit(-1);
        }

        while (!ibv_poll_cq(cq, 1, &wc))
            ;
        if (wc.status != IBV_WC_SUCCESS) {
            perror("ibv_poll_cq");
            exit(-1);
        }
    }

    else {
        if (!(evch = rdma_create_event_channel())) {
            perror("rdma_create_event_channel");
            exit(-1);
        }

        if (rdma_create_id(evch, &client_id, NULL, RDMA_PS_TCP)) {
            perror("rdma_create_id");
            exit(-1);
        }

        sin.sin_family = AF_INET;
        sin.sin_port = htons(args.port);
        sin.sin_addr.s_addr = inet_addr(args.remote_addr);

        if (rdma_resolve_addr
            (client_id, NULL, (struct sockaddr *)&sin, 2000)) {
            perror("rdma_resolve_addr");
            exit(-1);
        }

        if (rdma_get_cm_event(evch, &event)
            || event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
            perror("rdma_get_cm_event");
            exit(-1);
        }
        rdma_ack_cm_event(event);

        if (rdma_resolve_route(client_id, 2000)) {
            perror("rdma_resolve_route");
            exit(-1);
        }

        if (rdma_get_cm_event(evch, &event)
            || event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
            perror("rdma_get_cm_event");
            exit(-1);
        }
        rdma_ack_cm_event(event);

        if (!(pd = ibv_alloc_pd(client_id->verbs))) {
            perror("ibv_alloc_pd");
            exit(-1);
        }

        if (!(mr = ibv_reg_mr(pd, msg, 256,
                IBV_ACCESS_REMOTE_WRITE |
                IBV_ACCESS_LOCAL_WRITE |
                IBV_ACCESS_REMOTE_READ))) {
            perror("ibv_reg_mr");
            exit(-1);
        }

        if (!(cq = ibv_create_cq(client_id->verbs, 32, 0, 0, 0))) {
            perror("ibv_create_cq");
            exit(-1);
        }

        attr.send_cq = attr.recv_cq = cq;
        if (rdma_create_qp(client_id, pd, &attr)) {
            perror("rdma_create_qp");
            exit(-1);
        }

        sge.addr = (uint64_t)msg;
        sge.length = msg_len;
        sge.lkey = mr->lkey;
        rcv_wr.sg_list = &sge;
        rcv_wr.num_sge = 1;
        rcv_wr.next = NULL;
        if (ibv_post_recv(client_id->qp, &rcv_wr, NULL)) {
            perror("ibv_post_recv");
            exit(-1);
        }

        memset(&conn_param, 0, sizeof conn_param);
        if (rdma_connect(client_id, &conn_param)) {
            perror("rdma_connect");
            exit(-1);
        }

        if (rdma_get_cm_event(evch, &event)
            || event->event != RDMA_CM_EVENT_ESTABLISHED) {
            perror("rdma_get_cm_event");
            exit(-1);
        }
        rdma_ack_cm_event(event);

        while (!ibv_poll_cq(cq, 1, &wc)) ;
        if (wc.status != IBV_WC_SUCCESS) {
            perror("ibv_poll_cq");
            exit(-1);
        }

        fprintf(stdout, "Received %s \n", msg);
    }

    fprintf(stdout, "Done \n");
    return 0;
}

#2

Probably the best documentation and examples are in the librdmacm source. It includes full man pages around establishing connections (actually moving data is in the libibverbs man pages), as well as a number of example programs under the examples directory. The simplest example is rping.

最好的文档和示例可能在librdmacm源代码中。它包括关于建立连接的完整手册页(实际上移动数据在libibverb手册页中)，以及示例目录下的一些示例程序。最简单的例子是rping。

As the comment says, trying to code something up and then asking specific questions is more productive.

就像评论说的，试着把某件事编出来，然后问一些具体的问题会更有成效。