spdk 入门 hello world源码解析

hello_world.c main方法源码如下

int main(int argc, char **argv)
{
   int rc;
   struct spdk_env_opts opts;

   /*
    * SPDK relies on an abstraction around the local environment
    * named env that handles memory allocation and PCI device operations.
    * This library must be initialized first.
    *
    */
   spdk_env_opts_init(&opts);
   rc = parse_args(argc, argv, &opts);
   if (rc != 0) {
      return rc;
   }

   opts.name = "hello_world";
   if (spdk_env_init(&opts) < 0) {
      fprintf(stderr, "Unable to initialize SPDK env\n");
      return 1;
   }

   printf("Initializing NVMe Controllers\n");

   if (g_vmd && spdk_vmd_init()) {
      fprintf(stderr, "Failed to initialize VMD."
         " Some NVMe devices can be unavailable.\n");
   }

   /*
    * Start the SPDK NVMe enumeration process.  probe_cb will be called
    *  for each NVMe controller found, giving our application a choice on
    *  whether to attach to each controller.  attach_cb will then be
    *  called for each controller after the SPDK NVMe driver has completed
    *  initializing the controller we chose to attach.
    */
   rc = spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL);
   if (rc != 0) {
      fprintf(stderr, "spdk_nvme_probe() failed\n");
      rc = 1;
      goto exit;
   }

   if (TAILQ_EMPTY(&g_controllers)) {
      fprintf(stderr, "no NVMe controllers found\n");
      rc = 1;
      goto exit;
   }

   printf("Initialization complete.\n");
   hello_world();
   cleanup();
   if (g_vmd) {
      spdk_vmd_fini();
   }

exit:
   cleanup();
   spdk_env_fini();
   return rc;
}

main()的处理流程为：

1. 源码317行,上文12行 spdk_env_opts_init(&opts);
2. 源码320行,上文19行 spdk_env_init(&opts);
3. 源码331行,上文38行 rc = spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL);
4. 源码345行,上文52行 hello_world();
5. 源码346行,上文59行 cleanup();

步骤1-步骤2是spdk运行环境初始化

步骤3是调用函数spdk_nvme_probe()主动发现NVMe SSDs设备

步骤4是调用hello_world()，做简单的读写操作

步骤5调用cleanup()释放内存资源，detach NVMe SSD设备等

接下来分析下关键函数spdk_nvme_probe()

分析之前先搞清楚下面两个问题：

1、每一块NVMe SSD里都有一个控制器(Controller),那么发现的所有NVMe SSD（也就是NVMe Controllers）以什么方式组织在一起？

2、每一块NVMe SSD都可以划分为多个namespace（类似逻辑分区的概念），那么这些namespace以什么方式组织在一起呢？

答案：

链表结构，看下hello_world.c代码

struct ctrlr_entry {
   struct spdk_nvme_ctrlr    *ctrlr;
   TAILQ_ENTRY(ctrlr_entry)   link;
   char            name[1024];
};

struct ns_entry {
   struct spdk_nvme_ctrlr *ctrlr;
   struct spdk_nvme_ns    *ns;
   TAILQ_ENTRY(ns_entry)  link;
   struct spdk_nvme_qpair *qpair;
};

static TAILQ_HEAD(, ctrlr_entry) g_controllers = TAILQ_HEAD_INITIALIZER(g_controllers);
static TAILQ_HEAD(, ns_entry) g_namespaces = TAILQ_HEAD_INITIALIZER(g_namespaces);

上面的

g_controllers是管理所有的nvme ssd的全局链表头

g_namespaces是管理所有的namespace的全局链表头

所以main函数里面的

if (TAILQ_EMPTY(&g_controllers)) {
   fprintf(stderr, "no NVMe controllers found\n");
   rc = 1;
   goto exit;
}

就是因为g_controllers为空，就是因为没有找到nvme sdd

接下来看下main函数里面是如何使用spdk_nvme_probe()的

rc = spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL);

如上 probe_cb 和 attach_cb是两个回调函数（其实还有个remove_cb，只是上面未使用，传了个NULL）

probe_cb：当发现一个nvme设备的时候被调用

attach_cb：当一个nvme设备已经被attach到一个用户态的nvme驱动的时候被调用

probe_cb, attach_cb以及remove_cb的相关源码信息在spdk/include/spdk/nvme.h

/**
 * Enumerate the bus indicated by the transport ID and attach the userspace NVMe
 * driver to each device found if desired.
 *
 * This function is not thread safe and should only be called from one thread at
 * a time while no other threads are actively using any NVMe devices.
 *
 * If called from a secondary process, only devices that have been attached to
 * the userspace driver in the primary process will be probed.
 *
 * If called more than once, only devices that are not already attached to the
 * SPDK NVMe driver will be reported.
 *
 * To stop using the the controller and release its associated resources,
 * call spdk_nvme_detach() with the spdk_nvme_ctrlr instance from the attach_cb()
 * function.
 *
 * \param trid The transport ID indicating which bus to enumerate. If the trtype
 * is PCIe or trid is NULL, this will scan the local PCIe bus. If the trtype is
 * RDMA, the traddr and trsvcid must point at the location of an NVMe-oF discovery
 * service.
 * \param cb_ctx Opaque value which will be passed back in cb_ctx parameter of
 * the callbacks.
 * \param probe_cb will be called once per NVMe device found in the system.
 * \param attach_cb will be called for devices for which probe_cb returned true
 * once that NVMe controller has been attached to the userspace driver.
 * \param remove_cb will be called for devices that were attached in a previous
 * spdk_nvme_probe() call but are no longer attached to the system. Optional;
 * specify NULL if removal notices are not desired.
 *
 * \return 0 on success, -1 on failure.
 */
int spdk_nvme_probe(const struct spdk_nvme_transport_id *trid,
          void *cb_ctx,
          spdk_nvme_probe_cb probe_cb,
          spdk_nvme_attach_cb attach_cb,
          spdk_nvme_remove_cb remove_cb);
          
          
/**
 * Callback for spdk_nvme_probe() enumeration.
 *
 * \param cb_ctx Opaque value passed to spdk_nvme_probe().
 * \param trid NVMe transport identifier.
 * \param opts NVMe controller initialization options. This structure will be
 * populated with the default values on entry, and the user callback may update
 * any options to request a different value. The controller may not support all
 * requested parameters, so the final values will be provided during the attach
 * callback.
 *
 * \return true to attach to this device.
 */
typedef bool (*spdk_nvme_probe_cb)(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
               struct spdk_nvme_ctrlr_opts *opts);

/**
 * Callback for spdk_nvme_attach() to report a device that has been attached to
 * the userspace NVMe driver.
 *
 * \param cb_ctx Opaque value passed to spdk_nvme_attach_cb().
 * \param trid NVMe transport identifier.
 * \param ctrlr Opaque handle to NVMe controller.
 * \param opts NVMe controller initialization options that were actually used.
 * Options may differ from the requested options from the attach call depending
 * on what the controller supports.
 */
typedef void (*spdk_nvme_attach_cb)(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
                struct spdk_nvme_ctrlr *ctrlr,
                const struct spdk_nvme_ctrlr_opts *opts);

/**
 * Callback for spdk_nvme_remove() to report that a device attached to the userspace
 * NVMe driver has been removed from the system.
 *
 * The controller will remain in a failed state (any new I/O submitted will fail).
 *
 * The controller must be detached from the userspace driver by calling spdk_nvme_detach()
 * once the controller is no longer in use. It is up to the library user to ensure
 * that no other threads are using the controller before calling spdk_nvme_detach().
 *
 * \param cb_ctx Opaque value passed to spdk_nvme_remove_cb().
 * \param ctrlr NVMe controller instance that was removed.
 */
typedef void (*spdk_nvme_remove_cb)(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);

 接下来看下struct spdk_nvme_transport_id

/**
 * NVMe transport identifier.
 *
 * This identifies a unique endpoint on an NVMe fabric.
 *
 * A string representation of a transport ID may be