hello_world.c main方法源码如下
int main(int argc, char **argv)
{
int rc;
struct spdk_env_opts opts;
/*
* SPDK relies on an abstraction around the local environment
* named env that handles memory allocation and PCI device operations.
* This library must be initialized first.
*
*/
spdk_env_opts_init(&opts);
rc = parse_args(argc, argv, &opts);
if (rc != 0) {
return rc;
}
opts.name = "hello_world";
if (spdk_env_init(&opts) < 0) {
fprintf(stderr, "Unable to initialize SPDK env\n");
return 1;
}
printf("Initializing NVMe Controllers\n");
if (g_vmd && spdk_vmd_init()) {
fprintf(stderr, "Failed to initialize VMD."
" Some NVMe devices can be unavailable.\n");
}
/*
* Start the SPDK NVMe enumeration process. probe_cb will be called
* for each NVMe controller found, giving our application a choice on
* whether to attach to each controller. attach_cb will then be
* called for each controller after the SPDK NVMe driver has completed
* initializing the controller we chose to attach.
*/
rc = spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL);
if (rc != 0) {
fprintf(stderr, "spdk_nvme_probe() failed\n");
rc = 1;
goto exit;
}
if (TAILQ_EMPTY(&g_controllers)) {
fprintf(stderr, "no NVMe controllers found\n");
rc = 1;
goto exit;
}
printf("Initialization complete.\n");
hello_world();
cleanup();
if (g_vmd) {
spdk_vmd_fini();
}
exit:
cleanup();
spdk_env_fini();
return rc;
}
main()的处理流程为:
- 源码317行,上文12行 spdk_env_opts_init(&opts);
- 源码320行,上文19行 spdk_env_init(&opts);
- 源码331行,上文38行 rc = spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL);
- 源码345行,上文52行 hello_world();
- 源码346行,上文59行 cleanup();
步骤1-步骤2是spdk运行环境初始化
步骤3是调用函数spdk_nvme_probe()主动发现NVMe SSDs设备
步骤4是调用hello_world(),做简单的读写操作
步骤5调用cleanup()释放内存资源,detach NVMe SSD设备等
接下来分析下关键函数spdk_nvme_probe()
分析之前先搞清楚下面两个问题:
1、每一块NVMe SSD里都有一个控制器(Controller),那么发现的所有NVMe SSD(也就是NVMe Controllers)以什么方式组织在一起?
2、每一块NVMe SSD都可以划分为多个namespace(类似逻辑分区的概念),那么这些namespace以什么方式组织在一起呢?
答案:
链表结构,看下hello_world.c代码
struct ctrlr_entry {
struct spdk_nvme_ctrlr *ctrlr;
TAILQ_ENTRY(ctrlr_entry) link;
char name[1024];
};
struct ns_entry {
struct spdk_nvme_ctrlr *ctrlr;
struct spdk_nvme_ns *ns;
TAILQ_ENTRY(ns_entry) link;
struct spdk_nvme_qpair *qpair;
};
static TAILQ_HEAD(, ctrlr_entry) g_controllers = TAILQ_HEAD_INITIALIZER(g_controllers);
static TAILQ_HEAD(, ns_entry) g_namespaces = TAILQ_HEAD_INITIALIZER(g_namespaces);
上面的
g_controllers是管理所有的nvme ssd的全局链表头
g_namespaces是管理所有的namespace的全局链表头
所以main函数里面的
if (TAILQ_EMPTY(&g_controllers)) {
fprintf(stderr, "no NVMe controllers found\n");
rc = 1;
goto exit;
}
就是因为g_controllers为空,就是因为没有找到nvme sdd
接下来看下main函数里面是如何使用spdk_nvme_probe()的
rc = spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL);
如上 probe_cb 和 attach_cb是两个回调函数(其实还有个remove_cb,只是上面未使用,传了个NULL)
probe_cb:当发现一个nvme设备的时候被调用
attach_cb:当一个nvme设备已经被attach到一个用户态的nvme驱动的时候被调用
probe_cb, attach_cb以及remove_cb的相关源码信息在spdk/include/spdk/nvme.h
/**
* Enumerate the bus indicated by the transport ID and attach the userspace NVMe
* driver to each device found if desired.
*
* This function is not thread safe and should only be called from one thread at
* a time while no other threads are actively using any NVMe devices.
*
* If called from a secondary process, only devices that have been attached to
* the userspace driver in the primary process will be probed.
*
* If called more than once, only devices that are not already attached to the
* SPDK NVMe driver will be reported.
*
* To stop using the the controller and release its associated resources,
* call spdk_nvme_detach() with the spdk_nvme_ctrlr instance from the attach_cb()
* function.
*
* \param trid The transport ID indicating which bus to enumerate. If the trtype
* is PCIe or trid is NULL, this will scan the local PCIe bus. If the trtype is
* RDMA, the traddr and trsvcid must point at the location of an NVMe-oF discovery
* service.
* \param cb_ctx Opaque value which will be passed back in cb_ctx parameter of
* the callbacks.
* \param probe_cb will be called once per NVMe device found in the system.
* \param attach_cb will be called for devices for which probe_cb returned true
* once that NVMe controller has been attached to the userspace driver.
* \param remove_cb will be called for devices that were attached in a previous
* spdk_nvme_probe() call but are no longer attached to the system. Optional;
* specify NULL if removal notices are not desired.
*
* \return 0 on success, -1 on failure.
*/
int spdk_nvme_probe(const struct spdk_nvme_transport_id *trid,
void *cb_ctx,
spdk_nvme_probe_cb probe_cb,
spdk_nvme_attach_cb attach_cb,
spdk_nvme_remove_cb remove_cb);
/**
* Callback for spdk_nvme_probe() enumeration.
*
* \param cb_ctx Opaque value passed to spdk_nvme_probe().
* \param trid NVMe transport identifier.
* \param opts NVMe controller initialization options. This structure will be
* populated with the default values on entry, and the user callback may update
* any options to request a different value. The controller may not support all
* requested parameters, so the final values will be provided during the attach
* callback.
*
* \return true to attach to this device.
*/
typedef bool (*spdk_nvme_probe_cb)(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
struct spdk_nvme_ctrlr_opts *opts);
/**
* Callback for spdk_nvme_attach() to report a device that has been attached to
* the userspace NVMe driver.
*
* \param cb_ctx Opaque value passed to spdk_nvme_attach_cb().
* \param trid NVMe transport identifier.
* \param ctrlr Opaque handle to NVMe controller.
* \param opts NVMe controller initialization options that were actually used.
* Options may differ from the requested options from the attach call depending
* on what the controller supports.
*/
typedef void (*spdk_nvme_attach_cb)(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
struct spdk_nvme_ctrlr *ctrlr,
const struct spdk_nvme_ctrlr_opts *opts);
/**
* Callback for spdk_nvme_remove() to report that a device attached to the userspace
* NVMe driver has been removed from the system.
*
* The controller will remain in a failed state (any new I/O submitted will fail).
*
* The controller must be detached from the userspace driver by calling spdk_nvme_detach()
* once the controller is no longer in use. It is up to the library user to ensure
* that no other threads are using the controller before calling spdk_nvme_detach().
*
* \param cb_ctx Opaque value passed to spdk_nvme_remove_cb().
* \param ctrlr NVMe controller instance that was removed.
*/
typedef void (*spdk_nvme_remove_cb)(void *cb_ctx, struct spdk_nvme_ctrlr *ctrlr);
接下来看下struct spdk_nvme_transport_id
/**
* NVMe transport identifier.
*
* This identifies a unique endpoint on an NVMe fabric.
*
* A string representation of a transport ID may be converted to this type using
* spdk_nvme_transport_id_p