龙芯平台fio异步无法测试问题

最新推荐文章于 2024-03-24 09:36:31 发布

天涯旅馆

最新推荐文章于 2024-03-24 09:36:31 发布

阅读量796

点赞数 2

分类专栏： fio工具文章标签： linux

原文链接：https://www.showdoc.com.cn/p/6abefa0380f4d5f192eb108190e34c86

版权

fio工具专栏收录该内容

2 篇文章

订阅专栏

概要

fio是一个非常灵活的io测试工具，他可以通过多线程或进程模拟各种io操作

随着块设备的发展，特别是SSD盘的出现，设备的并行度越来越高。要想利用好这些设备，有个诀窍就是提高设备的iodepth, 一次喂给设备更多的IO请求，让电梯算法和设备有机会来安排合并以及内部并行处理，提高总体效率。

应用程序使用IO通常有二种方式：同步和异步。同步的IO一次只能发出一个IO请求，等待内核完成才返回，这样对于单个线程iodepth总是小于1，但是可以通过多个线程并发执行来解决，通常我们会用16-32个线程同时工作把iodepth塞满。异步的话就是用类似libaio这样的linux native aio一次提交一批，然后等待一批的完成，减少交互的次数，会更有效率。

调试环境

验证系统
loongnix1.0版本
验证工具版本
fio-3.12
libaio-0.3.112

问题说明

在龙芯平台上，使用fio同步测试是没有问题的，但异步是无法跑起来的,报错是调用io_submit接口时传的参数不正确，需要说明的是，fio异步使用的是libaio库实现的，实际使用的是内核中的接口。

调试步骤

从fio源码中定位

通过报错位置找到了td_verror(td, -ret, "io commit");,说明td->io_ops->commit(td); 出问题了
fio/ioengines.c

void td_io_commit(struct thread_data *td)
{
int ret;
dprint(FD_IO, "calling ->commit(), depth %d\n", td->cur_depth);
if (!td->cur_depth || !td->io_u_queued)
return;
io_u_mark_depth(td, td->io_u_queued);
if (td->io_ops->commit) {
ret = td->io_ops->commit(td);
if (ret)
td_verror(td, -ret, "io commit");
}
/*
* Reflect that events were submitted as async IO requests.
*/
td->io_u_in_flight += td->io_u_queued;
td->io_u_queued = 0;
}

查找函数指针td->io_ops->commit(td);初始化的地方,通过打印判定是static int fio_libaio_commit(struct thread_data *td)

fio/engines/libaio.c

static int fio_libaio_commit(struct thread_data *td)
{
struct libaio_data *ld = td->io_ops_data;
struct iocb **iocbs;
struct io_u **io_us;
struct timespec ts;
int ret, wait_start = 0;
if (!ld->queued)
return 0;
do {
long nr = ld->queued;
nr = min((unsigned int) nr, ld->entries - ld->tail);
io_us = ld->io_us + ld->tail;
iocbs = ld->iocbs + ld->tail;
ret = io_submit(ld->aio_ctx, nr, iocbs);
if (ret > 0) {
fio_libaio_queued(td, io_us, ret);
io_u_mark_submit(td, ret);
ld->queued -= ret;
ring_inc(ld, &ld->tail, ret);
ret = 0;
wait_start = 0;
} else if (ret == -EINTR || !ret) {
if (!ret)
io_u_mark_submit(td, ret);
wait_start = 0;
continue;
} else if (ret == -EAGAIN) {
/*
* If we get EAGAIN, we should break out without
* error and let the upper layer reap some
* events for us. If we have no queued IO, we
* must loop here. If we loop for more than 30s,
* just error out, something must be buggy in the
* IO path.
*/
if (ld->queued) {
ret = 0;
break;
}
if (!wait_start) {
fio_gettime(&ts, NULL);
wait_start = 1;
} else if (mtime_since_now(&ts) > 30000) {
log_err("fio: aio appears to be stalled, giving up\n");
break;
}
usleep(1);
continue;
} else if (ret == -ENOMEM) {
/*
* If we get -ENOMEM, reap events if we can. If
* we cannot, treat it as a fatal event since there's
* nothing we can do about it.
*/
if (ld->queued)
ret = 0;
break;
} else
break;
} while (ld->queued);
return ret;
}

通过打印断定是io_submit出问题了，下面是io_submit的原型,查看发现就是宏定义接口，查看libaio源码中的接口

#define io_syscall3(type,fname,sname,type1,arg1,type2,arg2,type3,arg3) \
type fname(type1 arg1,type2 arg2,type3 arg3) \
_body_io_syscall(sname, (long)arg1, (long)arg2, (long)arg3)
io_syscall3(int, io_submit, io_submit, io_context_t, ctx, long, nr, struct iocb **, iocbs)

查看libaio接口发现libaio实现的是数据结构，时间的接口调用的是内核的
libaio-0.3.112/src/libaio.h

extern int io_setup(int maxevents, io_context_t *ctxp);
extern int io_destroy(io_context_t ctx);
extern int io_submit(io_context_t ctx, long nr, struct iocb *ios[]);
extern int io_cancel(io_context_t ctx, struct iocb *iocb, struct io_event *evt);
extern int io_getevents(io_context_t ctx_id, long min_nr, long nr, struct io_event *events, struct timespec *timeout);
extern int io_pgetevents(io_context_t ctx_id, long min_nr, long nr,

查看内核中aio.c如下，发现到内核的数据不正确了

long do_io_submit(aio_context_t ctx_id, long nr,
struct iocb __user *__user *iocbpp, bool compat)
{
struct kioctx *ctx;
long ret = 0;
int i = 0;
struct blk_plug plug;
printk("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!%d\n",__LINE__);
printk(KERN_DEBUG "line%d,addr:%p,%p\n",__LINE__,*iocbpp,iocbpp);
printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
if (unlikely(nr < 0))
return -EINVAL;
printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
if (unlikely(nr > LONG_MAX/sizeof(*iocbpp)))
nr = LONG_MAX/sizeof(*iocbpp);
printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
return -EFAULT;
printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
ctx = lookup_ioctx(ctx_id);
if (unlikely(!ctx)) {
pr_debug("EINVAL: invalid context id\n");
return -EINVAL;
}
printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
blk_start_plug(&plug);
printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
/*
* AKPM: should this return a partial result if some of the IOs were
* successfully submitted?
*/
for (i=0; i<nr; i++) {
struct iocb __user *user_iocb;
struct iocb tmp;
printk(KERN_DEBUG "line%d,addr:%p\n",__LINE__,user_iocb);
printk(KERN_DEBUG "line%d,addr:%p\n",__LINE__,&tmp);
if (unlikely(__get_user(user_iocb, iocbpp + i))) {
ret = -EFAULT;
break;
}
printk(KERN_DEBUG "line%d,addr:%p\n",__LINE__,user_iocb);
if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
ret = -EFAULT;
break;
}
printk(KERN_DEBUG "line%d,addr:%p-%d\n",__LINE__,&tmp,sizeof(tmp));
printk(KERN_DEBUG "line:%d,func:%s\n",__LINE__,__func__);
ret = io_submit_one(ctx, user_iocb, &tmp, compat);
if (ret)
break;
}
blk_finish_plug(&plug);
put_ioctx(ctx);
return i ? i : ret;
}

同时打印出内核和应用中的数据进行对比，发现了问题，内核中的数据长度是64byte，而应用中的数据长度是83byte，这样数据肯定是对不上的，所以查看原因，发现是libaio中的问题。

问题解决

libaio源码本身是部支持mips64el的，为了修改支持需要修改两个地方syscall.h和libaio.h，

syscall.h中添加mips64和syscall-mips64el.h mips64是gcc中定义的宏

#if defined(__i386__)
#include "syscall-i386.h"
#elif defined(__x86_64__)
#include "syscall-x86_64.h"
#elif defined(__mips64)
#include "syscall-mips64el.h"
#elif defined(__ia64__)
#include "syscall-ia64.h"
#elif defined(__PPC__)
#include "syscall-ppc.h"
#elif defined(__s390__)
#include "syscall-s390.h"
#elif defined(__alpha__)
#include "syscall-alpha.h"
#elif defined(__arm__)
#include "syscall-arm.h"
#elif defined(__sparc__)
#include "syscall-sparc.h"
#elif defined(__aarch64__) || defined(__riscv)
#include "syscall-generic.h"
#else
#warning "using system call numbers from sys/syscall.h"
#endif

syscall-mips64el.h内容如下，从linux/include/asm-arm/unistd.h /usr/include/asm/unistd.h中获取

#define __NR_Linux 5000
#define __NR_io_setup (__NR_Linux + 200)
#define __NR_io_destroy (__NR_Linux + 201)
#define __NR_io_getevents (__NR_Linux + 202)
#define __NR_io_submit (__NR_Linux + 203)
#define __NR_io_cancel (__NR_Linux + 204)

libaio.h 添加内容如下,在64 bits处添加__mips64

/* little endian, 32 bits */
#if defined(__i386__) || (defined(__arm__) && !defined(__ARMEB__)) || \
defined(__sh__) || defined(__bfin__) || (defined(__MIPSEL__) && defined(__mips32)) || \
defined(__cris__) || (defined(__riscv) && __riscv_xlen == 32) || \
(defined(__GNUC__) && defined(__BYTE_ORDER__) && \
__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_LONG__ == 4)
#define PADDED(x, y) x; unsigned y
#define PADDEDptr(x, y) x; unsigned y
#define PADDEDul(x, y) unsigned long x; unsigned y
/* little endian, 64 bits */
#elif defined(__ia64__) || defined(__x86_64__) || defined(__alpha__) || \
(defined(__MIPSEL__) && defined(__mips64)) || \
(defined(__aarch64__) && defined(__AARCH64EL__)) || \
(defined(__riscv) && __riscv_xlen == 64) || \
(defined(__GNUC__) && defined(__BYTE_ORDER__) && \
__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ && __SIZEOF_LONG__ == 8)
#define PADDED(x, y) x, y
#define PADDEDptr(x, y) x
#define PADDEDul(x, y) unsigned long x