kdump+crash 解决休眠锁(mutex)死锁问题

原创

已于 2023-10-10 15:42:19 修改 · 1k 阅读

5 ·

CC 4.0 BY-SA版权

文章标签：

#linux

于 2023-10-10 15:22:19 首次发布

本文围绕使用crash解决Linux系统中休眠锁（mutex）死锁问题展开。介绍了实验目的、步骤，简述了hung task机制，包括其核心思想、触发原因和proc接口。重点分析了mutex死锁问题，涵盖确认死锁线程、查找对应休眠锁、解析锁内容及查看阻塞线程栈回溯等内容。

1、实验目的：使用crash 解决休眠锁(mutex)死锁问题

4.1.1、在crash中通过 ps | grep UN 命令确定D状态线程，bt pid 查看D状态进程信息

4.1.2、hung task 机制确定死锁线程

4.2、根据死锁线程找到对应休眠锁

4.3、分析mutex锁内容，解析出获取锁以及等待锁的线程

4.3.1、struct mutex 关键成员分析

4.3.2、解析mutex的2个关键成员

4.4、使用bt命令查看阻塞线程的栈回溯

环境：arm64，Linux version 5.10.66

1、实验目的：使用crash 解决休眠锁(mutex)死锁问题

实验程序如下，当程序编译成ko并使用insmod加载到设备后，串口输入 echo kdump-3 > /proc/dbug/dump 命令之后执行我们的测试程序，测试程序是典型的AB-BA死锁，等待2分钟后 hung task 检测到有线程2min没有被调度，调用panic触发kdump。kdump产生vmcore文件后使用crash命令来分析下休眠锁(mutex)死锁问题。

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <asm/uaccess.h>
#include <asm/irq.h>
#include <asm/io.h>

#include <linux/device.h>
#include <linux/proc_fs.h>
#include <linux/version.h>

#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/printk.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
/* 
写成功要返回写入的字节数，否则 linux 还会尝试写入
 */

#define PROC_DIR_NAME		"dbug"			//文件夹名
#define PROC_FILE_NAME		"kdump"			//文件名称
#define KBUFSIZE 40
#define NAMELEN 20

typedef struct kdump_mutex {
	char cName[NAMELEN];
	int iDelay;
	int iTime;
	struct mutex lock;
	struct task_struct *thread;
	void (*printk)(void *p);
} KdumpMutex_st;

static KdumpMutex_st *gpstKdMutex1 = NULL;
static KdumpMutex_st *gpstKdMutex2 = NULL;

void kdump_printk(KdumpMutex_st *pstKdMutex)
{
	if(pstKdMutex != NULL)
		printk("Mutex lock name %s time = %d \n", pstKdMutex->cName, pstKdMutex->iTime);
	return;
}

static int mutex_thread1(void* arg)
{
	
	KdumpMutex_st *pstKdMutex = NULL;
	pstKdMutex = (KdumpMutex_st *)arg;
	
	/* 获取互斥锁1 */
	mutex_lock(&pstKdMutex->lock);
	pstKdMutex->iTime++;
	pstKdMutex->printk(pstKdMutex);
	msleep(pstKdMutex->iDelay);

	/* 获取互斥锁2 */
	mutex_lock(&gpstKdMutex2->lock);

	mutex_unlock(&pstKdMutex->lock);
	mutex_unlock(&gpstKdMutex2->lock);
	return 0;
}

static int mutex_thread2(void* arg)
{
	
	KdumpMutex_st *pstKdMutex = NULL;
	pstKdMutex = (KdumpMutex_st *)arg;
	
	/* 获取互斥锁2 */
	mutex_lock(&pstKdMutex->lock);
	pstKdMutex->iTime++;
	pstKdMutex->printk(pstKdMutex);
	msleep(pstKdMutex->iDelay);

	/* 获取互斥锁1 */
	mutex_lock(&gpstKdMutex1->lock);

	mutex_unlock(&pstKdMutex->lock);
	mutex_unlock(&gpstKdMutex1->lock);
	return 0;
}

int proc_kdump_mutex(int delay)
{
	/* 填充线程1的数据并运行线程 */
	gpstKdMutex1 = kzalloc(sizeof(KdumpMutex_st), GFP_KERNEL);
	strncpy(gpstKdMutex1->cName, "mutex thread_1", NAMELEN);
	mutex_init(&gpstKdMutex1->lock);
	gpstKdMutex1->printk = (void (*)(void *p))kdump_printk;
	gpstKdMutex1->iDelay = delay;
	gpstKdMutex1->thread = kthread_run(mutex_thread1, gpstKdMutex1, gpstKdMutex1->cName);
	
	/* 填充线程2的数据并运行线程 */
	gpstKdMutex2 = kzalloc(sizeof(KdumpMutex_st), GFP_KERNEL);
	strncpy(gpstKdMutex2->cName, "mutex thread_2", NAMELEN);
	mutex_init(&gpstKdMutex2->lock);
	gpstKdMutex2->printk = (void (*)(void *p))kdump_printk;
	gpstKdMutex2->iDelay = delay;
	gpstKdMutex2->thread = kthread_run(mutex_thread2, gpstKdMutex2, gpstKdMutex2->cName);
	
	return 0;
}


/* 创建/proc/dbug/kdump 的调试接口，此部分不必关注 */
char kbuf[KBUFSIZE] = {0};		//保存用户层传进来的数捿
struct proc_dir_entry *proc_wrbuff_dir;

static int proc_wrbuff_open(struct inode *inode,struct file *file);  
static ssize_t proc_wrbuff_read(struct file *file, char __user *ubuf, size_t count, loff_t *offset);  
static ssize_t proc_wrbuff_write(struct file *file, const char __user *ubuf, size_t count, loff_t *offset);  


static int proc_wrbuff_open(struct inode *inode,struct file *file)  {  
	printk("open embedsky board device!\n");  
	return 0;  
}

static ssize_t proc_wrbuff_read(struct file *file, char __user *ubuf, size_t count, loff_t *offset)  {  
	if (count > strlen(kbuf))
		count = strlen(kbuf);
	if (count < 0 )
		return -3;
	if (copy_to_user(ubuf, kbuf, count)) {
		printk(KERN_ERR "copy_to_user failed! \n");
		return -4;
	}
	return count;
}

static ssize_t proc_wrbuff_write(struct file *file, const char __user *ubuf, size_t count, loff_t *offset) {
	
	int num = 0;
	size_t cnt = min((int)count, KBUFSIZE - 1);
	if(copy_from_user(kbuf,ubuf,cnt)) {
		printk(KERN_ERR "copy_to_user failed! \n");
		return -EFAULT;
	}
	kbuf[cnt] = '\0';
	printk("printk kbuf %s \n",kbuf);

	if(sscanf(kbuf, "kdump-%d", &num)) {
		proc_kdump_mutex(num);
	}

	return cnt;
}

#if LINUX_VERSION_CODE > KERNEL_VERSION(5, 10, 0)
static struct proc_ops fops_proc_wrbuffer = {

	.proc_open = proc_wrbuff_open,
	.proc_read = proc_wrbuff_read,
	.proc_write = proc_wrbuff_write,
 };
#else
static struct file_operations fops_proc_wrbuffer = {
	.owner = THIS_MODULE,
	.open = proc_wrbuff_open,
	.read = proc_wrbuff_read,
	.write = proc_wrbuff_write,
	.owner = THIS_MODULE,
 };
#endif


static int __init proc_wrbuff_init(void) {
	int ret = 0;  
	struct proc_dir_entry *proc_file;

	/* 1 create parent dir in /porc/dbug */
	proc_wrbuff_dir = proc_mkdir(P