softlockup

最新推荐文章于 2024-10-11 15:59:44 发布

RJ436

最新推荐文章于 2024-10-11 15:59:44 发布

阅读量671

点赞数

文章标签： linux

本文链接：https://blog.youkuaiyun.com/RJ436/article/details/128840574

版权

文章详细介绍了Linux内核中如何检测和防止lockup现象，包括soft和hardlockup的区别。通过NMIWatchdog，系统能检测到CPU是否被长时间占用，利用高精度计时器和中断机制来判断和处理lockup情况。此外，文章还展示了kernel线程、时钟中断和NMI中断在lockup检测中的作用，以及相关的测试用例。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

lockup是某段内核代码占着CPU不放，lockup分为soft和hard，如果lockup时没有屏蔽中断为soft，若果屏蔽了中断为hard。

使用NMI Watchdog检测lockup

当发生hard lockup时，唯一能把CPU抢下来的办法就是NMI，不可屏蔽中断。NMI包含两部分：

1 高精度计时器，递增hrtimer_interrupts，唤醒[watchdog/x]内核线程，更新一个时间戳。

2 NMI中断时，检查hrtimer_interrupts是否递增，如果没有说明hard lockup。

soft/hard lockup 的实现在kernel/watchdog.c中，主题涉及3个部分：kernel线程，时钟中断，NMI中断。这三个东西具有不一样的优先级，依次是kernel线程<时钟中断<NMI中断。而正是用到了他们的优先级的区别，才可以调试系统运行中的两种问题：

1.抢占被长时间关闭而导致进程无法调度（soft lockup）

2.中断被长时间关闭而导致更严重的问题（hard lockup）


static struct smp_hotplug_thread watchdog_threads = {
  .store      = &softlockup_watchdog,
  .thread_should_run  = watchdog_should_run,
  .thread_fn    = watchdog,
  .thread_comm    = "watchdog/%u",
  .setup      = watchdog_enable,
  .park      = watchdog_disable,
  .unpark      = watchdog_enable,
};
 
void __init lockup_detector_init(void)
{
  set_sample_period();
  if (smpboot_register_percpu_thread(&watchdog_threads)) {
    pr_err("Failed to create watchdog threads, disabled\n");
    watchdog_disabled = -ENODEV;
  }
}

首先系统会为每个cpu core 注册一个一般的kernel线程，名字叫watchdog/0，watchdog/1…以此类推。这个线程会定期调用watchdog函数。


static void __touch_watchdog(void)
{
  __this_cpu_write(watchdog_touch_ts, get_timestamp());
}
 
static void watchdog(unsigned int cpu)
{
  __this_cpu_write(soft_lockup_hrtimer_cnt,
       __this_cpu_read(hrtimer_interrupts));
  __touch_watchdog();
}

watchdog_touch_ts是看门狗触发后当前的时间戳，以上代码用来更新时间戳。

static void watchdog_enable(unsigned int cpu)
{
    struct hrtimer *hrtimer = raw_cpu_ptr(&watchdog_hrtimer);

/* kick off the timer for the hardlockup detector */
/*启动硬死锁检测器的计时器*/
    hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
    hrtimer->function = watchdog_timer_fn;

    /* Enable the perf event */
    watchdog_nmi_enable(cpu);

    /* done here because hrtimer_start can only pin to smp_processor_id() */
    hrtimer_start(hrtimer, ns_to_ktime(sample_period),
              HRTIMER_MODE_REL_PINNED);

/* initialize timestamp */
/*初始化时间戳*/
    watchdog_set_prio(SCHED_FIFO, MAX_RT_PRIO - 1);
    __touch_watchdog();
}

时钟中断处理函数是watchdog_timer_fn

/* watchdog kicker functions */
static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
{
    unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
    struct pt_regs *regs = get_irq_regs();
    int duration;
    int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;

    /* kick the hardlockup detector */
    watchdog_interrupt_count();

    /* kick the softlockup detector */
    wake_up_process(__this_cpu_read(softlockup_watchdog));

    /* .. and repeat */
    hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
  ......
......
 if (softlockup_all_cpu_backtrace) {
            /* Prevent multiple soft-lockup reports if one cpu is already
             * engaged in dumping cpu back traces
             */
            if (test_and_set_bit(0, &soft_lockup_nmi_warn)) {
                /* Someone else will report us. Let's give up */
                __this_cpu_write(soft_watchdog_warn, true);
                return HRTIMER_RESTART;
            }
        }

        pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
            smp_processor_id(), duration,
            current->comm, task_pid_nr(current));
        __this_cpu_write(softlockup_task_ptr_saved, current);
        print_modules();
        print_irqtrace_events(current);
......
......

更新hrtimer_interrupts变量


static void watchdog_interrupt_count(void)
{
    __this_cpu_inc(hrtimer_interrupts);
}

这里我们就要回顾之前创建的kernel线程了，多久调用一次就和hrtimer_interrupts的值密切相关。


static int watchdog_should_run(unsigned int cpu)
{
    return __this_cpu_read(hrtimer_interrupts) !=
        __this_cpu_read(soft_lockup_hrtimer_cnt);
}

kernel线程的频率默认为10*2/5 = 4秒一次。


int __read_mostly watchdog_thresh = 10;

/*
 * Hard-lockup warnings should be triggered after just a few seconds. Soft-
 * lockups can have false positives under extreme conditions. So we generally
 * want a higher threshold for soft lockups than for hard lockups. So we couple
 * the thresholds with a factor: we make the soft threshold twice the amount of
 * time the hard threshold is.
 */
/*硬死锁的警告应该在几秒后触发，软死锁在极端情况下可能会误报*/


static int get_softlockup_thresh(void)
{
    return watchdog_thresh * 2;
}

softlockup的测试方法
配置选项
CONFIG_LOCKUP_DETCTOR=y

CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y

CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=1

测试用例


#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/spinlock.h>
#include <asm-generic/delay.h>
#include <linux/kthread.h>


static int soft_lockup = 1;
module_param(soft_lockup, int, 0644);
MODULE_PARM_DESC(soft_lockup, "soft_lockup");

static struct task_struct *taskid;
static spinlock_t slock;

static int kthread_lockup(void *data)
{
    spinlock_t *lock;
    lock = (spinlock_t *)data;
    while(!kthread_should_stop()) {
        udelay(19000);
        if (soft_lockup) {
            spin_lock(lock);
            spin_lock(lock);
        } else {
            spin_lock_irq(lock); //disable local irq
            spin_lock_irq(lock);
        }
    }
    return 0;
}

static int __init lockup_init(void)
{
    int err;
    spin_lock_init(&slock);
    taskid = kthread_run(kthread_lockup, &slock, "lockup_test");
    if (IS_ERR(taskid)) {
    err = PTR_ERR(taskid);
    return err;
  }
    return 0;
}

static void __init lockup_exit(void)
{
    kthread_stop(taskid);
}

module_init(lockup_init);
module_exit(lockup_exit);
MODULE_AUTHOR("Haocheng Xie");
MODULE_LICENSE("GPL v2");

测试用例2

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/spinlock.h>
#include <asm-generic/delay.h>
#include <linux/kthread.h>


static int soft_lockup = 1;
module_param(soft_lockup, int, 0644);
MODULE_PARM_DESC(soft_lockup, "soft_lockup");

static struct task_struct *taskid;
static spinlock_t slock;
#if 0
static int kthread_lockup(void *data)
{
    spinlock_t *lock;
    lock = (spinlock_t *)data;
    while(!kthread_should_stop()) {
        udelay(19000);
        if (soft_lockup) {
            spin_lock(lock);
            spin_lock(lock);
        } else {
            spin_lock_irq(lock); //disable local irq
            spin_lock_irq(lock);
        }
    }
    return 0;
}
#endif 

static int __init lockup_init(void)
{
    int err;
    spin_lock_init(&slock);
#if 0
   // taskid = kthread_run(kthread_lockup, &slock, "lockup_test");
    if (IS_ERR(taskid)) {
    err = PTR_ERR(taskid);
    return err;
  }
#endif 
    while (1) {
        udelay(10000);
        spin_lock(&slock);
        spin_lock(&slock);
    }
    return 0;
}

static void __init lockup_exit(void)
{
    kthread_stop(taskid);
}

module_init(lockup_init);
module_exit(lockup_exit);
MODULE_AUTHOR("Haocheng Xie");
MODULE_LICENSE("GPL v2");