Linux内核协议栈对于timewait状态的处理

最近在做操作系统升级时，发现升级后的系统处于TIME_WAIT状态的连接数明显增多（内核版本 2.6.18 -> 2.6.32）。

原因

2.6.18 与 2.6.32 的 diff 结果

net/ipv4/inet_timewait_sock.c

C

@@ -178,15 +212,14 @@ need_timer = 0; if (inet_twdr_do_twkill_work(twdr, twdr->slot)) { twdr->thread_slots |= (1 << twdr->slot); - mb(); schedule_work(&twdr->twkill_work); need_timer = 1; } else { /* We purged the entire slot, anything left? */ if (twdr->tw_count) need_timer = 1; //这句话位置的变动引起TIME_WAIT状态增多 + twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1)); } - twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1)); if (need_timer) mod_timer(&twdr->tw_timer, jiffies + twdr->period); out:

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19

@@ -178,15 +212,14 @@       need_timer = 0;       if (inet_twdr_do_twkill_work(twdr, twdr->slot)) {         twdr->thread_slots |= (1 << twdr->slot);     -   mb();         schedule_work(&twdr->twkill_work);         need_timer = 1;       } else {         /* We purged the entire slot, anything left?  */           if (twdr->tw_count)           need_timer = 1;         //这句话位置的变动引起TIME_WAIT状态增多     +   twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));       }       - twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));       if (need_timer)         mod_timer(&twdr->tw_timer, jiffies + twdr->period);     out:

导致TIME_WAIT状态增多，正是由于

twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));

位置的改变。

具体分析

1. 关键数据结构

inet_timewait_death_row: 用于管理timewait控制块的数据结构，位置： include/net/inet_timewait_sock.h。

C

struct inet_timewait_death_row { /* Short-time timewait calendar */ int twcal_hand; int twcal_jiffie; struct timer_list twcal_timer; struct hlist_head twcal_row[INET_TWDR_RECYCLE_SLOTS]; spinlock_t death_lock; int tw_count; int period; u32 thread_slots; struct work_struct twkill_work; struct timer_list tw_timer; int slot; //INET_TWDR_TWKILL_SLOTS 值为 8 struct hlist_head cells[INET_TWDR_TWKILL_SLOTS]; struct inet_hashinfo *hashinfo; int sysctl_tw_recycle; int sysctl_max_tw_buckets; };

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21

struct inet_timewait_death_row {     /* Short-time timewait calendar */     int         twcal_hand;     int         twcal_jiffie;     struct timer_list   twcal_timer;     struct hlist_head   twcal_row[INET_TWDR_RECYCLE_SLOTS];       spinlock_t      death_lock;     int         tw_count;     int         period;     u32         thread_slots;     struct work_struct  twkill_work;     struct timer_list   tw_timer;     int         slot;     //INET_TWDR_TWKILL_SLOTS 值为 8     struct hlist_head   cells[INET_TWDR_TWKILL_SLOTS];     struct inet_hashinfo    *hashinfo;     int         sysctl_tw_recycle;     int         sysctl_max_tw_buckets; };

此数据结构，可以分为两部分看，一部分处理 tw_recycle 开启时timewait块的快速回收，另一部分为未开启时用于等待时间较长的timewait块的回收。由于系统没有开启 tw_recycle ， 因此我们主要关注等待时间较长的timewait块回收。

用于等待时间较长的主要成员变量：

int period : tw_timer 定时器的超时时间固定值为 TCP_TIMEWAIT_LEN / INET_TWDR_TWKILL_SLOTS，其中 TCP_TIMEWAIT_LEN 为 60 * HZ (60s)，INET_TWDR_TWKILL_SLOTS 为 8。

u32 thread_slots : 用于标识未完成的timewait块的位图。

struct work_struct twkill_work : 分批删除（默认值为每次删除100个）cells中timewait块时的工作队列。

struct timer_list tw_timer : 定时器，每过 period，触发一次 inet_twdr_hangman()。

以下是此数据结构的初始化：

C

struct inet_timewait_death_row tcp_death_row = { .sysctl_max_tw_buckets = NR_FILE * 2, .period = TCP_TIMEWAIT_LEN / INET_TWDR_TWKILL_SLOTS, .death_lock = __SPIN_LOCK_UNLOCKED(tcp_death_row.death_lock), .hashinfo = &tcp_hashinfo, .tw_timer = TIMER_INITIALIZER(inet_twdr_hangman, 0, (unsigned long)&tcp_death_row), .twkill_work = __WORK_INITIALIZER(tcp_death_row.twkill_work, inet_twdr_twkill_work, &tcp_death_row), /* Short-time timewait calendar */ .twcal_hand = -1, .twcal_timer = TIMER_INITIALIZER(inet_twdr_twcal_tick, 0, (unsigned long)&tcp_death_row), };

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17

struct inet_timewait_death_row tcp_death_row = {     .sysctl_max_tw_buckets = NR_FILE * 2,     .period   = TCP_TIMEWAIT_LEN / INET_TWDR_TWKILL_SLOTS,     .death_lock = __SPIN_LOCK_UNLOCKED(tcp_death_row.death_lock),     .hashinfo = &tcp_hashinfo,     .tw_timer = TIMER_INITIALIZER(inet_twdr_hangman, 0,                 (unsigned long)&tcp_death_row),     .twkill_work = __WORK_INITIALIZER(tcp_death_row.twkill_work,                  inet_twdr_twkill_work,                  &tcp_death_row),     /* Short-time timewait calendar */       .twcal_hand = -1,     .twcal_timer = TIMER_INITIALIZER(inet_twdr_twcal_tick, 0,                 (unsigned long)&tcp_death_row), };

inet_timewait_sock: 用于组成 tcp_timewait_sock 结构，其前部是 sock_common 的前部。位置： include/net/inet_timewait_sock.h。

C

struct inet_timewait_sock { /* * Now struct sock also uses sock_common, so please just * don't add nothing before this first member (__tw_common) --acme */ struct sock_common __tw_common; #define tw_family __tw_common.skc_family #define tw_state __tw_common.skc_state #define tw_reuse __tw_common.skc_reuse #define tw_bound_dev_if __tw_common.skc_bound_dev_if #define tw_node __tw_common.skc_node #define tw_bind_node __tw_common.skc_bind_node #define tw_refcnt __tw_common.skc_refcnt #define tw_hash __tw_common.skc_hash #define tw_prot __tw_common.skc_prot volatile unsigned char tw_substate; /* 3 bits hole, try to pack */ unsigned char tw_rcv_wscale; /* Socket demultiplex comparisons on incoming packets. */ /* these five are in inet_sock */ __u16 tw_sport; __u32 tw_daddr __attribute__((aligned(INET_TIMEWAIT_ADDRCMP_ALIGN_BYTES))); __u32 tw_rcv_saddr; __u16 tw_dport; __u16 tw_num; /* And these are ours. */ __u8 tw_ipv6only:1; /* 15 bits hole, try to pack */ __u16 tw_ipv6_offset; int tw_timeout; unsigned long tw_ttd; struct inet_bind_bucket *tw_tb; struct hlist_node tw_death_node; };

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35

struct inet_timewait_sock {     /*        * Now struct sock also uses sock_common, so please just      * don't add nothing before this first member (__tw_common) --acme      */     struct sock_common  __tw_common; #define tw_family   __tw_common.skc_family #define tw_state    __tw_common.skc_state #define tw_reuse    __tw_common.skc_reuse #define tw_bound_dev_if   __tw_common.skc_bound_dev_if #define tw_node     __tw_common.skc_node #define tw_bind_node    __tw_common.skc_bind_node #define tw_refcnt   __tw_common.skc_refcnt #define tw_hash     __tw_common.skc_hash #define tw_prot     __tw_common.skc_prot     volatile unsigned char  tw_substate;     /* 3 bits hole, try to pack */     unsigned char   tw_rcv_wscale;     /* Socket demultiplex comparisons on incoming packets. */     /* these five are in inet_sock */     __u16     tw_sport;     __u32     tw_daddr __attribute__((aligned(INET_TIMEWAIT_ADDRCMP_ALIGN_BYTES)));     __u32     tw_rcv_saddr;     __u16     tw_dport;     __u16     tw_num;     /* And these are ours. */     __u8      tw_ipv6only:1;     /* 15 bits hole, try to pack */     __u16     tw_ipv6_offset;     int     tw_timeout;     unsigned long   tw_ttd;     struct inet_bind_bucket *tw_tb;     struct hlist_node tw_death_node; };

此数据结构暂时只需要知道 tw_substate 即可。 tw_substate : TCP状态迁移到 FIN_WAIT2 或 TIME_WAIT 状态时，协议栈会用 timewait 块取代 tcp_sock 块，因为这两种状态都需要由定时器处理，超时立即释放。其对外状态都表现为 TIME_WAIT ， 但其内部状态还是有分别，通过 tw_substate 进行区分。

---

2. timewait块释放时的逻辑

inet_twdr_hangman() 此函数是定时器到期时执行的函数，用于释放timewait块。

C

void inet_twdr_hangman(unsigned long data) { struct inet_timewait_death_row *twdr; int unsigned need_timer; twdr = (struct inet_timewait_death_row *)data; spin_lock(&twdr->death_lock); if (twdr->tw_count == 0) goto out; need_timer = 0; //inet_twdr_do_twkill_work 释放timewait块的具体函数，每次释放100个 //释放完成返回0, 否则返回1 if (inet_twdr_do_twkill_work(twdr, twdr->slot)) { //一次遍历未完全删除timewait块时，剩余的time块放入twkill_work的工作队列中处理。 //thread_slots标识未完成的timewait块 twdr->thread_slots |= (1 << twdr->slot); mb(); schedule_work(&twdr->twkill_work); //未删除所有timewait块，需要重新调度定时器 need_timer = 1; } else { /* We purged the entire slot, anything left? */ if (twdr->tw_count) need_timer = 1; } //此句是关键,代码出自2.6.18内核，不管定时器例程一次有没有释放完timewait块，都进行 + 1 操作 twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1)); if (need_timer) mod_timer(&twdr->tw_timer, jiffies + twdr->period); out: spin_unlock(&twdr->death_lock); }

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37

void inet_twdr_hangman(unsigned long data) {     struct inet_timewait_death_row *twdr;     int unsigned need_timer;       twdr = (struct inet_timewait_death_row *)data;     spin_lock(&twdr->death_lock);       if (twdr->tw_count == 0)         goto out;       need_timer = 0;       //inet_twdr_do_twkill_work 释放timewait块的具体函数，每次释放100个     //释放完成返回0, 否则返回1     if (inet_twdr_do_twkill_work(twdr, twdr->slot)) {         //一次遍历未完全删除timewait块时，剩余的time块放入twkill_work的工作队列中处理。         //thread_slots标识未完成的timewait块         twdr->thread_slots |= (1 << twdr->slot);         mb();         schedule_work(&twdr->twkill_work);         //未删除所有timewait块，需要重新调度定时器         need_timer = 1;     } else {       /* We purged the entire slot, anything left?  */       if (twdr->tw_count)           need_timer = 1;     }       //此句是关键,代码出自2.6.18内核，不管定时器例程一次有没有释放完timewait块，都进行 + 1 操作     twdr->slot = ((twdr->slot + 1) & (INET_TWDR_TWKILL_SLOTS - 1));     if (need_timer)         mod_timer(&twdr->tw_timer, jiffies + twdr->period);   out:       spin_unlock(&twdr->death_lock); }

用于慢timewait块释放的逻辑可参考下图：

---

3. 结论

按照 2.6.18 中的逻辑，如果一次没有全部删除一个slot中的timewait控制块， twdr->slot 仍然会执行 + 1 操作。此时如果有一个tcp_sock进入FIN_WAIT2状态，则此时的timewait(tw_substate = fin_wait2)块会被放在上一个slot中，而此时有一个线程正在处理那个队列，因此会导致处于FIN_WAIT2状态的timewait块被提前释放，若此时对端的FIN分节到达，协议栈会回复一个RST分节。

为了修复此BUG，2.6.32 协议栈中修改了 twdr->slot + 1 的时机，每次必须完全释放一个slot中所有的timewait块后，才会进行 + 1 操作。这也就是说协议栈不保证在 TCP_TWKILL_PERIOD 周期内，移动一个格子，所以当系统繁忙时，会导致timewait块的等待时间大于 TCP_TIMEWAIT_LEN。