Big Packet lose in K8s (三) - Simulate the Packets lost issue on the K8s environment by manual-优快云博客

博客聚焦在虚拟化环境中手工模拟丢包问题，明确此为纯粹网络问题，与K8s、Docker、flannel无关。介绍了宿主机、虚拟机和客户端物理机的IP地址，阐述复现步骤，包括在虚拟机建网络环境、设iptables转发规则、运行测试http服务，最后进行客户端测试。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

在虚拟化的环境中手工模拟丢包问题

进一步明确该问题是一个纯粹的网络问题，而与K8s、Docker、flannel都没有关系

环境

宿主机： 10.10.88.128
虚拟机： 10.10.88.118/24
客户端物理机：10.10.88.2

复现步骤

在虚拟机中建立网络环境

test.sh:


#!/bin/sh

brctl addbr docker0

ip link add wyveth0 type veth peer name wyveth1

brctl addif docker0 wyveth0

ip netns add wy

ip link set wyveth1 netns wy



ip link set docker0 up

ip link set wyveth0 up

ip netns exec wy ip link set wyveth1 up



ip addr add 10.0.3.1/24 dev docker0

ip netns exec wy ip addr add 10.0.3.5/24 dev wyveth1

ip netns exec wy ip addr



echo 1 > /proc/sys/net/ipv4/ip_forward

echo 1 >/proc/sys/net/bridge/bridge-nf-call-iptables

bash test.sh

设置iptables转发规则

来自于K8s的配置

wuyan.iptables


# Generated by iptables-save v1.4.21 on Thu Jul 26 07:32:12 2018

*filter

:INPUT ACCEPT [983:66443]

:FORWARD DROP [0:0]

:OUTPUT ACCEPT [665:185799]

:DOCKER - [0:0]

:DOCKER-ISOLATION - [0:0]

:KUBE-EXTERNAL-SERVICES - [0:0]

:KUBE-FIREWALL - [0:0]

:KUBE-FORWARD - [0:0]

:KUBE-SERVICES - [0:0]

-A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES

-A INPUT -j KUBE-FIREWALL

-A FORWARD -j DOCKER-ISOLATION

-A FORWARD -o docker0 -j DOCKER

-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT

-A FORWARD -i docker0 ! -o docker0 -j ACCEPT

-A FORWARD -i docker0 -o docker0 -j ACCEPT

-A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD

-A FORWARD -s 10.244.0.0/16 -j ACCEPT

-A FORWARD -d 10.244.0.0/16 -j ACCEPT

-A OUTPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES

-A OUTPUT -j KUBE-FIREWALL

-A DOCKER-ISOLATION -j RETURN

-A KUBE-EXTERNAL-SERVICES -p tcp -m comment --comment "kube-system/grafana-proxy-service:proxy has no endpoints" -m addrtype --dst-type LOCAL -m tcp --dport 31063 -j REJECT --reject-with icmp-port-unreachable

-A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP

-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT

-A KUBE-FORWARD -s 10.244.0.0/16 -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT

-A KUBE-FORWARD -d 10.244.0.0/16 -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT

COMMIT

# Completed on Thu Jul 26 07:32:12 2018

# Generated by iptables-save v1.4.21 on Thu Jul 26 07:32:12 2018

*raw

:PREROUTING ACCEPT [60131:65785774]

:OUTPUT ACCEPT [39490:4150752]

COMMIT

# Completed on Thu Jul 26 07:32:12 2018

# Generated by iptables-save v1.4.21 on Thu Jul 26 07:32:12 2018

*nat

:PREROUTING ACCEPT [1030:35860]

:INPUT ACCEPT [11:660]

:OUTPUT ACCEPT [2:168]

:POSTROUTING ACCEPT [2:168]

:DOCKER - [0:0]

:KUBE-MARK-DROP - [0:0]

:KUBE-MARK-MASQ - [0:0]

:KUBE-NODEPORTS - [0:0]

:KUBE-POSTROUTING - [0:0]

:KUBE-SEP-TROAH4R4NOH4BZQH - [0:0]

:KUBE-SERVICES - [0:0]

:KUBE-SVC-JRXTEHDDTAFMSEAS - [0:0]

-A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES

-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER

-A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES

-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER

-A POSTROUTING -s 10.0.8.0/24 ! -o docker0 -j MASQUERADE

-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING

-A POSTROUTING -s 10.244.0.0/16 -d 10.244.0.0/16 -j RETURN

-A POSTROUTING -s 10.244.0.0/16 ! -d 224.0.0.0/4 -j MASQUERADE

-A POSTROUTING ! -s 10.244.0.0/16 -d 10.244.0.0/24 -j RETURN

-A POSTROUTING ! -s 10.244.0.0/16 -d 10.244.0.0/16 -j MASQUERADE

-A DOCKER -i docker0 -j RETURN

-A KUBE-MARK-DROP -j MARK --set-xmark 0x8000/0x8000

-A KUBE-MARK-MASQ -j MARK --set-xmark 0x4000/0x4000

-A KUBE-NODEPORTS -p tcp -m comment --comment "kube-system/monitoring-grafana:" -m tcp --dport 30063 -j KUBE-MARK-MASQ

-A KUBE-NODEPORTS -p tcp -m comment --comment "kube-system/monitoring-grafana:" -m tcp --dport 30063 -j KUBE-SVC-JRXTEHDDTAFMSEAS

-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -m mark --mark 0x4000/0x4000 -j MASQUERADE

-A KUBE-SEP-TROAH4R4NOH4BZQH -s 10.244.2.234/32 -m comment --comment "kube-system/monitoring-grafana:" -j KUBE-MARK-MASQ

-A KUBE-SEP-TROAH4R4NOH4BZQH -p tcp -m comment --comment "kube-system/monitoring-grafana:" -m tcp -j DNAT --to-destination 10.0.3.5:3001

-A KUBE-SERVICES -d 10.111.220.243/32 -p tcp -m comment --comment "kube-system/monitoring-grafana: cluster IP" -m tcp --dport 80 -j KUBE-SVC-JRXTEHDDTAFMSEAS

-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS

-A KUBE-SVC-JRXTEHDDTAFMSEAS -m comment --comment "kube-system/monitoring-grafana:" -j KUBE-SEP-TROAH4R4NOH4BZQH

COMMIT

# Completed on Thu Jul 26 07:32:12 2018

iptables-restore < wuyan.iptable

运行一个测试的http 服务

http.py


import SimpleHTTPServer

import SocketServer



PORT = 3001



Handler = SimpleHTTPServer.SimpleHTTPRequestHandler



httpd = SocketServer.TCPServer(("10.0.3.5", PORT), Handler)



print "serving at port", PORT

httpd.serve_forever()

ip netns exec wy python http.py

在客户端测试


curl 'http://10.10.88.118:30063/api/datasources/proxy/1/query?db=k8s&q=SELECT%20sum(%22value%22)%20FROM%20%22memory%2Fusage%22%20WHERE%20%22type%22%20%3D%20%27node%27%20AND%20time%20%3E%20now()%20-%205m%20GROUP%20BY%20time(500ms)%2C%20%22nodename%22%20fill(null)%3BSELECT%20sum(%22value%22)%20FROM%20%22memory%2Fworking_set%22%20WHERE%20%22type%22%20%3D%20%27node%27%20AND%20time%20%3E%20now()%20-%205m%20GROUP%20BY%20time(500ms)%2C%20%22nodename%22%20fill(null)%3BSELECT%20sum(%22value%22)%20FROM%20%22memory%2Flimit%22%20WHERE%20%22type%22%20%3D%20%27node%27%20AND%20time%20%3E%20now()%20-%205m%20GROUP%20BY%20time(500ms)%2C%20%22nodename%22%20fill(null)%3BSELECT%20sum(%22value%22)%20FROM%20%22memory%2Frequest%22%20WHERE%20%22type%22%20%3D%20%27node%27%20AND%20time%20%3E%20now()%20-%205m%20GROUP%20BY%20time(500ms)%2C%20%22nodename%22%20fill(null)&epoch=ms' -H 'Host: 10.10.88.2:30063' -H 'Accept: application/json, text/plain, */*' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36' -H 'Referer: http://10.10.88.2:30063/dashboard/file/qfusion-cluster.json?from=now-5m&to=now' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en,zh-CN;q=0.9,zh;q=0.8' -H 'Cookie: accountName=admin; sessionUuid=ed815b652c1b49caa21a5e4a6544db5f; accountUuid=36c27e8ff05c4780bf6d2fa65700f22e; userUuid=36c27e8ff05c4780bf6d2fa65700f22e; QFusion_Module_auth=%7B%22userId%22%3A%2236c27e8ff05c4780bf6d2fa65700f22e%22%2C%22userName%22%3A%22admin%22%7D; grafana_sess=34eb37666da74622' -H 'Max-Forwards: 100' -H 'X-Forwarded-For: 10.10.200.28' -H 'X-Forwarded-Host: 10.10.88.2:30063' -H 'X-Forwarded-Server: grafana.com' -H 'Connection: Keep-Alive'