blackhole引起k8s DNS访问异常

环境背景

3个节点的k8s集群：

[root@kube-196 ~]# kubectl get nodes
NAME       STATUS   ROLES    AGE   VERSION
kube-195   Ready    <none>   8d    v1.14.2
kube-196   Ready    master   12d   v1.14.1
kube-197   Ready    <none>   12d   v1.14.1

网络方案为flannel host-gw+ipvs。

部署情况

[root@kube-196 ~]# kubectl get all -A -o wide
NAMESPACE     NAME                                             READY   STATUS             RESTARTS   AGE     IP                NODE       NOMINATED NODE   READINESS GATES
default       pod/apm-kafka-0                                  0/1     CrashLoopBackOff   16         47m     10.244.1.16       kube-197   <none>           <none>
default       pod/apm-kafka-zookeeper-0                        1/1     Running            0          47m     10.244.1.15       kube-197   <none>           <none>
default       pod/apm-kafka-zookeeper-1                        1/1     Running            0          46m     10.244.2.31       kube-195   <none>           <none>
default       pod/apm-kafka-zookeeper-2                        1/1     Running            1          45m     10.244.2.32       kube-195   <none>           <none>
 
default       pod/testclient                                   1/1     Running            0          29m     10.244.1.17       kube-197   <none>           <none>
 
kube-system   pod/coredns-5f6bc7cdd5-phjzg                     1/1     Running            0          4d23h   10.244.2.10       kube-195   <none>           <none>
kube-system   pod/coredns-5f6bc7cdd5-tvrx8                     1/1     Running            0          79m     10.244.1.14       kube-197   <none>           <none>
kube-system   pod/coredns-5f6bc7cdd5-v96x8                     1/1     Running            0          99m     10.244.1.13       kube-197   <none>           <none>
kube-system   pod/etcd-kube-196                                1/1     Running            4          12d     192.168.200.196   kube-196   <none>           <none>
kube-system   pod/kube-apiserver-kube-196                      1/1     Running            4          22h     192.168.200.196   kube-196   <none>           <none>
kube-system   pod/kube-controller-manager-kube-196             1/1     Running            4          22h     192.168.200.196   kube-196   <none>           <none>
kube-system   pod/kube-flannel-ds-amd64-24l6b                  1/1     Running            0          81m     192.168.200.196   kube-196   <none>           <none>
kube-system   pod/kube-flannel-ds-amd64-d9tbn                  1/1     Running            2          22h     192.168.200.197   kube-197   <none>           <none>
kube-system   pod/kube-flannel-ds-amd64-scmfp                  1/1     Running            0          22h     192.168.200.195   kube-195   <none>           <none>
kube-system   pod/kube-proxy-5nd72                             1/1     Running            0          80m     192.168.200.196   kube-196   <none>           <none>
kube-system   pod/kube-proxy-qrgnf                             1/1     Running            12         12d     192.168.200.197   kube-197   <none>           <none>
kube-system   pod/kube-proxy-sfgr2                             1/1     Running            0          8d      192.168.200.195   kube-195   <none>           <none>
kube-system   pod/kube-scheduler-kube-196                      1/1     Running            5          12d     192.168.200.196   kube-196   <none>           <none>
 
NAMESPACE     NAME                                   TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                      AGE     SELECTOR
default       service/apm-kafka                      ClusterIP   10.110.242.169   <none>        9092/TCP                     47m     app=kafka,release=apm-kafka
default       service/apm-kafka-0-external           NodePort    10.101.213.10    <none>        19092:31090/TCP              47m     app=kafka,release=apm-kafka,statefulset.kubernetes.io/pod-name=apm-kafka-0
default       service/apm-kafka-1-external           NodePort    10.111.193.151   <none>        19092:31091/TCP              47m     app=kafka,release=apm-kafka,statefulset.kubernetes.io/pod-name=apm-kafka-1
default       service/apm-kafka-2-external           NodePort    10.96.221.224    <none>        19092:31092/TCP              47m     app=kafka,release=apm-kafka,statefulset.kubernetes.io/pod-name=apm-kafka-2
default       service/apm-kafka-headless             ClusterIP   None             <none>        9092/TCP                     47m     app=kafka,release=apm-kafka
default       service/apm-kafka-zookeeper            ClusterIP   10.107.245.32    <none>        2181/TCP                     47m     app=zookeeper,release=apm-kafka
default       service/apm-kafka-zookeeper-headless   ClusterIP   None             <none>        2181/TCP,3888/TCP,2888/TCP   47m     app=zookeeper,release=apm-kafka
 
default       service/kubernetes                     ClusterIP   10.96.0.1        <none>        443/TCP                      12d     <none>
kube-system   service/kube-dns                       ClusterIP   10.96.0.10       <none>        53/UDP,53/TCP,9153/TCP       4d23h   k8s-app=kube-dns


NAMESPACE     NAME                                     DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR                     AGE   CONTAINERS     IMAGES                                   SELECTOR
kube-system   daemonset.apps/kube-flannel-ds-amd64     3         3         3       3            3           beta.kubernetes.io/arch=amd64     22h   kube-flannel   quay.io/coreos/flannel:v0.11.0-amd64     app=flannel,tier=node
kube-system   daemonset.apps/kube-proxy                3         3         3       3            3           <none>                            12d   kube-proxy     k8s.gcr.io/kube-proxy:v1.14.0            k8s-app=kube-proxy

NAMESPACE     NAME                                         READY   UP-TO-DATE   AVAILABLE   AGE     CONTAINERS                   IMAGES                                                               SELECTOR
kube-system   deployment.apps/coredns                      3/3     3            3           4d23h   coredns                      k8s.gcr.io/coredns:1.5.0                                             k8s-app=kube-dns
 

NAMESPACE     NAME                                                   DESIRED   CURRENT   READY   AGE     CONTAINERS                   IMAGES                                                               SELECTOR
kube-system   replicaset.apps/coredns-5f6bc7cdd5                     3         3         3       4d23h   coredns                      k8s.gcr.io/coredns:1.5.0                                             k8s-app=kube-dns,pod-template-hash=5f6bc7cdd5
 

NAMESPACE   NAME                                   READY   AGE   CONTAINERS     IMAGES
default     statefulset.apps/apm-kafka             0/3     47m   kafka-broker   confluentinc/cp-kafka:5.0.1
default     statefulset.apps/apm-kafka-zookeeper   3/3     47m   zookeeper      registry.cn-hangzhou.aliyuncs.com/appstore/k8szk:v2

可以看到从192.168.200.195主机的容器ping 192.168.200.197主机上的其他容器的ip和headless service名字，是可以ping通的。

[root@kube-196 ~]# kubectl exec -it testclient /bin/sh 
# ping apm-kafka-headless
PING apm-kafka-headless.default.svc.cluster.local (10.244.1.16): 56 data bytes
64 bytes from 10.244.1.16: icmp_seq=0 ttl=64 time=0.097 ms
64 bytes from 10.244.1.16: icmp_seq=1 ttl=64 time=0.103 ms
64 bytes from 10.244.1.16: icmp_seq=2 ttl=64 time=0.081 ms
^C--- apm-kafka-headless.default.svc.cluster.local ping statistics ---
3 packets transmitted, 3 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.081/0.094/0.103/0.000 ms
# ping apm-kafka-zookeeper-headless 
PING apm-kafka-zookeeper-headless.default.svc.cluster.local (10.244.1.15): 56 data bytes
64 bytes from 10.244.1.15: icmp_seq=0 ttl=64 time=0.116 ms
64 bytes from 10.244.1.15: icmp_seq=1 ttl=64 time=0.109 ms
64 bytes from 10.244.1.15: icmp_seq=2 ttl=64 time=0.100 ms
64 bytes from 10.244.1.15: icmp_seq=3 ttl=64 time=0.098 ms
64 bytes from 10.244.1.15: icmp_seq=4 ttl=64 time=0.093 ms
64 bytes from 10.244.1.15: icmp_seq=5 ttl=64 time=0.107 ms
64 bytes from 10.244.1.15: icmp_seq=6 ttl=64 time=0.103 ms
^C--- apm-kafka-zookeeper-headless.default.svc.cluster.local ping statistics ---
7 packets transmitted, 7 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.093/0.104/0.116/0.000 ms
 
# ping apm-kafka-zookeeper-headless
PING apm-kafka-zookeeper-headless.default.svc.cluster.local (10.244.1.15): 56 data bytes
64 bytes from 10.244.1.15: icmp_seq=0 ttl=64 time=0.095 ms
64 bytes from 10.244.1.15: icmp_seq=1 ttl=64 time=0.099 ms
64 bytes from 10.244.1.15: icmp_seq=2 ttl=64 time=0.118 ms
64 bytes from 10.244.1.15: icmp_seq=3 ttl=64 time=0.113 ms
64 bytes from 10.244.1.15: icmp_seq=4 ttl=64 time=0.145 ms
^C--- apm-kafka-zookeeper-headless.default.svc.cluster.local ping statistics ---
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.095/0.114/0.145/0.000 ms

但ping 不同192.168.200.195上的容器（10.244.2.31）。

# ping 10.244.2.31
PING 10.244.2.31 (10.244.2.31): 56 data bytes

直接ping 192.168.200.195的宿主机也是通的，也就是说网络包到达195的宿主机之后，没有路由到容器内。

查看195的路由

[root@kube-195 ~]# ip route
default via 192.168.200.254 dev eth0  proto static  metric 100 
10.244.0.0/24 via 10.244.0.0 dev flannel.1 onlink 
10.244.1.0/24 via 10.244.1.0 dev flannel.1 onlink 
blackhole 10.244.2.0/24  proto bird 
10.244.2.0/24 dev cni0  proto kernel  scope link  src 10.244.2.1 
169.254.169.254 via 192.168.200.130 dev eth0  proto dhcp  metric 100 
192.168.82.0/24 dev docker0  proto kernel  scope link  src 192.168.82.1 
192.168.200.0/24 dev eth0  proto kernel  scope link  src 192.168.200.195  metric 100 

发现有条

blackhole 10.244.2.0/24  proto bird

的路由规则。其在

10.244.2.0/24 dev cni0  proto kernel  scope link  src 10.244.2.1

之前。所以所有在195主机上路由到10.244.2.0/24 子网的包都会被丢弃。

[root@kube-195 ~]# ip route del 10.244.2.0/24

删除该路由规则。

[root@kube-195 ~]# ip route
default via 192.168.200.254 dev eth0  proto static  metric 100 
10.244.0.0/24 via 10.244.0.0 dev flannel.1 onlink 
10.244.1.0/24 via 10.244.1.0 dev flannel.1 onlink 
10.244.2.0/24 dev cni0  proto kernel  scope link  src 10.244.2.1 
169.254.169.254 via 192.168.200.130 dev eth0  proto dhcp  metric 100 
192.168.82.0/24 dev docker0  proto kernel  scope link  src 192.168.82.1 
192.168.200.0/24 dev eth0  proto kernel  scope link  src 192.168.200.195  metric 100 

就可以ping通了：

# ping apm-kafka-zookeeper-headless
PING apm-kafka-zookeeper-headless.default.svc.cluster.local (10.244.2.31): 56 data bytes
64 bytes from 10.244.2.31: icmp_seq=0 ttl=62 time=0.920 ms
64 bytes from 10.244.2.31: icmp_seq=1 ttl=62 time=0.765 ms
64 bytes from 10.244.2.31: icmp_seq=2 ttl=62 time=0.940 ms
64 bytes from 10.244.2.31: icmp_seq=3 ttl=62 time=0.924 ms
^C--- apm-kafka-zookeeper-headless.default.svc.cluster.local ping statistics ---
4 packets transmitted, 4 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.765/0.887/0.940/0.071 ms

发现calico网络也有类似的问题：
https://github.com/projectcalico/calico/issues/2457
因为我们的集群之前也安装过calico,而该规则适用于calico网络下本机访问的。

参考

blackhole可以用于阻止登录。

In the world of systems administration, having the commands to add or remove a blackhole / nullroute a host or problematic IP address can be very useful. So in this simple, yet useful tutorial, we’ll cover how to exactly go about doing this on nearly any type of Linux OS!

• How to add a blackhole nullroute:

For this example, let’s assume that we are receiving unwanted SSH login attempts from 192.168.0.195 .

root@server:~# netstat -na | grep :22
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN
tcp 0 0 192.168.0.197:22 192.168.0.195:57776 ESTABLISHED

To add the blackhole for 192.168.0.195:

root@server:~# ip route add blackhole 192.168.0.195/32

To verify the route is in place will will use “ip route show “:

root@server:~# ip route show
default via 192.168.0.1 dev eth0 metric 100
blackhole 192.168.0.195

Subsequently, connections to and from that IP will fail:

root@attacker:~$ ssh 192.168.0.2
ssh: connect to host 192.168.0.2 port 22: No route to host

• Removing a blackhole:

Typically, blackholes are useful when your server is under attack. After the attack has subsided, or you wish to remove the blackhole you can do so as follows:

root@server:~# ip route del 192.168.0.195
root@server:~# ip route show
default via 192.168.0.1 dev eth0 metric 100