本实验为验证Antrea在原生Kubernetes集群的安装和拓扑研究
测试环境
Antrea安装
下载地址：https://github.com/antrea-io/antrea/releases/tag/v0.13.3，以及安装使用该页面上antrea.yml
[root@master-01 ~]# kubectl exec -it antrea-controller-787747dbb7-fck8l -n kube-system -- antctl version
antctlVersion: v0.13.3
controllerVersion: v0.13.3
Kubernetes环境集成
Cluster
[root@master-01 ~]# kubectl get node -owide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
master-01 Ready control-plane,master 4d15h v1.21.2 192.168.110.61 <none> CentOS Linux 7 (Core) 3.10.0-1160.31.1.el7.x86_64 docker://20.10.7
worker-01 Ready <none> 4d15h v1.21.2 192.168.110.66 <none> CentOS Linux 7 (Core) 3.10.0-1062.el7.x86_64 docker://20.10.7
worker-02 Ready <none> 4d15h v1.21.2 192.168.110.67 <none> CentOS Linux 7 (Core) 3.10.0-1062.el7.x86_64 docker://20.10.7
| Name | Role | IP Addr |
|---|---|---|
| master-01 | control-plane,master | 192.168.110.61 |
| worker-01 | worker | 192.168.110.66 |
| worker-02 | worker | 192.168.110.67 |
Worker-01
[root@worker-01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:50:56:b2:35:31 brd ff:ff:ff:ff:ff:ff
inet 192.168.110.66/24 brd 192.168.110.255 scope global noprefixroute ens192
valid_lft forever preferred_lft forever
inet6 fe80::f343:5255:f35a:fa51/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::bed0:b654:1f36:9ded/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::2126:395:4153:6fe4/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000
link/ether 52:54:00:5a:d2:28 brd ff:ff:ff:ff:ff:ff
inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 1000
link/ether 52:54:00:5a:d2:28 brd ff:ff:ff:ff:ff:ff
5: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:87:6a:8a:03 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
6: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 3e:60:c2:54:9f:de brd ff:ff:ff:ff:ff:ff
7: genev_sys_6081: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc noqueue master ovs-system state UNKNOWN group default qlen 1000
link/ether aa:ce:13:21:f1:bc brd ff:ff:ff:ff:ff:ff
inet6 fe80::a8ce:13ff:fe21:f1bc/64 scope link
valid_lft forever preferred_lft forever
8: antrea-gw0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 36:ab:6b:31:0c:75 brd ff:ff:ff:ff:ff:ff
inet 10.211.1.1/24 brd 10.211.1.255 scope global antrea-gw0
valid_lft forever preferred_lft forever
inet6 fe80::34ab:6bff:fe31:c75/64 scope link
valid_lft forever preferred_lft forever
11: coredns--c4f3d4@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master ovs-system state UP group default
link/ether de:72:46:4d:6b:85 brd ff:ff:ff:ff:ff:ff link-netnsid 2
inet6 fe80::dc72:46ff:fe4d:6b85/64 scope link
valid_lft forever preferred_lft forever
Worker-02
[root@worker-02 harbor.corp.tanzu]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:50:56:b2:46:58 brd ff:ff:ff:ff:ff:ff
inet 192.168.110.67/24 brd 192.168.110.255 scope global noprefixroute ens192
valid_lft forever preferred_lft forever
inet6 fe80::bed0:b654:1f36:9ded/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::f343:5255:f35a:fa51/64 scope link noprefixroute
valid_lft forever preferred_lft forever
3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000
link/ether 52:54:00:5a:d2:28 brd ff:ff:ff:ff:ff:ff
inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
valid_lft forever preferred_lft forever
4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 1000
link/ether 52:54:00:5a:d2:28 brd ff:ff:ff:ff:ff:ff
5: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default
link/ether 02:42:72:d7:4a:a4 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 brd 172.17.255.255 scope global docker0
valid_lft forever preferred_lft forever
6: ovs-system: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether ea:d9:69:03:42:6e brd ff:ff:ff:ff:ff:ff
8: antrea-gw0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 12:c8:e9:86:7e:7b brd ff:ff:ff:ff:ff:ff
inet 10.211.2.1/24 brd 10.211.2.255 scope global antrea-gw0
valid_lft forever preferred_lft forever
inet6 fe80::10c8:e9ff:fe86:7e7b/64 scope link
valid_lft forever preferred_lft forever
26: genev_sys_6081: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 65000 qdisc noqueue master ovs-system state UNKNOWN group default qlen 1000
link/ether 6e:54:7d:ca:22:8b brd ff:ff:ff:ff:ff:ff
inet6 fe80::6c54:7dff:feca:228b/64 scope link
valid_lft forever preferred_lft forever
28: ako-0-d03cf7@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master ovs-system state UP group default
link/ether e6:5b:9f:f6:6e:11 brd ff:ff:ff:ff:ff:ff link-netnsid 1
inet6 fe80::e45b:9fff:fef6:6e11/64 scope link
valid_lft forever preferred_lft forever
29: coredns--c35d50@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master ovs-system state UP group default
link/ether 76:b3:51:a3:58:5b brd ff:ff:ff:ff:ff:ff link-netnsid 2
inet6 fe80::74b3:51ff:fea3:585b/64 scope link
valid_lft forever preferred_lft forever
worker-01和02的端口情况来看，我们可以看到在Node上antrea-gw0和turn0情况
| HostName | Role name | IP addr | MAC |
|---|---|---|---|
| worker-01 | genev_sys_6081 | 36: ab:6b:31:0c:75 | |
| worker-02 | genev_sys_6081 | 6e:54:7d:ca:22:8b | |
| worker-01 | antrea-gw0 | 10.211.1.1/24 | |
| worker-02 | antrea-gw0 | 10.211.2.1/24 |
- “genev_sys” is the actual tunnel interface (antrea-tun0) used on the OVS in the worker nodes for overlay encapsulation functionality of Antrea. It acts as a slave interface of “ovs-system” interface.
- “ovs-system” interface is the OVS internal port which is not used for any IP / OSI Layer 3 function here. For more info please check this introduction video.
- “antrea-gw0” interface is basically the default gateway interface for all the pods on Worker .
Route
[root@worker-01 ~]# ip route
default via 192.168.110.1 dev ens192 proto static metric 100
10.211.0.0/24 via 10.211.0.1 dev antrea-gw0 onlink
10.211.1.0/24 dev antrea-gw0 proto kernel scope link src 10.211.1.1
10.211.2.0/24 via 10.211.2.1 dev antrea-gw0 onlink
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
192.168.110.0/24 dev ens192 proto kernel scope link src 192.168.110.66 metric 100
192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1
从上面的结果可以看到，对于Worker Node，其Pod路由通过antrea-gw0转发，而外部路由通过主机端口转发（有NAT），默认路由即主机端口
Antrea Agent Pods
前面我们说过：在每个工作节点上，Antrea 以 DaemonSet 的形式部署运行 antrea-agent 和 OVS 用户态守护进程，并以 init container 的形式将 antrea-cni 插件安装到主机并加载 OVS 内核模块。antrea-agent 负责 OVS 网桥和 Pod网络接口的管理，并通过OpenFlow协议编程OVS来实现各种网络和安全功能。
[root@master-01 ~]# kubectl get po -A -owide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
avi-system ako-0 1/1 Running 2 3d22h 10.211.2.20 worker-02 <none> <none>
default backend1 1/1 Running 0 175m 10.211.1.15 worker-01 <none> <none>
default backend2 1/1 Running 0 175m 10.211.2.30 worker-02 <none> <none>
default frontend 1/1 Running 0 175m 10.211.1.14 worker-01 <none> <none>
kube-system antrea-agent-5fx86 2/2 Running 7 4d19h 192.168.110.61 master-01 <none> <none>
kube-system antrea-agent-bj42h 2/2 Running 3 4d19h 192.168.110.66 worker-01 <none> <none>
kube-system antrea-agent-j28lw 2/2 Running 8 4d19h 192.168.110.67 worker-02 <none> <none>
kube-system antrea-controller-787747dbb7-fck8l 1/1 Running 2 4d19h 192.168.110.66 worker-01 <none> <none>
kube-system coredns-558bd4d5db-gtvr4 1/1 Running 1 4d19h 10.211.1.4 worker-01 <none> <none>
kube-system coredns-558bd4d5db-vgjs2 1/1 Running 3 4d19h 10.211.2.21 worker-02 <none> <none>
kube-system etcd-master-01 1/1 Running 7 4d20h 192.168.110.61 master-01 <none> <none>
kube-system kube-apiserver-master-01 1/1 Running 8 4d20h 192.168.110.61 master-01 <none> <none>
kube-system kube-controller-manager-master-01 1/1 Running 34 4d20h 192.168.110.61 master-01 <none> <none>
kube-system kube-proxy-nkql8 1/1 Running 3 4d19h 192.168.110.67 worker-02 <none> <none>
kube-system kube-proxy-q2mzj 1/1 Running 1 4d19h 192.168.110.66 worker-01 <none> <none>
kube-system kube-proxy-q7fx9 1/1 Running 5 4d19h 192.168.110.61 master-01 <none> <none>
kube-system kube-scheduler-master-01 1/1 Running 30 4d20h 192.168.110.61 master-01 <none> <none>
kube-system antrea-agent-bj42h 2/2 Running 3 4d19h 192.168.110.66 worker-01
kube-system antrea-agent-j28lw 2/2 Running 8 4d19h 192.168.110.67 worker-02
OVS
借助于"ovs-vsctl show" 命令可以查看 antrea-ovs的状况，这里用的上一步的antrea-agent 容器，以worker-01为例
[root@master-01 ~]# kubectl exec -n kube-system -it antrea-agent-bj42h -c antrea-ovs -- ovs-vsctl show
7ccc18b5-068f-4977-b7cb-a9fefdd147c3
Bridge br-int
datapath_type: system
Port backend1-911dea
Interface backend1-911dea
Port coredns--c4f3d4
Interface coredns--c4f3d4
Port antrea-tun0
Interface antrea-tun0
type: geneve
options: {csum="true", key=flow, remote_ip=flow}
Port antrea-gw0
Interface antrea-gw0
type: internal
Port frontend-fbd015
Interface frontend-fbd015
ovs_version: "2.14.0"
借助于"ovs-ofctl show br-int" 命令可以查看 antrea-ovs的端口情况，同样以worker-01为例
[root@master-01 ~]# kubectl exec -n kube-system -it antrea-agent-bj42h -c antrea-ovs -- ovs-ofctl show br-int
OFPT_FEATURES_REPLY (xid=0x2): dpid:0000166b68c6424a
n_tables:254, n_buffers:0
capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src mod_dl_dst mod_nw_src mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst
**1(antrea-tun0): addr:06:c6:e4:f2:94:51**
config: 0
state: 0
speed: 0 Mbps now, 0 Mbps max
**2(antrea-gw0): addr:36:ab:6b:31:0c:75**
config: 0
state: 0
speed: 0 Mbps now, 0 Mbps max
5(coredns--c4f3d4): addr:de:72:46:4d:6b:85
config: 0
state: 0
current: 10GB-FD COPPER
speed: 10000 Mbps now, 0 Mbps max
**15(frontend-fbd015): addr:86:e0:33:27:a8:c0**
config: 0
state: 0
current: 10GB-FD COPPER
speed: 10000 Mbps now, 0 Mbps max
**16(backend1-911dea): addr:c6:97:94:e1:2e:a5**
config: 0
state: 0
current: 10GB-FD COPPER
speed: 10000 Mbps now, 0 Mbps max
OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0
可以看到该OVS@worker-01上有turn0，gw0，frontend和backend1的接入。
测试用例App
采用集成网络工具https://hub.docker.com/r/praqma/network-multitool/，下面的yaml仅部署了三个 Pod（前端、后端 1、后端 2）、一个服务 (ClusterIP) 和两个网络策略（一个用于前端 Pod，一个用于后端 Pod）。
apiVersion: v1
kind: Pod
metadata:
labels:
role: frontend
name: frontend
spec:
containers:
- image: harbor.corp.tanzu/library/network-multitool@sha256:1a546071c99290fa1d02f8ded26070e1e5711efeb02b3208752b92834f058948
name: frontend
nodeName: worker-01
---
apiVersion: v1
kind: Pod
metadata:
labels:
role: backend
name: backend1
spec:
containers:
- image: harbor.corp.tanzu/library/network-multitool@sha256:1a546071c99290fa1d02f8ded26070e1e5711efeb02b3208752b92834f058948
name: backend1
nodeName: worker-01
---
apiVersion: v1
kind: Pod
metadata:
labels:
role: backend
name: backend2
spec:
containers:
- image: harbor.corp.tanzu/library/network-multitool@sha256:1a546071c99290fa1d02f8ded26070e1e5711efeb02b3208752b92834f058948
name: backend2
nodeName: worker-02
---
apiVersion: v1
kind: Service
metadata:
labels:
app: backendsvc
name: backendsvc
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
role: backend
type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: frontendpolicy
namespace: default
spec:
podSelector:
matchLabels:
role: frontend
policyTypes:
- Egress
- Ingress
egress:
- to:
- podSelector:
matchLabels:
role: backend
ports:
- protocol: TCP
port: 80
- ports:
- protocol: UDP
port: 53
ingress:
- ports:
- protocol: TCP
port : 80
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: backendpolicy
namespace: default
spec:
podSelector:
matchLabels:
role: backend
policyTypes:
- Ingress
- Egress
ingress:
- from:
- podSelector:
matchLabels:
role: frontend
ports:
- protocol: TCP
port: 80
在上面显示的 yaml 文件中，直接将特定的 pod 固定到特定的工作节点。因此，在 yaml 中使用了“Nodename”。
运行以后：
[root@master-01 ~]# kubectl get po -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
backend1 1/1 Running 0 20s 10.211.1.15 worker-01 <none> <none>
backend2 1/1 Running 0 20s 10.211.2.30 worker-02 <none> <none>
frontend 1/1 Running 0 20s 10.211.1.14 worker-01 <none> <none>
[root@master-01 ~]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
backendsvc ClusterIP 10.101.216.214 <none> 80/TCP 47s
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 4d17h
[root@master-01 ~]# kubectl get ep
NAME ENDPOINTS AGE
backendsvc 10.211.1.15:80,10.211.2.30:80 63s
kubernetes 192.168.110.61:6443 4d17h
查看运行情况，登录到frontend上查看后端运行，这个可以使用svc：backendsvc
[root@master-01 ~]# kubectl exec -it frontend -- sh
/ # curl backendsvc
Praqma Network MultiTool (with NGINX) - backend1 - 10.211.1.15
/ # curl backendsvc
Praqma Network MultiTool (with NGINX) - backend1 - 10.211.1.15
/ # curl backendsvc
Praqma Network MultiTool (with NGINX) - backend2 - 10.211.2.30
可以查看到两个后端backend1和2工作正常。
为了让前端 pod 在其 IP 上向 backendsvc 服务发送请求，它实际上向 Kubernetes DNS 服务发送了 DNS 请求。
测试拓扑图
综合上面的分析，我们得到了测试的拓扑图