1、节点
root@ubuntu:~# kubectl get nodes --show-labels NAME STATUS ROLES AGE VERSION LABELS bogon Ready worker 5d21h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=bogon,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker,rr-group=rr1 centos7 Ready worker 26h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=centos7,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker,rr-group=rr2 cloud Ready worker 5d21h v1.21.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=cloud,kubernetes.io/os=linux,node-role.kubernetes.io/worker=worker,rr-group=rr2,rr-id=rr2 ubuntu Ready master 5d21h v1.18.1 beta.kubernetes.io/arch=arm64,beta.kubernetes.io/os=linux,kubernetes.io/arch=arm64,kubernetes.io/hostname=ubuntu,kubernetes.io/os=linux,node-role.kubernetes.io/master=,rr-group=rr1,rr-id=rr1 root@ubuntu:~#
2、bgpconfig
root@ubuntu:~# ./calicoctl get bgpconfig NAME LOGSEVERITY MESHENABLED ASNUMBER default Info false 64512 root@ubuntu:~#
3、RR1
./calicoctl patch node ubuntu -p ‘{"spec": {"bgp": {"asNumber": "64514"}}}‘ root@ubuntu:~# ./calicoctl get node ubuntu --export -o yaml > ubuntu.yaml root@ubuntu:~# cat ubuntu.yaml apiVersion: projectcalico.org/v3 kind: Node metadata: annotations: projectcalico.org/kube-labels: ‘{"beta.kubernetes.io/arch":"arm64","beta.kubernetes.io/os":"linux","kubernetes.io/arch":"arm64","kubernetes.io/hostname":"ubuntu","kubernetes.io/os":"linux","node-role.kubernetes.io/master":"","rr-group":"rr1","rr-id":"rr1"}‘ creationTimestamp: null labels: beta.kubernetes.io/arch: arm64 beta.kubernetes.io/os: linux kubernetes.io/arch: arm64 kubernetes.io/hostname: ubuntu kubernetes.io/os: linux node-role.kubernetes.io/master: "" rr-group: rr1 rr-id: rr1 name: ubuntu spec: addresses: - address: 10.10.16.82/24 type: CalicoNodeIP - address: 10.10.16.82 type: InternalIP bgp: asNumber: 64514 ipv4Address: 10.10.16.82/24 routeReflectorClusterID: 244.0.0.1 orchRefs: - nodeName: ubuntu orchestrator: k8s status: podCIDRs: - 10.244.0.0/24
root@ubuntu:~# cat rr1-to-leaf1-peer.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: rr1-to-leaf1-peer ## 给BGPPeer取一个名称，方便识别 spec: nodeSelector: rr-id == ‘rr1‘ ## 通过节点选择器添加有rr-id == ‘rr1‘标签的节点 peerIP: 10.10.18.31 ## leaf01交换机的地址 asNumber: 64514 ## leaf01交换机的AS号 root@ubuntu:~# cat rr1-to-node-peer.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: rr1-to-node-peer ## 给BGPPeer取一个名称，方便识别 spec: nodeSelector: rr-group == ‘rr1‘ ## 通过节点选择器添加有rr-group == ‘rr1’标签的节点 peerSelector: rr-id == ‘rr1‘ ## 通过peer选择器添加有rr-id == ‘rr1’标签的路由反射器 root@ubuntu:~#
leaf1
host-10-10-18-31# sh run Building configuration... Current configuration: ! frr version 7.5 frr defaults traditional hostname host-10-10-18-31 no ip forwarding no ipv6 forwarding hostname bgpd log stdout no service integrated-vtysh-config ! password zebra ! router bgp 64514 bgp router-id 10.10.18.31 no bgp ebgp-requires-policy neighbor 10.10.16.82 remote-as 64514 neighbor 10.10.18.34 remote-as 64513 ! address-family ipv4 unicast network 31.31.31.31/32 exit-address-family ! line vty ! end host-10-10-18-31# sh ip bgp sum IPv4 Unicast Summary: BGP router identifier 10.10.18.31, local AS number 64514 vrf-id 0 BGP table version 5 RIB entries 5, using 960 bytes of memory Peers 2, using 43 KiB of memory Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt 10.10.16.82 4 64514 22 19 0 0 0 00:15:57 2 1 10.10.18.34 4 64513 85 85 0 0 0 01:19:45 0 1 Total number of neighbors 2 host-10-10-18-31#
host-10-10-18-31# sh ip bgp nei 10.10.16.82 routes BGP table version is 5, local router ID is 10.10.18.31, vrf id 0 Default local pref 100, local AS 64514 Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, i internal, r RIB-failure, S Stale, R Removed Nexthop codes: @NNN nexthop‘s vrf id, < announce-nh-self Origin codes: i - IGP, e - EGP, ? - incomplete Network Next Hop Metric LocPrf Weight Path i10.244.29.0/26 10.10.16.81 100 0 64512 i i10.244.243.192/26 10.10.16.82 100 0 i Displayed 2 routes and 3 total paths host-10-10-18-31#
Displayed 2 routes and 3 total paths host-10-10-18-31# sh ip bgp nei 10.10.18.34 routes host-10-10-18-31#
34没有把学习到的路由通告给31
rr2
root@cloud:~# cat cloud.yaml apiVersion: projectcalico.org/v3 kind: Node metadata: annotations: projectcalico.org/kube-labels: ‘{"beta.kubernetes.io/arch":"arm64","beta.kubernetes.io/os":"linux","kubernetes.io/arch":"arm64","kubernetes.io/hostname":"cloud","kubernetes.io/os":"linux","node-role.kubernetes.io/worker":"worker","rr-group":"rr2","rr-id":"rr2"}‘ creationTimestamp: null labels: beta.kubernetes.io/arch: arm64 beta.kubernetes.io/os: linux kubernetes.io/arch: arm64 kubernetes.io/hostname: cloud kubernetes.io/os: linux node-role.kubernetes.io/worker: worker rr-group: rr2 rr-id: rr2 name: cloud spec: addresses: - address: 10.10.16.47/24 type: CalicoNodeIP - address: 10.10.16.47 type: InternalIP bgp: asNumber: 64513 ipv4Address: 10.10.16.47/24 routeReflectorClusterID: 10.10.16.47 orchRefs: - nodeName: cloud orchestrator: k8s status: podCIDRs: - 10.244.1.0/24 root@cloud:~#
root@ubuntu:~# cat rr2-to-leaf2-peer.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: rr2-to-leaf2-peer ## 给BGPPeer取一个名称，方便识别 spec: nodeSelector: rr-id == ‘rr2‘ ## 通过节点选择器添加有rr-id == ‘rr2‘标签的节点 peerIP: 10.10.18.34 ## leaf02交换机的地址 asNumber: 64513 ## leaf02交换机的AS号 root@ubuntu:~# cat rr2-to-node-peer.yaml apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: rr2-to-node-peer ## 给BGPPeer取一个名称，方便识别 spec: nodeSelector: rr-group == ‘rr2‘ ## 通过节点选择器添加有rr-group == ‘rr2’标签的节点 peerSelector: rr-id == ‘rr2‘ ## 通过peer选择器添加有rr-id == ‘rr2’标签的路由反射器 root@ubuntu:~#
leaf2
host-10-10-18-34# sh run Building configuration... Current configuration: ! frr version 7.5 frr defaults traditional hostname host-10-10-18-34 log file /var/log/frr/frr.log no ip forwarding no ipv6 forwarding hostname bgpd log stdout no service integrated-vtysh-config ! debug bgp neighbor-events debug bgp nht debug bgp updates in debug bgp updates out debug bgp zebra ! password zebra ! router bgp 64513 bgp router-id 10.10.18.34 no bgp ebgp-requires-policy neighbor 10.10.16.47 remote-as 64513 neighbor 10.10.18.31 remote-as 64514 ! line vty ! end host-10-10-18-34# sh ip bgp sum IPv4 Unicast Summary: BGP router identifier 10.10.18.34, local AS number 64513 vrf-id 0 BGP table version 3 RIB entries 1, using 192 bytes of memory Peers 2, using 43 KiB of memory Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt 10.10.16.47 4 64513 0 0 0 0 0 never Active 0 10.10.18.31 4 64514 55 56 0 0 0 00:49:21 1 1 Total number of neighbors 2 host-10-10-18-34# sh ip bgp sum IPv4 Unicast Summary: BGP router identifier 10.10.18.34, local AS number 64513 vrf-id 0 BGP table version 3 RIB entries 5, using 960 bytes of memory Peers 2, using 43 KiB of memory Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd PfxSnt 10.10.16.47 4 64513 14 13 0 0 0 00:09:14 2 1 10.10.18.31 4 64514 64 65 0 0 0 00:58:56 1 1 Total number of neighbors 2 host-10-10-18-34# sh ip bgp nei 10.10.16.47 routes BGP table version is 3, local router ID is 10.10.18.34, vrf id 0 Default local pref 100, local AS 64513 Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, i internal, r RIB-failure, S Stale, R Removed Nexthop codes: @NNN nexthop‘s vrf id, < announce-nh-self Origin codes: i - IGP, e - EGP, ? - incomplete Network Next Hop Metric LocPrf Weight Path i10.244.2.0/24 10.10.16.47 100 0 i i10.244.41.0/26 10.10.16.47 100 0 i Displayed 2 routes and 3 total paths host-10-10-18-34# ping
host-10-10-18-34# sh ip bgp nei 10.10.18.31 routes BGP table version is 3, local router ID is 10.10.18.34, vrf id 0 Default local pref 100, local AS 64513 Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, i internal, r RIB-failure, S Stale, R Removed Nexthop codes: @NNN nexthop‘s vrf id, < announce-nh-self Origin codes: i - IGP, e - EGP, ? - incomplete Network Next Hop Metric LocPrf Weight Path *> 31.31.31.31/32 10.10.18.31 0 0 64514 i Displayed 1 routes and 3 total paths host-10-10-18-34#
node bgp 信息
ubuntu
root@ubuntu:~# ./calicoctl get nodes -o wide NAME ASN IPV4 IPV6 bogon (64512) 10.10.16.81/24 centos7 (64512) 10.10.16.251/24 cloud 64513 10.10.16.47/24 ubuntu 64514 10.10.16.82/24 root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-leaf1-peer 10.10.18.31 rr-id == ‘rr1‘ 64514 rr1-to-node-peer rr-group == ‘rr1‘ 0 rr2-to-leaf2-peer 10.10.18.34 rr-id == ‘rr2‘ 64513 rr2-to-node-peer rr-group == ‘rr2‘ 0 root@ubuntu:~# ./calicoctl get bgppeer NAME PEERIP NODE ASN rr1-to-leaf1-peer 10.10.18.31 rr-id == ‘rr1‘ 64514 rr1-to-node-peer rr-group == ‘rr1‘ 0 rr2-to-leaf2-peer 10.10.18.34 rr-id == ‘rr2‘ 64513 rr2-to-node-peer rr-group == ‘rr2‘ 0 root@ubuntu:~# root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.81 | node specific | up | 08:46:14 | Established | | 10.10.18.31 | node specific | up | 08:53:41 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
cloud
root@cloud:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.251 | node specific | up | 08:36:20 | Established | | 10.10.18.34 | node specific | up | 08:39:35 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@cloud:~#
bogon
[root@bogon ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.82 | node specific | up | 08:46:16 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. You have mail in /var/spool/mail/root [root@bogon ~]#
centos7
[root@centos7 ~]# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.16.47 | node specific | up | 08:36:20 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. [root@centos7 ~]#
创建pod
root@ubuntu:~# cat web-anti-affinity.yaml apiVersion: apps/v1 kind: Deployment metadata: name: web-nginx spec: selector: matchLabels: app: web-nginx replicas: 4 template: metadata: labels: app: web-nginx spec: affinity: #pod 反亲和性, 打散 web-nginx 各个副本 podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - web-nginx topologyKey: "kubernetes.io/hostname" containers: - image: nginx imagePullPolicy: IfNotPresent name: web2-worker ports: - containerPort: 80 protocol: TCP root@ubuntu:~#
root@ubuntu:~# kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES web-nginx-75bff6b7cf-6tcdp 1/1 Running 0 8s 10.244.129.129 centos7 <none> <none> web-nginx-75bff6b7cf-bs4nh 1/1 Running 0 8s 10.244.29.6 bogon <none> <none> web-nginx-75bff6b7cf-dpcvx 1/1 Running 0 8s 10.244.243.199 ubuntu <none> <none> web-nginx-75bff6b7cf-vgztc 1/1 Running 0 8s 10.244.41.5 cloud <none> <none> root@ubuntu:~#
leaf1路由
host-10-10-18-31# sh ip bgp nei 10.10.16.82 routes BGP table version is 5, local router ID is 10.10.18.31, vrf id 0 Default local pref 100, local AS 64514 Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, i internal, r RIB-failure, S Stale, R Removed Nexthop codes: @NNN nexthop‘s vrf id, < announce-nh-self Origin codes: i - IGP, e - EGP, ? - incomplete Network Next Hop Metric LocPrf Weight Path i10.244.29.0/26 10.10.16.81 100 0 64512 i i10.244.243.192/26 10.10.16.82 100 0 i Displayed 2 routes and 3 total paths host-10-10-18-31#
host-10-10-18-31# sh ip bgp nei 10.10.18.34 routes host-10-10-18-31#
leaf2路由
host-10-10-18-34# sh ip bgp nei 10.10.16.47 routes BGP table version is 3, local router ID is 10.10.18.34, vrf id 0 Default local pref 100, local AS 64513 Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, i internal, r RIB-failure, S Stale, R Removed Nexthop codes: @NNN nexthop‘s vrf id, < announce-nh-self Origin codes: i - IGP, e - EGP, ? - incomplete Network Next Hop Metric LocPrf Weight Path i10.244.2.0/24 10.10.16.47 100 0 i i10.244.41.0/26 10.10.16.47 100 0 i i10.244.129.128/26 10.10.16.251 100 0 64512 i Displayed 3 routes and 4 total paths host-10-10-18-34#
host-10-10-18-34# sh ip bgp nei 10.10.18.31 routes BGP table version is 3, local router ID is 10.10.18.34, vrf id 0 Default local pref 100, local AS 64513 Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, i internal, r RIB-failure, S Stale, R Removed Nexthop codes: @NNN nexthop‘s vrf id, < announce-nh-self Origin codes: i - IGP, e - EGP, ? - incomplete Network Next Hop Metric LocPrf Weight Path *> 31.31.31.31/32 10.10.18.31 0 0 64514 i Displayed 1 routes and 4 total paths host-10-10-18-34#
nei 10.10.18.31 没有把从ubuntu和gogon学习到的路由告诉leaf2
bgp 学习到的路由没有发送给内核
[root@host-10-10-18-34 ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.18.254 0.0.0.0 UG 100 0 0 eth0 10.10.18.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0 31.31.31.31 10.10.18.31 255.255.255.255 UGH 20 0 0 eth0 169.254.169.254 10.10.18.254 255.255.255.255 UGH 100 0 0 eth0 [root@host-10-10-18-34 ~]# ip r default via 10.10.18.254 dev eth0 proto dhcp metric 100 10.10.18.0/24 dev eth0 proto kernel scope link src 10.10.18.34 metric 100 31.31.31.31 via 10.10.18.31 dev eth0 proto bgp metric 20 169.254.169.254 via 10.10.18.254 dev eth0 proto dhcp metric 100 [root@host-10-10-18-34 ~]#
开启ip forward也没有用
[root@host-10-10-18-34 ~]# sysctl net.ipv4.ip_forward net.ipv4.ip_forward=1 net.ipv4.ip_forward = 1 net.ipv4.ip_forward = 1 [root@host-10-10-18-34 ~]# sysctl -p /etc/sysctl.conf [root@host-10-10-18-34 ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.18.254 0.0.0.0 UG 100 0 0 eth0 10.10.18.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0 31.31.31.31 10.10.18.31 255.255.255.255 UGH 20 0 0 eth0 169.254.169.254 10.10.18.254 255.255.255.255 UGH 100 0 0 eth0 [root@host-10-10-18-34 ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.18.254 0.0.0.0 UG 100 0 0 eth0 10.10.18.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0 31.31.31.31 10.10.18.31 255.255.255.255 UGH 20 0 0 eth0 169.254.169.254 10.10.18.254 255.255.255.255 UGH 100 0 0 eth0 [root@host-10-10-18-34 ~]# cat /proc/sys/net/ipv4/ip_forward 1 [root@host-10-10-18-34 ~]#
更改as
bogon 和ubutnu一致
centos7 和cloud一致
root@cloud:~# ./calicoctl patch node bogon -p ‘{"spec": {"bgp": {"asNumber": "64514"}}}‘ Successfully patched 1 ‘Node‘ resource root@cloud:~# ./calicoctl patch node centos7 -p ‘{"spec": {"bgp": {"asNumber": "64513"}}}‘ Successfully patched 1 ‘Node‘ resource root@cloud:~# ./calicoctl get node -o wide NAME ASN IPV4 IPV6 bogon 64514 10.10.16.81/24 centos7 64513 10.10.16.251/24 cloud 64513 10.10.16.47/24 ubuntu 64514 10.10.16.82/24
root@cloud:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.18.34 | node specific | up | 08:39:35 | Established | | 10.10.16.251 | node specific | up | 09:49:32 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found.
root@ubuntu:~# ./calicoctl node status Calico process is running. IPv4 BGP status +--------------+---------------+-------+----------+-------------+ | PEER ADDRESS | PEER TYPE | STATE | SINCE | INFO | +--------------+---------------+-------+----------+-------------+ | 10.10.18.31 | node specific | up | 08:53:42 | Established | | 10.10.16.81 | node specific | up | 09:49:21 | Established | +--------------+---------------+-------+----------+-------------+ IPv6 BGP status No IPv6 peers found. root@ubuntu:~#
还是没用
[root@host-10-10-18-34 ~]# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 0.0.0.0 10.10.18.254 0.0.0.0 UG 100 0 0 eth0 10.10.18.0 0.0.0.0 255.255.255.0 U 100 0 0 eth0 31.31.31.31 10.10.18.31 255.255.255.255 UGH 20 0 0 eth0 169.254.169.254 10.10.18.254 255.255.255.255 UGH 100 0 0 eth0 [root@host-10-10-18-34 ~]# vrysh -bash: vrysh: command not found [root@host-10-10-18-34 ~]# vtysh Hello, this is FRRouting (version 7.5). Copyright 1996-2005 Kunihiro Ishiguro, et al. host-10-10-18-34# sh ip bgp nei 10.10.16.47 routes BGP table version is 3, local router ID is 10.10.18.34, vrf id 0 Default local pref 100, local AS 64513 Status codes: s suppressed, d damped, h history, * valid, > best, = multipath, i internal, r RIB-failure, S Stale, R Removed Nexthop codes: @NNN nexthop‘s vrf id, < announce-nh-self Origin codes: i - IGP, e - EGP, ? - incomplete Network Next Hop Metric LocPrf Weight Path i10.244.2.0/24 10.10.16.47 100 0 i i10.244.41.0/26 10.10.16.47 100 0 i i10.244.129.128/26 10.10.16.251 100 0 i Displayed 3 routes and 4 total paths host-10-10-18-34#
BGP的同步规则分析
这是由于BGP的同步规则，在同步开启的情况下，从IBGP学到的路由是不会传递给他的EBGP邻居的，除非IGP路由信息中有该路由