本文实验如何通过ovn的l2gateway连接到外部网络。
逻辑拓扑如下
image.png
执行下面命令，创建出逻辑拓扑
//创建 logical switch ls1
ovn-nbctl ls-add ls1
//添加第一个 logical port ls1-vm1
ovn-nbctl lsp-add ls1 ls1-vm1
ovn-nbctl lsp-set-addresses ls1-vm1 00:00:00:00:00:03
ovn-nbctl lsp-set-port-security ls1-vm1 00:00:00:00:00:03
//添加第二个 logical port ls1-vm2
ovn-nbctl lsp-add ls1 ls1-vm2
ovn-nbctl lsp-set-addresses ls1-vm2 00:00:00:00:00:04
ovn-nbctl lsp-set-port-security ls1-vm2 00:00:00:00:00:04
//添加第三个 logical port ls1-l2gateway，类型为 l2gateway，用来连接外部网络
ovn-nbctl lsp-add ls1 ls1-l2gateway
ovn-nbctl lsp-set-type ls1-l2gateway l2gateway
ovn-nbctl lsp-set-addresses ls1-l2gateway unknown
ovn-nbctl lsp-set-options ls1-l2gateway network_name=externalnet l2gateway-chassis=master
//因为选项 l2gateway-chassis 指定了 chassis 为 master，所以只在master节点上执行如下命令
ovs-vsctl add-br br-ens8
ovs-vsctl add-port br-ens8 ens8
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=externalnet:br-ens8
ip link set dev br-ens8 up
ip addr add 10.10.10.4/24 dev br-ens8
//在master上创建vm1 namespace
ip netns add vm1
ovs-vsctl add-port br-int vm1 -- set interface vm1 type=internal
ip link set vm1 netns vm1
ip netns exec vm1 ip link set vm1 address 00:00:00:00:00:03
ip netns exec vm1 ip addr add 10.10.10.2/24 dev vm1
ip netns exec vm1 ip link set vm1 up
//通过iface-id=ls1-vm1和逻辑端口ls1-vm1绑定
ovs-vsctl set Interface vm1 external_ids:iface-id=ls1-vm1
//在node1上创建vm2 namespace
ip netns add vm2
ovs-vsctl add-port br-int vm2 -- set interface vm2 type=internal
ip link set vm2 netns vm2
ip netns exec vm2 ip link set vm2 address 00:00:00:00:00:04
ip netns exec vm2 ip addr add 10.10.10.3/24 dev vm2
ip netns exec vm2 ip link set vm2 up
//通过iface-id=ls1-vm2和逻辑端口ls1-vm2绑定
ovs-vsctl set Interface vm2 external_ids:iface-id=ls1-vm2
生成的物理网络拓扑如下
image.png
ens8所在的网络为外部网络。
因为l2gateway端口绑定到了master chassis，所以只在master的br-int通过patch端口和br-ens8互连，用来连接外部网络。
其他chassis上的vm如果想访问外部网络，必须先经过tunnel network发送到master上的br-int，再发给br-ens8来实现。
ping结果如下，vm之间，vm和外部网络都是可以ping通的。
root@master:~# ip netns exec vm1 ping 10.10.10.3
PING 10.10.10.3 (10.10.10.3) 56(84) bytes of data.
64 bytes from 10.10.10.3: icmp_seq=1 ttl=64 time=9.12 ms
^C
--- 10.10.10.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 9.119/9.119/9.119/0.000 ms
root@master:~# ip netns exec vm1 ping 10.10.10.4
PING 10.10.10.4 (10.10.10.4) 56(84) bytes of data.
64 bytes from 10.10.10.4: icmp_seq=1 ttl=64 time=0.695 ms
^C
--- 10.10.10.4 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.695/0.695/0.695/0.000 ms
root@master:~# ip netns exec vm1 ping 10.10.10.5
PING 10.10.10.5 (10.10.10.5) 56(84) bytes of data.
64 bytes from 10.10.10.5: icmp_seq=1 ttl=64 time=0.612 ms
^C
--- 10.10.10.5 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.612/0.612/0.612/0.000 ms
root@node1:~# ip netns exec vm2 ping 10.10.10.2
PING 10.10.10.2 (10.10.10.2) 56(84) bytes of data.
64 bytes from 10.10.10.2: icmp_seq=1 ttl=64 time=20.3 ms
^C
--- 10.10.10.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 20.302/20.302/20.302/0.000 ms
root@node1:~# ip netns exec vm2 ping 10.10.10.4
PING 10.10.10.4 (10.10.10.4) 56(84) bytes of data.
64 bytes from 10.10.10.4: icmp_seq=1 ttl=64 time=9.68 ms
^C
--- 10.10.10.4 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 9.681/9.681/9.681/0.000 ms
root@node1:~# ip netns exec vm2 ping 10.10.10.5
PING 10.10.10.5 (10.10.10.5) 56(84) bytes of data.
64 bytes from 10.10.10.5: icmp_seq=1 ttl=64 time=7.22 ms
^C
--- 10.10.10.5 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 7.217/7.217/7.217/0.000 ms
ping报文路径
vm1(10.10.10.2) ping vm2(10.10.10.3):
vm1 -> br-int(master) -> ovn-node1-0 -> ens3(master) -> ens3(node1) -> ovn-master-0 -> br-int(node1) -> vm2
vm1(10.10.10.2) ping 10.10.10.4:
vm1 -> br-int(master) -> patch -> br-ens8(master)
vm1(10.10.10.2) ping 10.10.10.5:
vm1 -> br-int(master) -> patch -> br-ens8(master) -> ens8(master) -> ens8(node1)
vm2(10.10.10.3) ping vm1(10.10.10.2):
vm2 -> br-int(node1) -> ovn-master-0 -> ens3(node1) -> ens3(master) -> ovn-node1-0 -> br-int(master) -> vm1
//因为node1上没有连接外部网络的通道，只能通过隧道发送到master，再转发到master上的br-ens8来连接外部网络
vm2(10.10.10.3) ping 10.10.10.4:
vm2 -> br-int(node1) -> ovn-master-0 -> ens3(node1) -> ens3(master) -> ovn-node1-0 -> br-int(master) -> patch -> br-ens8(master)
vm2(10.10.10.3) ping 10.10.10.5:
vm2 -> br-int(node1) -> ovn-master-0 -> ens3(node1) -> ens3(master) -> ovn-node1-0 -> br-int(master) -> patch -> br-ens8(master) -> ens8(master) -> ens8(node1)
nbdb信息查看
root@master:~# ovn-nbctl show
switch 58d5c727-ce93-47bd-8770-cf4d42d31122 (ls1)
port ls1-l2gateway
type: l2gateway
addresses: ["unknown"]
port ls1-vm2
addresses: ["00:00:00:00:00:04"]
port ls1-vm1
addresses: ["00:00:00:00:00:03"]
root@master:~# ovn-nbctl list logical_switch
_uuid : 58d5c727-ce93-47bd-8770-cf4d42d31122
acls : []
dns_records : []
external_ids : {}
forwarding_groups : []
load_balancer : []
name : ls1
other_config : {}
ports : [4c5a6083-60f5-40fe-8525-2c6d5454573e, effab4a2-30f9-42b5-8816-269462a4efbd, f99d86d1-87e2-4406-9462-403b42229075]
qos_rules : []
root@master:~# ovn-nbctl list logical_switch_port
_uuid : f99d86d1-87e2-4406-9462-403b42229075
addresses : ["00:00:00:00:00:03"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {}
ha_chassis_group : []
name : ls1-vm1
options : {}
parent_name : []
port_security : ["00:00:00:00:00:03"]
tag : []
tag_request : []
type : ""
up : true
_uuid : effab4a2-30f9-42b5-8816-269462a4efbd
addresses : ["00:00:00:00:00:04"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {}
ha_chassis_group : []
name : ls1-vm2
options : {}
parent_name : []
port_security : ["00:00:00:00:00:04"]
tag : []
tag_request : []
type : ""
up : true
_uuid : 4c5a6083-60f5-40fe-8525-2c6d5454573e
addresses : [unknown]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {}
ha_chassis_group : []
name : ls1-l2gateway
options : {l2gateway-chassis=master, network_name=externalnet}
parent_name : []
port_security : []
tag : []
tag_request : []
type : l2gateway
up : true
sbdb信息查看
root@master:~# ovn-sbctl show
Chassis node1
hostname: node1
Encap geneve
ip: "192.168.122.21"
options: {csum="true"}
Port_Binding ls1-vm2
Chassis master
hostname: master
Encap geneve
ip: "192.168.122.20"
options: {csum="true"}
Port_Binding ls1-l2gateway
Port_Binding ls1-vm1
root@master:~# ovn-sbctl list port_binding
_uuid : 1024fed9-c0af-4af7-9926-ad2d9565cbe0
chassis : 29a2b734-b27b-4dd9-b1ae-935292757377
datapath : 88d29262-a796-4e36-b36c-656d37557927
encap : []
external_ids : {}
gateway_chassis : []
ha_chassis_group : []
logical_port : ls1-vm2
mac : ["00:00:00:00:00:04"]
nat_addresses : []
options : {}
parent_port : []
tag : []
tunnel_key : 2
type : ""
up : true
virtual_parent : []
_uuid : e880c926-b0a3-4033-913c-1a57f00dbfdd
chassis : b0261728-db55-4e0b-bfd5-b930081010fc
datapath : 88d29262-a796-4e36-b36c-656d37557927
encap : []
external_ids : {}
gateway_chassis : []
ha_chassis_group : []
logical_port : ls1-l2gateway
mac : [unknown]
nat_addresses : []
options : {l2gateway-chassis=master, network_name=externalnet}
parent_port : []
tag : []
tunnel_key : 3
type : l2gateway
up : true
virtual_parent : []
_uuid : 9279eadf-5341-4385-8c32-f7af33040586
chassis : b0261728-db55-4e0b-bfd5-b930081010fc
datapath : 88d29262-a796-4e36-b36c-656d37557927
encap : []
external_ids : {}
gateway_chassis : []
ha_chassis_group : []
logical_port : ls1-vm1
mac : ["00:00:00:00:00:03"]
nat_addresses : []
options : {}
parent_port : []
tag : []
tunnel_key : 1
type : ""
up : true
virtual_parent : []
master和node1节点上的ovsdb信息查看
root@master:~# ovs-vsctl show
a891c32e-dec1-4168-8e17-1516fa55341b
Bridge br-int
fail_mode: secure
Port patch-br-int-to-ls1-l2gateway
Interface patch-br-int-to-ls1-l2gateway
type: patch
options: {peer=patch-ls1-l2gateway-to-br-int}
Port ovn-node1-0
Interface ovn-node1-0
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.21"}
Port br-int
Interface br-int
type: internal
Port vm1
Interface vm1
type: internal
Bridge br-ens8
Port patch-ls1-l2gateway-to-br-int
Interface patch-ls1-l2gateway-to-br-int
type: patch
options: {peer=patch-br-int-to-ls1-l2gateway}
Port ens8
Interface ens8
Port br-ens8
Interface br-ens8
type: internal
root@node1:~# ovs-vsctl show
c9da68e6-3d3f-49a3-b649-9f0345985648
Bridge br-int
fail_mode: secure
Port vm2
Interface vm2
type: internal
Port br-int
Interface br-int
type: internal
Port ovn-master-0
Interface ovn-master-0
type: geneve
options: {csum="true", key=flow, remote_ip="192.168.122.20"}
Bridge br-ens8
Port br-ens8
Interface br-ens8
type: internal
Port ens8
Interface ens8
