topo
+-----------------------------------------------------+ +-----------------------------------------------------+
|ls1 | |ls2 |
| | | |
|h0_v0_vnet1, h0_v1_vnet1, h1_v0_vnet1, h1_v1_vnet1| |h0_v0_vnet2, h0_v1_vnet2, h1_v0_vnet2, h1_v1_vnet2|
+-----+-------------+-------------+-------------+-----+ +-----+-------------+-------------+-------------+-----+
| | | | | | | |
| +--------o-------------o-------------o----------------+ | | |
| | | | | | | |
| | | | +--------o------------------------------+ | |
| | | | | | | |
| | | | | | +----------------------+ |
| | | | | | | |
| | | | | | | +-------+
| | | | | | | |
| | | | | +---------------------o--------------------+ |
| | | +----o------------------------+ | | |
| | +----------+ | | | | |
| | | | | | | |
| | | | | | | |
+-----o----o-------------------o-------o--------------+ +----o-----o--------------------o-------o--------------+
| | | | | | | | | | | |
| +--+----+----+ +-+-------+--+ | | +-+-----+----+ +-+-------+--+ |
| |v0 | |v1 | | | |v0 | |v1 | |
| +------------+ +------------+ | | +------------+ +------------+ |
| | | |
|h0(server+computing) | |h1(computing) |
+-----------------------------------------------------+ +------------------------------------------------------+
Topo有点乱
使用脚本安装虚拟机
在h0和h1上面各安装两个虚拟机v0和v1。
rhel_version=rhel$(rpm -E %rhel)
# libvirt && kvm
yum -y install virt-install
yum -y install libvirt
yum install -y python3-lxml.x86_64
rpm -qa | grep qemu-kvm >/dev/null || yum -y install qemu-kvm
if (($rhel_version < 7)); then
service libvirtd restart
else
systemctl restart libvirtd
systemctl start virtlogd.socket
fi
# work around for failure of virt-install
chmod 666 /dev/kvm
# define default vnet
virsh net-define /usr/share/libvirt/networks/default.xml
virsh net-start default
virsh net-autostart default
# define vm name and mac
vm_name=v1
mac4vm=a4:a4:a4:a4:a4:a1
# download image
wget http://netqe-bj.usersys.redhat.com/share/vms/rhel8.4.qcow2 -O /var/lib/libvirt/images/$vm_name.qcow2
# install vm
virt-install \
--name $vm_name \
--vcpus=2 \
--ram=2048 \
--disk path=/var/lib/libvirt/images/$vm_name.qcow2,device=disk,bus=virtio,format=qcow2 \
--network bridge=virbr0,model=virtio,mac=$mac4vm \
--boot hd \
--accelerate \
--graphics vnc,listen=0.0.0.0 \
--force \
--os-type=linux \
--noautoconsol
setup on computing node
这里的步骤包括启动ovn、把网卡绑定到虚拟机
systemctl start openvswitch
ovs-vsctl set Open_vSwitch . external-ids:ovn-remote=tcp:177.1.1.1:6642
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-type=geneve
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-ip=177.1.1.2
systemctl start ovn-controller
mac_v0_vnet1=02:ac:10:ff:01:94
mac_v0_vnet2=02:ac:10:ff:01:95
mac_v1_vnet1=02:ac:10:ff:01:96
mac_v1_vnet2=02:ac:10:ff:01:97
cat <<-EOF > v0-vnet1.xml
<interface type='bridge'>
<target dev='h1_v0_vnet1'/>
<mac address='${mac_v0_vnet1}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
cat <<-EOF > v0-vnet2.xml
<interface type='bridge'>
<target dev='h1_v0_vnet2'/>
<mac address='${mac_v0_vnet2}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
cat <<-EOF > v1-vnet1.xml
<interface type='bridge'>
<target dev='h1_v1_vnet1'/>
<mac address='${mac_v1_vnet1}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
cat <<-EOF > v1-vnet2.xml
<interface type='bridge'>
<target dev='h1_v1_vnet2'/>
<mac address='${mac_v1_vnet2}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
virsh attach-device v0 v0-vnet1.xml
virsh attach-device v0 v0-vnet2.xml
virsh attach-device v1 v1-vnet1.xml
virsh attach-device v1 v1-vnet2.xml
ovs-vsctl set interface h1_v0_vnet1 external-ids:iface-id=h1_v0_vnet1
ovs-vsctl set interface h1_v0_vnet2 external-ids:iface-id=h1_v0_vnet2
ovs-vsctl set interface h1_v1_vnet1 external-ids:iface-id=h1_v1_vnet1
ovs-vsctl set interface h1_v1_vnet2 external-ids:iface-id=h1_v1_vnet2
setup on server node
这里的步骤包括启动ovn、把网卡绑定到虚拟机
systemctl start openvswitch
systemctl start ovn-northd
ovn-sbctl set-connection ptcp:6642
ovn-nbctl set-connection ptcp:6641
ovs-vsctl set Open_vSwitch . external-ids:ovn-remote=tcp:177.1.1.1:6642
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-type=geneve
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-ip=177.1.1.1
systemctl restart ovn-controller
mac_v0_vnet1=04:ac:10:ff:01:94
mac_v0_vnet2=04:ac:10:ff:01:95
mac_v1_vnet1=04:ac:10:ff:01:96
mac_v1_vnet2=04:ac:10:ff:01:97
cat <<-EOF > v0-vnet1.xml
<interface type='bridge'>
<target dev='h0_v0_vnet1'/>
<mac address='${mac_v0_vnet1}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
cat <<-EOF > v0-vnet2.xml
<interface type='bridge'>
<target dev='h0_v0_vnet2'/>
<mac address='${mac_v0_vnet2}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
cat <<-EOF > v1-vnet1.xml
<interface type='bridge'>
<target dev='h0_v1_vnet1'/>
<mac address='${mac_v1_vnet1}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
cat <<-EOF > v1-vnet2.xml
<interface type='bridge'>
<target dev='h0_v1_vnet2'/>
<mac address='${mac_v1_vnet2}'/>
<source bridge='br-int'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
</interface>
EOF
virsh attach-device v0 v0-vnet1.xml
virsh attach-device v0 v0-vnet2.xml
virsh attach-device v1 v1-vnet1.xml
virsh attach-device v1 v1-vnet2.xml
ovs-vsctl set interface h0_v0_vnet1 external-ids:iface-id=h0_v0_vnet1
ovs-vsctl set interface h0_v0_vnet2 external-ids:iface-id=h0_v0_vnet2
ovs-vsctl set interface h0_v1_vnet1 external-ids:iface-id=h0_v1_vnet1
ovs-vsctl set interface h0_v1_vnet2 external-ids:iface-id=h0_v1_vnet2
在server node创建ovn topo
这步用到的mac要与上面两步中的mac一致
mac_h0_v0_vnet1=04:ac:10:ff:01:94
mac_h0_v0_vnet2=04:ac:10:ff:01:95
mac_h0_v1_vnet1=04:ac:10:ff:01:96
mac_h0_v1_vnet2=04:ac:10:ff:01:97
mac_h1_v0_vnet1=02:ac:10:ff:01:94
mac_h1_v0_vnet2=02:ac:10:ff:01:95
mac_h1_v1_vnet1=02:ac:10:ff:01:96
mac_h1_v1_vnet2=02:ac:10:ff:01:97
# add logical switch
ovn-nbctl ls-add ls1 -- add Logical_Switch ls1 other_config subnet=172.16.1.0/24
ovn-nbctl ls-add ls2 -- add Logical_Switch ls2 other_config subnet=172.16.2.0/24
# setup ls ipv6_prefix
ovn-nbctl set Logical-switch ls1 other_config:ipv6_prefix=2001:db8:1::0
ovn-nbctl set Logical-switch ls2 other_config:ipv6_prefix=2001:db8:2::0
# create dhcp_options
dhcp_options1=$(ovn-nbctl create DHCP_Options cidr=172.16.1.0/24 \
options="\"server_id\"=\"172.16.1.254\" \"server_mac\"=\"00:00:00:00:01:00\" \
\"lease_time\"=\"$((36000 + RANDOM % 3600))\" \"router\"=\"172.16.1.254\" \"dns_server\"=\"172.16.1.254\"")
dhcp_options2=$(ovn-nbctl create DHCP_Options cidr=172.16.2.0/24 \
options="\"server_id\"=\"172.16.2.254\" \"server_mac\"=\"00:00:00:00:02:00\" \
\"lease_time\"=\"$((36000 + RANDOM % 3600))\" \"router\"=\"172.16.2.254\" \"dns_server\"=\"172.16.2.254\"")
dhcpv6_options1=$(ovn-nbctl create DHCP_Options cidr="2001\:db8\:1\:\:0/64" \
options="\"server_id\"=\"00:00:00:00:01:00\" \"dns_server\"=\"2001:db8:1::254\"")
dhcpv6_options2=$(ovn-nbctl create DHCP_Options cidr="2001\:db8\:2\:\:0/64" \
options="\"server_id\"=\"00:00:00:00:02:00\" \"dns_server\"=\"2001:db8:2::254\"")
# create logical switch port and setup dhcp_option
lsp_name=h0_v0_vnet1
mac=$mac_h0_v0_vnet1
ovn-nbctl lsp-add ls1 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options1}
lsp_name=h0_v0_vnet2
mac=$mac_h0_v0_vnet2
ovn-nbctl lsp-add ls2 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options2}
lsp_name=h0_v1_vnet1
mac=$mac_h0_v1_vnet1
ovn-nbctl lsp-add ls1 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options1}
lsp_name=h0_v1_vnet2
mac=$mac_h0_v1_vnet2
ovn-nbctl lsp-add ls2 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options2}
lsp_name=h1_v0_vnet1
mac=$mac_h1_v0_vnet1
ovn-nbctl lsp-add ls1 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options1}
lsp_name=h1_v0_vnet2
mac=$mac_h1_v0_vnet2
ovn-nbctl lsp-add ls2 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options2}
lsp_name=h1_v1_vnet1
mac=$mac_h1_v1_vnet1
ovn-nbctl lsp-add ls1 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options1}
lsp_name=h1_v1_vnet2
mac=$mac_h1_v1_vnet2
ovn-nbctl lsp-add ls2 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac dynamic"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options2}
bridged logical switch VS overlay logical switch
上面创建的ls都是overlay logical switch，
对于overlay logical switch，跨chassis的通信走tunnel。
如果为ls创建了一个localnet类型的lsp，那么这个ls就成为bridged logical switch。
对于bridged logical switch，跨chassis的通信走物理网络。
# execute on server node
# setup bridged logical switch instead of overlay logical switch
ovn-nbctl lsp-add ls2 ls2-localnet
ovn-nbctl lsp-set-addresses ls2-localnet unknown
ovn-nbctl lsp-set-type ls2-localnet localnet
ovn-nbctl lsp-set-options ls2-localnet network_name=outNet
# execute on all computing node 每个chassis上都要把某个ovs bridge映射到localnet `outNet`
# node1
ovs-vsctl add-br br-out
ovs-vsctl add-port br-out ens2f1
ip link set ens2f1 up
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=outNet:br-out
# node2
ovs-vsctl add-br br-out
ovs-vsctl add-port br-out ens3f1np1
ip link set ens3f1np1 up
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=outNet:br-out
# 上面的命令把ls2变成了bridged logical switch，现在经过ls2转发并且跨chassis的通信都走物理网络了。
[root@hp-dl380pg8-15 ~]# tcpdump -i ens2f1 -enn icmp
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ens2f1, link-type EN10MB (Ethernet), capture size 262144 bytes
23:06:23.858617 02:ac:10:ff:01:95 > 04:ac:10:ff:01:95, ethertype IPv4 (0x0800), length 98: 172.16.2.4 > 172.16.2.2: ICMP echo request, id 7890, seq 23, length 64
23:06:23.858803 04:ac:10:ff:01:95 > 02:ac:10:ff:01:95, ethertype IPv4 (0x0800), length 98: 172.16.2.2 > 172.16.2.4: ICMP echo reply, id 7890, seq 23, length 64
23:06:24.882661 02:ac:10:ff:01:95 > 04:ac:10:ff:01:95, ethertype IPv4 (0x0800), length 98: 172.16.2.4 > 172.16.2.2: ICMP echo request, id 7890, seq 24, length 64
23:06:24.882857 04:ac:10:ff:01:95 > 02:ac:10:ff:01:95, ethertype IPv4 (0x0800), length 98: 172.16.2.2 > 172.16.2.4: ICMP echo reply, id 7890, seq 24, length 64
23:06:25.906680 02:ac:10:ff:01:95 > 04:ac:10:ff:01:95, ethertype IPv4 (0x0800), length 98: 172.16.2.4 > 172.16.2.2: ICMP echo request, id 7890, seq 25, length 64
23:06:25.906884 04:ac:10:ff:01:95 > 02:ac:10:ff:01:95, ethertype IPv4 (0x0800), length 98: 172.16.2.2 > 172.16.2.4: ICMP echo reply, id 7890, seq 25, length 64
# 但是经过ls1转发的就仍然走tunnel
# bridged logical switch同时也让内部逻辑网络能够访问物理网络
localnet VS l2gateway
使用localnet的方式需要在所有node上面配置mapping，然后每个node访问外网时都从本地的物理连接出去。
使用l2gateway的方式只需要在一个node上面配置mapping，并且指定l2gateway-chassis为该node，然后其他node访问外网时，
要先通过tunnel发送到l2gateway-chassis，再由该chassis上面的物理网卡发送出去。
# on central node(server node)
ovn-nbctl lsp-add ls2 ls2-gateway
ovn-nbctl lsp-set-addresses ls2-gateway unknown
ovn-nbctl lsp-set-type ls2-gateway l2gateway
ovn-nbctl lsp-set-options ls2-gateway network_name=outNet l2gateway-chassis=dfe51fb0-0c76-437c-ad95-709b98998b4f
# on l2gateway-chassis
ovs-vsctl add-br br-out
ovs-vsctl add-port br-out ens2f1
ip link set ens2f1 up
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=outNet:br-out