L8LiLiang

OVN - TAG

  • 2021年 06月25日

如果虚拟机里面有容器，可以通过配置子接口的方式把container接入OVN网络，
就是创建容器的lsp的时候，把VM的VIF当作父接口。
当需要向容器发送报文时，先把报文发送到VM的VIF（进而找到HV），
然后再由HV上的ovs根据目的mac为报文打上vlan tag，发送到VM里面。
也就是说，这个tag只有ovs处理，外部看不到它。
而且假如vm1中的container1使用tag2，vm2中的container2使用tag3，他们之间也是可以通信的。

install vm

在两台机器上各安装两个虚拟机

rhel_version=rhel$(rpm -E %rhel)
  
# libvirt && kvm
yum -y install virt-install
yum -y install libvirt
yum install -y python3-lxml.x86_64
rpm -qa | grep qemu-kvm >/dev/null || yum -y install qemu-kvm
if (($rhel_version < 7)); then
        service libvirtd restart
else
        systemctl restart libvirtd
        systemctl start virtlogd.socket
fi

# work around for failure of virt-install
chmod 666 /dev/kvm


# define default vnet
virsh net-define /usr/share/libvirt/networks/default.xml
virsh net-start default
virsh net-autostart default

# define vm name and mac
vm_name=v0
mac4vm=a4:a4:a4:a4:a4:a0

# download image
wget http://netqe-bj.usersys.redhat.com/share/vms/rhel8.4.qcow2 -O /var/lib/libvirt/images/$vm_name.qcow2

# install vm
virt-install \
        --name $vm_name \
        --vcpus=2 \
        --ram=2048 \
        --disk path=/var/lib/libvirt/images/$vm_name.qcow2,device=disk,bus=virtio,format=qcow2 \
        --network bridge=virbr0,model=virtio,mac=$mac4vm \
        --boot hd \
        --accelerate \
        --graphics vnc,listen=0.0.0.0 \
        --force \
        --os-type=linux \
        --noautoconsol

setup on central node

systemctl start openvswitch
systemctl start ovn-northd
ovn-sbctl set-connection ptcp:6642
ovn-nbctl set-connection ptcp:6641

ovs-vsctl set Open_vSwitch . external-ids:system-id=hv0
ovs-vsctl set Open_vSwitch . external-ids:ovn-remote=tcp:177.1.1.1:6642
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-type=geneve
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-ip=177.1.1.1
systemctl restart ovn-controller


mac_v0_vnet1=04:ac:10:ff:01:94
mac_v0_vnet2=04:ac:10:ff:01:95
mac_v1_vnet1=04:ac:10:ff:01:96
mac_v1_vnet2=04:ac:10:ff:01:97

cat <<-EOF > v0-vnet1.xml 
<interface type='bridge'>
        <target dev='h0_v0_vnet1'/>
        <mac address='${mac_v0_vnet1}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF


cat <<-EOF > v0-vnet2.xml 
<interface type='bridge'>
        <target dev='h0_v0_vnet2'/>
        <mac address='${mac_v0_vnet2}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF

cat <<-EOF > v1-vnet1.xml 
<interface type='bridge'>
        <target dev='h0_v1_vnet1'/>
        <mac address='${mac_v1_vnet1}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF

cat <<-EOF > v1-vnet2.xml 
<interface type='bridge'>
        <target dev='h0_v1_vnet2'/>
        <mac address='${mac_v1_vnet2}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF

virsh attach-device v0 v0-vnet1.xml
virsh attach-device v1 v1-vnet1.xml

sleep 2

ovs-vsctl set interface h0_v0_vnet1 external-ids:iface-id=h0_v0_vnet1
ovs-vsctl set interface h0_v1_vnet1 external-ids:iface-id=h0_v1_vnet1

setup on computing node

systemctl start openvswitch
ovs-vsctl set Open_vSwitch . external-ids:system-id=hv1
ovs-vsctl set Open_vSwitch . external-ids:ovn-remote=tcp:177.1.1.1:6642
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-type=geneve
ovs-vsctl set Open_vSwitch . external-ids:ovn-encap-ip=177.1.1.2
systemctl start ovn-controller

mac_v0_vnet1=02:ac:10:ff:01:94
mac_v0_vnet2=02:ac:10:ff:01:95
mac_v1_vnet1=02:ac:10:ff:01:96
mac_v1_vnet2=02:ac:10:ff:01:97

cat <<-EOF > v0-vnet1.xml
<interface type='bridge'>
        <target dev='h1_v0_vnet1'/>
        <mac address='${mac_v0_vnet1}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF


cat <<-EOF > v0-vnet2.xml
<interface type='bridge'>
        <target dev='h1_v0_vnet2'/>
        <mac address='${mac_v0_vnet2}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF

cat <<-EOF > v1-vnet1.xml
<interface type='bridge'>
        <target dev='h1_v1_vnet1'/>
        <mac address='${mac_v1_vnet1}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF

cat <<-EOF > v1-vnet2.xml
<interface type='bridge'>
        <target dev='h1_v1_vnet2'/>
        <mac address='${mac_v1_vnet2}'/>
        <source bridge='br-int'/>
        <virtualport type='openvswitch'/>
        <model type='virtio'/>
</interface>
EOF

virsh attach-device v0 v0-vnet2.xml
virsh attach-device v1 v1-vnet2.xml

sleep 2

ovs-vsctl set interface h1_v0_vnet2 external-ids:iface-id=h1_v0_vnet2
ovs-vsctl set interface h1_v1_vnet2 external-ids:iface-id=h1_v1_vnet2

create topo on central node

mac_h0_v0_vnet1=04:ac:10:ff:01:94
mac_h0_v0_vnet2=04:ac:10:ff:01:95
mac_h0_v1_vnet1=04:ac:10:ff:01:96
mac_h0_v1_vnet2=04:ac:10:ff:01:97
mac_h1_v0_vnet1=02:ac:10:ff:01:94
mac_h1_v0_vnet2=02:ac:10:ff:01:95
mac_h1_v1_vnet1=02:ac:10:ff:01:96
mac_h1_v1_vnet2=02:ac:10:ff:01:97

# add logical switch
ovn-nbctl ls-add ls1 -- add Logical_Switch ls1 other_config subnet=172.16.1.0/24
ovn-nbctl ls-add ls2 -- add Logical_Switch ls2 other_config subnet=172.16.2.0/24

# setup ls ipv6_prefix
ovn-nbctl set Logical-switch ls1 other_config:ipv6_prefix=2001:db8:1::0
ovn-nbctl set Logical-switch ls2 other_config:ipv6_prefix=2001:db8:2::0

# create dhcp_options
dhcp_options1=$(ovn-nbctl create DHCP_Options cidr=172.16.1.0/24 \
        options="\"server_id\"=\"172.16.1.254\" \"server_mac\"=\"00:00:00:00:01:00\" \
        \"lease_time\"=\"$((36000 + RANDOM % 3600))\" \"router\"=\"172.16.1.254\" \"dns_server\"=\"172.16.1.254\"")
dhcp_options2=$(ovn-nbctl create DHCP_Options cidr=172.16.2.0/24 \
        options="\"server_id\"=\"172.16.2.254\" \"server_mac\"=\"00:00:00:00:02:00\" \
        \"lease_time\"=\"$((36000 + RANDOM % 3600))\" \"router\"=\"172.16.2.254\" \"dns_server\"=\"172.16.2.254\"")

dhcpv6_options1=$(ovn-nbctl create DHCP_Options cidr="2001\:db8\:1\:\:0/64" \
                                options="\"server_id\"=\"00:00:00:00:01:00\" \"dns_server\"=\"2001:db8:1::254\"")
dhcpv6_options2=$(ovn-nbctl create DHCP_Options cidr="2001\:db8\:2\:\:0/64" \
                                options="\"server_id\"=\"00:00:00:00:02:00\" \"dns_server\"=\"2001:db8:2::254\"")

# create logical switch port and setup dhcp_option
lsp_name=h0_v0_vnet1
mac=$mac_h0_v0_vnet1
ovn-nbctl lsp-add ls1 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac 172.16.1.2"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options1}

lsp_name=h0_v1_vnet1
mac=$mac_h0_v1_vnet1
ovn-nbctl lsp-add ls1 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac 172.16.1.3"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options1}

lsp_name=h1_v0_vnet2
mac=$mac_h1_v0_vnet2
ovn-nbctl lsp-add ls2 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac 172.16.2.2"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options2}

lsp_name=h1_v1_vnet2
mac=$mac_h1_v1_vnet2
ovn-nbctl lsp-add ls2 $lsp_name
ovn-nbctl lsp-set-addresses $lsp_name "$mac 172.16.2.3"
ovn-nbctl lsp-set-dhcpv4-options $lsp_name ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port $lsp_name dhcpv6_options ${dhcpv6_options2}

# create logical router lr1
ovn-nbctl lr-add lr1
#ovn-nbctl create Logical_Router name=lr1 options:chassis=hv0
ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:01:00 172.16.1.254/24
ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:02:00 172.16.2.254/24

# connect lr1 and ls1
ovn-nbctl lsp-add ls1 ls1-lr1
ovn-nbctl lsp-set-type ls1-lr1 router
ovn-nbctl lsp-set-addresses ls1-lr1 "00:00:00:00:01:00 172.16.1.254"
ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1

# connect lr1 and ls2
ovn-nbctl lsp-add ls2 ls2-lr1
ovn-nbctl lsp-set-type ls2-lr1 router
ovn-nbctl lsp-set-addresses ls2-lr1 "00:00:00:00:02:00 172.16.2.254"
ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2

# create container port
ovn-nbctl lsp-add ls1 h0_v0_vnet1_vlan2 h0_v0_vnet1 2 -- lsp-set-addresses h0_v0_vnet1_vlan2 "a6:22:22:22:22:21 172.16.1.21"
ovn-nbctl lsp-set-dhcpv4-options  h0_v0_vnet1_vlan2 ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port h0_v0_vnet1_vlan2 dhcpv6_options ${dhcpv6_options1}

ovn-nbctl lsp-add ls1 h0_v1_vnet1_vlan2 h0_v1_vnet1 2 -- lsp-set-addresses h0_v1_vnet1_vlan2 "a6:22:22:22:22:22 172.16.1.22"
ovn-nbctl lsp-set-dhcpv4-options  h0_v1_vnet1_vlan2 ${dhcp_options1}
ovn-nbctl add Logical_Switch_Port h0_v1_vnet1_vlan2 dhcpv6_options ${dhcpv6_options1}

ovn-nbctl lsp-add ls2 h1_v0_vnet2_vlan3 h1_v0_vnet2 3 -- lsp-set-addresses h1_v0_vnet2_vlan3 "a8:22:22:22:22:21 172.16.2.21"
ovn-nbctl lsp-set-dhcpv4-options  h1_v0_vnet2_vlan3 ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port h1_v0_vnet2_vlan3 dhcpv6_options ${dhcpv6_options2}

ovn-nbctl lsp-add ls2 h1_v1_vnet2_vlan3 h1_v1_vnet2 3 -- lsp-set-addresses h1_v1_vnet2_vlan3 "a8:22:22:22:22:22 172.16.2.22"
ovn-nbctl lsp-set-dhcpv4-options  h1_v1_vnet2_vlan3 ${dhcp_options2}
ovn-nbctl add Logical_Switch_Port h1_v1_vnet2_vlan3 dhcpv6_options ${dhcpv6_options2}

check connectivity and vlan id

# 从其他netns ping h0_v0_vnet1_vlan2
[root@hp-dl380pg8-15 ~]# tcpdump -i h0_v0_vnet1 -enn
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on h0_v0_vnet1, link-type EN10MB (Ethernet), capture size 262144 bytes
04:07:20.391897 00:00:00:00:01:00 > a6:22:22:22:22:21, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.2.21 > 172.16.1.21: ICMP echo request, id 5338, seq 5, length 64
04:07:20.392078 a6:22:22:22:22:21 > 00:00:00:00:01:00, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.1.21 > 172.16.2.21: ICMP echo reply, id 5338, seq 5, length 64
04:07:21.393073 00:00:00:00:01:00 > a6:22:22:22:22:21, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.2.21 > 172.16.1.21: ICMP echo request, id 5338, seq 6, length 64
04:07:21.393203 a6:22:22:22:22:21 > 00:00:00:00:01:00, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.1.21 > 172.16.2.21: ICMP echo reply, id 5338, seq 6, length 64

# 从其他netns ping h0_v0_vnet1
[root@hp-dl380pg8-15 ~]# tcpdump -i h0_v0_vnet1 -enn
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on h0_v0_vnet1, link-type EN10MB (Ethernet), capture size 262144 bytes
04:06:13.025160 00:00:00:00:01:00 > 04:ac:10:ff:01:94, ethertype IPv4 (0x0800), length 98: 172.16.2.21 > 172.16.1.2: ICMP echo request, id 5336, seq 60, length 64
04:06:13.025292 04:ac:10:ff:01:94 > 00:00:00:00:01:00, ethertype IPv4 (0x0800), length 98: 172.16.1.2 > 172.16.2.21: ICMP echo reply, id 5336, seq 60, length 64
04:06:14.049161 00:00:00:00:01:00 > 04:ac:10:ff:01:94, ethertype IPv4 (0x0800), length 98: 172.16.2.21 > 172.16.1.2: ICMP echo request, id 5336, seq 61, length 64
04:06:14.049277 04:ac:10:ff:01:94 > 00:00:00:00:01:00, ethertype IPv4 (0x0800), length 98: 172.16.1.2 > 172.16.2.21: ICMP echo reply, id 5336, seq 61, length 64

# 从h0_v0_vnet1 ping 其他VM
[root@hp-dl380pg8-15 ~]# tcpdump -i h0_v0_vnet1 -enn
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on h0_v0_vnet1, link-type EN10MB (Ethernet), capture size 262144 bytes
04:00:48.537966 04:ac:10:ff:01:94 > 00:00:00:00:01:00, ethertype IPv4 (0x0800), length 98: 172.16.1.2 > 172.16.2.2: ICMP echo request, id 5349, seq 4, length 64
04:00:48.538399 00:00:00:00:01:00 > 04:ac:10:ff:01:94, ethertype IPv4 (0x0800), length 98: 172.16.2.2 > 172.16.1.2: ICMP echo reply, id 5349, seq 4, length 64
04:00:49.561963 04:ac:10:ff:01:94 > 00:00:00:00:01:00, ethertype IPv4 (0x0800), length 98: 172.16.1.2 > 172.16.2.2: ICMP echo request, id 5349, seq 5, length 64
04:00:49.562501 00:00:00:00:01:00 > 04:ac:10:ff:01:94, ethertype IPv4 (0x0800), length 98: 172.16.2.2 > 172.16.1.2: ICMP echo reply, id 5349, seq 5, length 64

# 从h0_v0_vnet1 ping 其他netns
[root@hp-dl380pg8-15 ~]# tcpdump -i h0_v0_vnet1 -enn
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on h0_v0_vnet1, link-type EN10MB (Ethernet), capture size 262144 bytes
04:01:43.629761 04:ac:10:ff:01:94 > 00:00:00:00:01:00, ethertype IPv4 (0x0800), length 98: 172.16.1.2 > 172.16.2.21: ICMP echo request, id 5363, seq 4, length 64
04:01:43.630170 00:00:00:00:01:00 > 04:ac:10:ff:01:94, ethertype IPv4 (0x0800), length 98: 172.16.2.21 > 172.16.1.2: ICMP echo reply, id 5363, seq 4, length 64
04:01:44.631403 04:ac:10:ff:01:94 > 00:00:00:00:01:00, ethertype IPv4 (0x0800), length 98: 172.16.1.2 > 172.16.2.21: ICMP echo request, id 5363, seq 5, length 64
04:01:44.631945 00:00:00:00:01:00 > 04:ac:10:ff:01:94, ethertype IPv4 (0x0800), length 98: 172.16.2.21 > 172.16.1.2: ICMP echo reply, id 5363, seq 5, length 64


# 从h0_v0_vnet1_vlan2 ping 其他VM
[root@hp-dl380pg8-15 ~]# tcpdump -i h0_v0_vnet1 -enn
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on h0_v0_vnet1, link-type EN10MB (Ethernet), capture size 262144 bytes
04:03:34.078125 a6:22:22:22:22:21 > 00:00:00:00:01:00, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.1.21 > 172.16.2.2: ICMP echo request, id 5366, seq 4, length 64
04:03:34.078570 00:00:00:00:01:00 > a6:22:22:22:22:21, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.2.2 > 172.16.1.21: ICMP echo reply, id 5366, seq 4, length 64
04:03:35.079797 a6:22:22:22:22:21 > 00:00:00:00:01:00, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.1.21 > 172.16.2.2: ICMP echo request, id 5366, seq 5, length 64
04:03:35.080217 00:00:00:00:01:00 > a6:22:22:22:22:21, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.2.2 > 172.16.1.21: ICMP echo reply, id 5366, seq 5, length 64

# 从h0_v0_vnet1_vlan2 ping 其他netns
[root@hp-dl380pg8-15 ~]# tcpdump -i h0_v0_vnet1 -enn
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on h0_v0_vnet1, link-type EN10MB (Ethernet), capture size 262144 bytes
04:02:58.264910 a6:22:22:22:22:21 > 00:00:00:00:01:00, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.1.21 > 172.16.2.21: ICMP echo request, id 5364, seq 4, length 64
04:02:58.265341 00:00:00:00:01:00 > a6:22:22:22:22:21, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.2.21 > 172.16.1.21: ICMP echo reply, id 5364, seq 4, length 64
04:02:59.288891 a6:22:22:22:22:21 > 00:00:00:00:01:00, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.1.21 > 172.16.2.21: ICMP echo request, id 5364, seq 5, length 64
04:02:59.289284 00:00:00:00:01:00 > a6:22:22:22:22:21, ethertype 802.1Q (0x8100), length 102: vlan 2, p 0, ethertype IPv4, 172.16.2.21 > 172.16.1.21: ICMP echo reply, id 5364, seq 5, length 64

# 可以看出container的vlan tag只在本地HV可见，其他HV看不见，报文由ovs发送出去之后就不携带vlan tag了。
搜索