I have a bunch of jails on a FreeBSD 15.0-RELEASE box and currently use pci-passthru of 2x 1 gigabit NICs for OPNSense as a Bhyve guest. I have 2g/2g fiber being installed tomorrow at home and the gigabit ports won't really cut it anymore.
The box as 2x 40gigabit ports on a PCIe card and it is the only available slot in the 1U case. While maintaining the jails on their VLAN-aware bridge0, how do I get vm-bhyve to pass through 2 virtio-net devices, each on seperate VLANs (one for LAN and one for WAN)?
Below is what I believe to be all of the pertinent information of my current config. I have been hacking away at this for a full day and made no progress with every method I could think of - vswitches wouldn't create, or pre-conigured taps would not attach to the guest, or packets would not go through the taps untagged/retagged properly.
The LAN and WAN transport VLANs are to be 3998 and 3999 respectively.
The host's /etc/rc.conf network config:
/vms/bob/bob.conf
The box as 2x 40gigabit ports on a PCIe card and it is the only available slot in the 1U case. While maintaining the jails on their VLAN-aware bridge0, how do I get vm-bhyve to pass through 2 virtio-net devices, each on seperate VLANs (one for LAN and one for WAN)?
Below is what I believe to be all of the pertinent information of my current config. I have been hacking away at this for a full day and made no progress with every method I could think of - vswitches wouldn't create, or pre-conigured taps would not attach to the guest, or packets would not go through the taps untagged/retagged properly.
The LAN and WAN transport VLANs are to be 3998 and 3999 respectively.
The host's /etc/rc.conf network config:
Code:
#
# Network Configuration
#
# 1GigE interfaces (disabled)
ifconfig_em0="down"
ifconfig_em1="down"
# 40GigE interfaces
ifconfig_mlxen0="-lro -tso up"
ifconfig_mlxen1="-lro -tso up"
# Build LACP lagg and vlan interface for host
cloned_interfaces="lagg0 bridge0"
ifconfig_lagg0="laggproto lacp lagghash l3,l4 laggport mlxen0 laggport mlxen1 up"
vlans_bridge0="vlan140"
create_args_vlan140="vlan 140"
# Enable VLAN filtering on bridge and add lagg0 to it
ifconfig_bridge0="vlanfilter addm lagg0 tagged 100,101,110,140,150,3998,3999 up"
# Configure host interface on VLAN 140
ifconfig_vlan140="inet 172.23.40.10/24"
ifconfig_vlan140_ipv6="inet6 fd33:58bc:59a0:2340::10/64 accept_rtadv"
defaultrouter="172.23.40.1"/vms/bob/bob.conf
Code:
loader="bhyveload"
priority="1"
cpu="3"
memory="4G"
disk0_type="nvme"
disk0_name="disk0.img"
uuid=<redacted>
passthru0="3/0/0" # LAN
passthru1="5/0/0" # Modem ifconfig
Code:
em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=4e524bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,LRO,WOL_MAGIC,VLAN_HWFILTE
R,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,HWSTATS,MEXTPG>
ether 00:25:90:24:7e:87
media: Ethernet autoselect
status: no carrier
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
em1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=4e524bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,LRO,WOL_MAGIC,VLAN_HWFILTE
R,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,HWSTATS,MEXTPG>
ether 00:25:90:24:7e:89
media: Ethernet autoselect
status: no carrier
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lo0: flags=1008049<UP,LOOPBACK,RUNNING,MULTICAST,LOWER_UP> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
inet 127.0.0.1 netmask 0xff000000
inet6 ::1 prefixlen 128
inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
mlxen0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
options=ed00bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,VLAN_HWFILTER,VLAN_HWTSO,LI
NKSTATE,RXCSUM_IPV6,TXCSUM_IPV6,HWSTATS>
ether 24:8a:07:c1:a9:80
media: Ethernet autoselect (40Gbase-CR4 <full-duplex,rxpause,txpause>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
mlxen1: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
options=ed00bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,VLAN_HWFILTER,VLAN_HWTSO,LI
NKSTATE,RXCSUM_IPV6,TXCSUM_IPV6,HWSTATS>
ether 24:8a:07:c1:a9:80
hwaddr 24:8a:07:c1:a9:81
media: Ethernet autoselect (40Gbase-CR4 <full-duplex,rxpause,txpause>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lagg0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
options=ed00bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,VLAN_HWFILTER,VLAN_HWTSO,LI
NKSTATE,RXCSUM_IPV6,TXCSUM_IPV6,HWSTATS>
ether 24:8a:07:c1:a9:80
hwaddr 00:00:00:00:00:00
laggproto lacp lagghash l3,l4
laggport: mlxen0 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
laggport: mlxen1 flags=1c<ACTIVE,COLLECTING,DISTRIBUTING>
groups: lagg
media: Ethernet autoselect
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=1008843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
options=10<VLAN_HWTAGGING>
ether 58:9c:fc:10:26:cd
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
bridge flags=1<VLANFILTER>
member: netbox_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 44 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: pgdb1_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 41 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: unifi_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 38 priority 128 path cost 2000 vlan protocol 802.1q untagged 100
member: repocache_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 35 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: lb1_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 32 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: web3_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 29 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: web1_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 26 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: redis_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 23 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: nms_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 20 priority 128 path cost 2000 vlan protocol 802.1q untagged 101
member: mdb1_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 17 priority 128 path cost 2000 vlan protocol 802.1q untagged 150
member: ns1_vnet0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 14 priority 128 path cost 2000 vlan protocol 802.1q untagged 101
member: lagg0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
port 6 priority 128 path cost 2000000 vlan protocol 802.1q tagged 100-101,110,140,150,3998-3999
groups: bridge
nd6 options=9<PERFORMNUD,IFDISABLED>
vlan140: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1496
options=0
ether 58:9c:fc:10:26:cd
inet 172.23.40.10 netmask 0xffffff00 broadcast 172.23.40.255
inet6 fe80::5a9c:fcff:fe10:26cd%vlan140 prefixlen 64 scopeid 0xa
inet6 fd33:58bc:59a0:2340::10 prefixlen 64
inet6 <redacted> prefixlen 64 autoconf pltime 600 vltime 1800
groups: vlan
vlan: 140 vlanproto: 802.1q vlanpcp: 0 parent interface: bridge0
nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
ns1_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:ns1
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:f4:03
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
mdb1_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:mdb1
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:ac:d0
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
nms_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:nms
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:af:21
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
redis_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:redis
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:de:de
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
web1_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:web1
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:d5:77
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
web3_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:web3
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:ac:ff
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lb1_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:lb1
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:31:ff
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
repocache_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:repocache
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:16:db
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
unifi_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:unifi
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:b4:68
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
pgdb1_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:pgdb1
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:7e:1b
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
netbox_vnet0: flags=1008943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST,LOWER_UP> metric 0 mtu 1500
description: jail:netbox
options=60000b<RXCSUM,TXCSUM,VLAN_MTU,RXCSUM_IPV6,TXCSUM_IPV6>
ether 58:9c:fc:10:df:3a
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>