How to use PCI passthrough in (vm-)bhyve on AMD Zen2 CPU?
I am using FreeBSD 13.0-BETA as host and OpenBSD as guest. I can try -CURRENT or other guest if that will help debugging.
BIOS/UEFI is on the latest version and shows that virtualization is enabled. Enabling SR-IOV in it doesn't have effect.
Tried reserving ppt devices with loader.conf and devctl method - no effect.
Guest without PCI passthrough works.
Does something from ACPI tables is missing on my system?
Can I use PCI passthru for internal LAN or WLAN (in M.2 slot) or I should use network controllers in PCIe slots only?
I am using FreeBSD 13.0-BETA as host and OpenBSD as guest. I can try -CURRENT or other guest if that will help debugging.
BIOS/UEFI is on the latest version and shows that virtualization is enabled. Enabling SR-IOV in it doesn't have effect.
Tried reserving ppt devices with loader.conf and devctl method - no effect.
Guest without PCI passthrough works.
Code:
# dmesg
---<<BOOT>>---
Copyright (c) 1992-2021 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 13.0-BETA1 #14 r13.0-n244485-07eed38ae2de: Wed Feb 10 23:20:58 CET 2021
root@zen-pobro:/usr/obj/usr/src/amd64.amd64/sys/GENERIC amd64
FreeBSD clang version 11.0.1 (git@github.com:llvm/llvm-project.git llvmorg-11.0.1-0-g43ff75f2c3fe)
VT(efifb): resolution 1920x1080
CPU: AMD Ryzen 7 PRO 4750G with Radeon Graphics (3593.32-MHz K8-class CPU)
Origin="AuthenticAMD" Id=0x860f01 Family=0x17 Model=0x60 Stepping=1
Features=0x178bfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,MMX,FXSR,SSE,SSE2,HTT>
Features2=0x7ed8320b<SSE3,PCLMULQDQ,MON,SSSE3,FMA,CX16,SSE4.1,SSE4.2,MOVBE,POPCNT,AESNI,XSAVE,OSXSAVE,AVX,F16C,RDRAND>
AMD Features=0x2e500800<SYSCALL,NX,MMX+,FFXSR,Page1GB,RDTSCP,LM>
AMD Features2=0x75c237ff<LAHF,CMP,SVM,ExtAPIC,CR8,ABM,SSE4A,MAS,Prefetch,OSVW,IBS,SKINIT,WDT,TCE,Topology,PCXC,PNXC,DBE,PL2I,MWAITX,ADMSKX>
Structured Extended Features=0x219c91a9<FSGSBASE,BMI1,AVX2,SMEP,BMI2,PQM,PQE,RDSEED,ADX,SMAP,CLFLUSHOPT,CLWB,SHA>
Structured Extended Features2=0x400004<UMIP,RDPID>
XSAVE Features=0xf<XSAVEOPT,XSAVEC,XINUSE,XSAVES>
AMD Extended Feature Extensions ID EBX=0x90cf757<CLZERO,IRPerf,XSaveErPtr,RDPRU,MCOMMIT,WBNOINVD,IBPB,IBRS,STIBP,PREFER_IBRS,SSBD>
SVM: NP,NRIP,VClean,AFlush,DAssist,NAsids=32768
TSC: P-state invariant, performance statistics
dmesg | egrep -i 'ivrs | ivh | amd-vi | mmu'
<nothing>
# acpidump -t | grep DMAR
acpidump: RSDT entry 20 (sig SSDT) is corrupt
acpidump -t | grep -i IVRS
acpidump: RSDT entry 20 (sig SSDT) is corrupt
dmesg | grep -i IOMMU
# pciconf -lv | grep ppt -A4
ppt0@pci0:6:0:0: class=0x020000 rev=0x05 hdr=0x00 vendor=0x10ec device=0x8125 subvendor=0x1849 subdevice=0x8168
vendor = 'Realtek Semiconductor Co., Ltd.'
device = 'RTL8125 2.5GbE Controller'
class = network
subclass = ethernet
ppt1@pci0:7:0:0: class=0x028000 rev=0x32 hdr=0x00 vendor=0x168c device=0x003e subvendor=0x1043 subdevice=0x86e0
vendor = 'Qualcomm Atheros'
device = 'QCA6174 802.11ac Wireless Network Adapter'
class = network
# pciconf -lc
...
ppt0@pci0:6:0:0: class=0x020000 rev=0x05 hdr=0x00 vendor=0x10ec device=0x8125 subvendor=0x1849 subdevice=0x8168
cap 01[40] = powerspec 3 supports D0 D1 D2 D3 current D0
cap 05[50] = MSI supports 1 message, 64 bit, vector masks
cap 10[70] = PCI-Express 2 endpoint MSI 1 max data 256(256) RO
max read 512
link x1(x1) speed 5.0(5.0) ASPM L1(L0s/L1) ClockPM disabled
cap 11[b0] = MSI-X supports 32 messages
Table in map 0x20[0x0], PBA in map 0x20[0x800]
cap 03[d0] = VPD
ecap 0001[100] = AER 2 0 fatal 0 non-fatal 0 corrected
ecap 0002[148] = VC 1 max VC0
ecap 0003[168] = Serial 1 e8922d59a1a80000
ecap 0017[178] = TPH Requester 1
ecap 0018[204] = LTR 1
ecap 001e[20c] = L1 PM Substates 1
ecap 000b[21c] = Vendor [1] ID 0002 Rev 4 Length 256
ppt1@pci0:7:0:0: class=0x028000 rev=0x32 hdr=0x00 vendor=0x168c device=0x003e subvendor=0x1043 subdevice=0x86e0
cap 01[40] = powerspec 3 supports D0 D3 current D0
cap 05[50] = MSI supports 8 messages, vector masks
cap 10[70] = PCI-Express 2 endpoint max data 256(256) RO
max read 512
link x1(x1) speed 2.5(2.5) ASPM L1(L0s/L1) ClockPM disabled
ecap 0001[100] = AER 2 0 fatal 0 non-fatal 0 corrected
ecap 0002[148] = VC 1 max VC0
ecap 0003[168] = Serial 1 0000000000000000
ecap 0018[178] = LTR 1
ecap 001e[180] = L1 PM Substates 1
# sed -e 's/#.*$//' -e '/^$/d' /boot/loader.conf
autoboot_delay="1"
opensolaris_load="YES"
zfs_load="YES"
hw.amdgpu.exp_hw_support=1
vmm_load="YES"
hw.vmm.amdvi.enable=1
pptdevs="6/0/0 7/0/0"
kern.racct.enable="1"
# sysctl hw.vmm.amdvi
hw.vmm.amdvi.domain_id: 0
hw.vmm.amdvi.disable_io_fault: 0
hw.vmm.amdvi.ptp_level: 4
hw.vmm.amdvi.host_ptp: 1
hw.vmm.amdvi.enable: 1
hw.vmm.amdvi.count: 0
# vm passthru | grep Yes
ppt0 6/0/0 Yes RTL8125 2.5GbE Controller
ppt1 7/0/0 Yes QCA6174 802.11ac Wireless Network Adapter
# vm-bhyve router.conf:
loader="grub"
cpu=1
memory=256M
network0_type="virtio-net"
network0_switch="public"
disk0_type="virtio-blk"
disk0_name="disk0.img"
grub_install0="kopenbsd -h com0 /6.8/amd64/bsd.rd"
grub_run0="kopenbsd -h com0 -r sd0a /bsd"
bhyve_options="-w"
uuid="cb0d1058-63ad-11eb-8f35-a8a1592d92e8"
network0_mac="58:9c:fc:02:0f:f9"
debug="yes"
passthru0="7/0/0"
# vm start -f router
Starting router
* found guest in /mnt/vm/router
* booting...
acpidump: RSDT entry 20 (sig SSDT) is corrupt
# cat vm-bhyve.log
...
Feb 11 19:18:56: initialising
Feb 11 19:18:56: [loader: grub]
Feb 11 19:18:56: [cpu: 1]
Feb 11 19:18:56: [memory: 256M]
Feb 11 19:18:56: [hostbridge: standard]
Feb 11 19:18:56: [com ports: com1]
Feb 11 19:18:56: [uuid: cb0d1058-63ad-11eb-8f35-a8a1592d92e8]
Feb 11 19:18:56: [utctime: yes]
Feb 11 19:18:56: [debug mode: yes]
Feb 11 19:18:56: [primary disk: disk0.img]
Feb 11 19:18:56: [primary disk dev: file]
Feb 11 19:18:56: initialising network device tap0
Feb 11 19:18:56: adding tap0 -> vm-public (public addm)
Feb 11 19:18:56: bring up tap0 -> vm-public (public addm)
Feb 11 19:18:56: booting
Feb 11 19:18:56: create file /mnt/vm/router/device.map
Feb 11 19:18:56: -> (hd0) /mnt/vm/router/disk0.img
Feb 11 19:18:56: create file /mnt/vm/router/grub.cfg
Feb 11 19:18:56: -> timeout=3
Feb 11 19:18:56: -> menuentry 'router (bhyve run)' {
Feb 11 19:18:56: -> root=hd0,1
Feb 11 19:18:56: -> kopenbsd -h com0 -r sd0a /bsd
Feb 11 19:18:56: -> }
Feb 11 19:18:56: /usr/local/sbin/grub-bhyve -S -m /mnt/vm/router/device.map -M 256M -r host -d /mnt/vm/router router
Feb 11 19:18:57: [bhyve options: -c 1 -m 256M -AHP -w -U cb0d1058-63ad-11eb-8f35-a8a1592d92e8 -u -S]
Feb 11 19:18:57: [bhyve devices: -s 0,hostbridge -s 31,lpc -s 4:0,virtio-blk,/mnt/vm/router/disk0.img -s 5:0,virtio-net,tap0,mac=58:9c:fc:02:0f:f9 -s 6:0,passthru,7/0/0 ]
Feb 11 19:18:57: [bhyve console: -l com1,stdio]
Feb 11 19:18:57: starting bhyve (run 1)
Feb 11 19:18:57: bhyve exited with status 4
Feb 11 19:18:57: destroying network device tap0
Feb 11 19:18:57: stopped
# cat bhyve.log
bhyve: PCI device at 7/0/0 is not using the ppt(4) driver
device emulation initialization error: Device busyDoes something from ACPI tables is missing on my system?
Can I use PCI passthru for internal LAN or WLAN (in M.2 slot) or I should use network controllers in PCIe slots only?
