general/other I just can't use any VM

BaronBS · Nov 2, 2022

So I just need virtualization working with a graphical setup and network.
I tried:
FreeBSD host with VBox
FreeBSD host with Bhyve
VM-Bhyve

Long history short: with VBox the vm's don't even boot, just a black screen. And with bhyve (any setup), everything works fine beside network.
I'm using a T430 with wifi setup. Altough I use a lagg interface, I currently don't have access to a wired network.

Here is my /boot/loader.conf

Code:

aesni_load="YES"
geom_eli_load="YES"
security.bsd.allow_destructive_dtrace=0
kern.geom.label.disk_ident.enable="0"
kern.geom.label.gptid.enable="0"
cryptodev_load="YES"
zfs_load="YES"
###########################################
# Boot
boot_mute="YES"
autoboot_delay="3"
vesa_load="YES"
#splash_pcx_load="YES"
#bitmap_load="YES"
#bitmap_name="/boot/freebsd-boot-alt.pcx"
rc_startmsgs="NO"
kern.vty="vt"
## Add colors to vt(4)
kern.vt.color.0.rgb="#000000"
kern.vt.color.1.rgb="#e67e80"
kern.vt.color.2.rgb="#a7c080"
kern.vt.color.3.rgb="#dbbc7f"
kern.vt.color.4.rgb="#7fbbb3"
kern.vt.color.5.rgb="#d699b6"
kern.vt.color.6.rgb="#83c092"
kern.vt.color.7.rgb="#ff0000"
kern.vt.color.8.rgb="#4b565c"
kern.vt.color.9.rgb="#e67e80"
kern.vt.color.10.rgb="#a7c080"
kern.vt.color.11.rgb="#dbbc7f"
kern.vt.color.12.rgb="#7fbbb3"
kern.vt.color.13.rgb="#d699b6"
kern.vt.color.14.rgb="#83c092"
kern.vt.color.15.rgb="#ff0000"

###########################################
# Security
## Meltdown mitigation
vm.pmap.pti=1
hw.ibrs_disable=0
###########################################
# Webcamd
cuse_load="YES"
###########################################
# Enhance shared memory X11 interface and
# other stuffs
kern.ipc.shmmax=67108864
kern.ipc.shmall=32768
kern.ipc.shmseg=1024
kern.ipc.shmmni=1024
kern.maxproc=1000000
###########################################
# Trunk Network
if_lagg_load="YES"
###########################################
# VirtualBox
vboxdrv_load="YES"
## If you are using AIO, then increase
## these limits (PR#168298):
vfs.aio.max_buf_aio=8192
vfs.aio.max_aio_queue_per_proc=65536
vfs.aio.max_aio_per_proc=8192
vfs.aio.max_aio_queue=65536
##########################################
# Sysctl
sysctlinfo_load="YES"
sysctlbyname_load="YES"
##########################################
# Misc
libiconv_load="YES"
libmchain_load="YES"

/etc/sysctl.conf

Code:

# $FreeBSD$
#
#  This file is read when going to multi-user and its contents piped thru
#  ``sysctl'' to adjust kernel values.  ``man 5 sysctl.conf'' for details.
#

# Security
## Uncomment this to prevent users from seeing information about processes that
## are being run under another UID.
security.bsd.see_other_uids=0
security.bsd.see_other_gids=0
security.bsd.see_jail_proc=0
security.bsd.unprivileged_read_msgbuf=0
security.bsd.unprivileged_proc_debug=0
## Kernel harderning
kern.randompid=1
kern.elf32.aslr.enable=1
kern.elf32.aslr.pie_enable=1
kern.elf32.aslr.honor_sbrk=0
kern.elf64.aslr.enable=1
kern.elf64.aslr.pie_enable=1
kern.elf64.aslr.honor_sbrk=0
vfs.zfs.min_auto_ashift=12
###############################################
# Boost performance
kern.sched.preempt_thresh=224
kern.ipc.shm_allow_removed=1
kern.ipc.shmmax=67108864
kern.ipc.shmall=32768
kern.threads.max_threads_per_proc=4096
#kern.sched.slice=5
kern.coredump=0
###############################################
# Network
net.local.stream.recvspace=65536
net.local.stream.sendspace=65536
###############################################
# Suspend on lid close
hw.acpi.lid_switch_state=S3

/etc/rc.conf

Code:

clear_tmp_enable="YES"
syslogd_flags="-ss"
sendmail_enable="NONE"
hostname="FreeBaSeD-T430"
ifconfig_em0="DHCP"
sshd_enable="YES"
moused_enable="YES"
ntpd_enable="YES"
powerd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
zfs_enable="YES"
kld_list="i915kms linux linux64 fusefs coretemp sysctlinfo sysctlbyname_improved"
################################################################
# Boot
#allscreens_flags"-r lightred black"
################################################################
# Firewall
firewall_enable="YES"
firewall_quiet="YES"
firewall_type="workstation"
firewall_logdeny="YES"
firewall_myservises="80/tcp 443/tcp 22/tcp"
firewall_allowservises="any"
#################################################################
# Network
## Trunk
ifconfig_em0="up"
wlans_iwn0="wlan0"
ifconfig_wlan0="WPA"
create_args_wlan0="wlanaddr 00:21:cc:d9:fd:75 up"
cloned_interfaces="lagg0"
ifconfig_lagg0="up laggproto failover laggport em0 laggport wlan0 DHCP"
#ifconfig_lagg0_ipv6="inet6 accept_rtadv"
rtsold_enable="YES"
## VPN
#openvpn_enable="YES"
#################################################################
# DBUS
dbus_enable="YES"
#################################################################
# Webcamd
webcamd_enable="YES"
#################################################################
# Enabling drive monitoring
#smartd_enable="YES"
#################################################################
# Enabling Microcode updates
microcode_update_enable="YES"
acpi_ibm_load="YES"
#################################################################
# mixertui stuff
sysctlinfo_load="YES"
sysctlbyname_improved_load="YES"
#################################################################
# VM-Bhyve
vm_enable="YES"
vm_dir="zfs:zroot/vm"
ifconfig_vlan0="DHCP"
# VirtualBox
vboxnet_enable="YES"
#################################################################
# Loading /etc/devfs.rules
devfs_system_ruleset="system"
ifconfig_vboxnet0="DHCP"
#################################################################
# Linux Compat
linux_enable="YES"

scottro · Nov 2, 2022

If using VirtualBox, try, before starting a VirtualBox VM

Code:

kldunload vmm

The vmm is the module that autoloads if you're using bhyve (or maybe just vm-bhyve) and it stops a VirtualBox VM from working properly.

BaronBS · Nov 2, 2022

scottro said:
If using VirtualBox, try, before starting a VirtualBox VM

Code:

kldunload vmm

The vmm is the module that autoloads if you're using bhyve (or maybe just vm-bhyve) and it stops a VirtualBox VM from working properly.

This solved the graphical problem, but the performance it is horrible (worst then vm-bhyve) and again no network.

SirDice · Nov 2, 2022

Yes, you can't use Virtualbox and byhve at the same time. You will need to unload the bhyve kernel modules when you want to use Virtualbox and vice versa.

BaronBS · Nov 2, 2022

SirDice said:
Yes, you can't use Virtualbox and byhve at the same time. You will need to unload the bhyve kernel modules when you want to use Virtualbox and vice versa.

I don't want to use at the same time, I would prefer to use Bhyve, but any one working it is fine.

SirDice · Nov 2, 2022

BobSlacker said:
I would prefer to use Bhyve, but any one working it is fine.

Then remove the Virtualbox modules and services from loader.conf and rc.conf so this isn't being loaded.

Phishfry · Nov 2, 2022

BobSlacker said:
I just need virtualization working with a graphical setup and network

FreeBSD graphical or other OS?

BaronBS · Nov 2, 2022

Phishfry said:
FreeBSD graphical or other OS?

Linux and BSD's.

Phishfry · Nov 2, 2022

Everything you want to do is possible. My bhyve graphical desktops were FreeBSD Xorg instances.
I passed thru video cards and dedicated USB3 cards for keyboard and mouse.
Apparently not all video cards work but I used GT7xx Nvidia and it worked.
You will need to get a grip on PCIe bus addressees for passthru.
Also consider passing thru a network card too. That is ideal in my opinion. No worries about bridges and taps.

BaronBS · Nov 2, 2022

Phishfry said:
Everything you want to do is possible. My bhyve graphical desktops were FreeBSD Xorg instances.
I passed thru video cards and dedicated USB3 cards for keyboard and mouse.
Apparently not all video cards work but I used GT7xx Nvidia and it worked.
You will need to get a grip on PCIe bus addressees for passthru.
Also consider passing thru a network card too. That is ideal in my opinion. No worries about bridges and taps.

Quoting from the wiki:

The PCI device at bus/slot/function 2/0/0 would be masked from the host with:

This means that the host won't be able to use that bus/slot/function anymore?

jardows · Nov 2, 2022

BobSlacker said:
Quoting from the wiki:

This means that the host won't be able to use that bus/slot/function anymore?

You are correct.

I have followed Klara System's tutorial for setting up bhyve VM's using sysutils/vm-bhyve, and it's been pretty successful. I haven't tried doing any PCI passthrough yet (well, I did try by putting a second GPU in my computer, but that crashed things badly that had nothing to do with bhyve), and I haven't tried on my laptop using a wi-fi connection yet for networking. I may give it a go there (because that does have a secondary GPU installed that I'd love to pass through) and report back any findings.

BaronBS · Nov 2, 2022

jardows said:
You are correct.

Fck, so pass thru isn't a option. Back to taps and bridges.
I will try BVCP for now,

Phishfry · Nov 2, 2022

BobSlacker said:
This means that the host won't be able to use that bus/slot/function anymore?

This somewhat depends on the card in use.
For example a 4 port network card will have 4 pci addresses.
So you could dedicate 2 ports to host and two to VM's.
There are also USB Controllers with multiple controller PCI addresses that work great for virtualization..
Video cards only offer one usable video card PCI address. (plus HDMI audio)

BaronBS · Nov 2, 2022

Phishfry said:
This somewhat depends on the card in use.
For example a 4 port network card will have 4 pci addresses.
So you could dedicate 2 ports to host and two to VM's.
There are also USB Controllers with multiple controller PCI addresses that work great for virtualization..
Video cards only offer one usable video card PCI address. (plus HDMI audio)

I'm using a T430 with a em0(4) (Intel(R) 82579LM) and a iwn(4) (Intel Centrino Advanced-N 6205).

BaronBS · Nov 13, 2022

BobSlacker said:
I will try BVCP for now,

That didn't work either. Ubuntu boots, but again, no network.

elgrande · Nov 13, 2022

BobSlacker said:
That didn't work either. Ubuntu boots, but again, no network.

NAT may be required for wireless network devices:

NAT Configuration

Shell based, minimal dependency bhyve manager. Contribute to churchers/vm-bhyve development by creating an account on GitHub.

github.com

BaronBS · Nov 13, 2022

elgrande said:
NAT may be required for wireless network devices:

NAT Configuration

Shell based, minimal dependency bhyve manager. Contribute to churchers/vm-bhyve development by creating an account on GitHub.

github.com

That didn't work either. T.T

elgrande · Nov 13, 2022

Maybe you want to post your bhyve config and you Ubuntu guest network config...

BaronBS · Nov 13, 2022

elgrande said:
Maybe you want to post your bhyve config and you Ubuntu guest network config...

Code:

{beastie} @ FreeBaSeD-T430 > /home/beastie
 → cat /boot/loader.conf                                    0 18:39:24 22-11-13
aesni_load="YES"
geom_eli_load="YES"
security.bsd.allow_destructive_dtrace=0
kern.geom.label.disk_ident.enable="0"
kern.geom.label.gptid.enable="0"
cryptodev_load="YES"
zfs_load="YES"
###########################################
# Boot
boot_mute="YES"
autoboot_delay="5"
vesa_load="YES"
#splash_pcx_load="YES"
#bitmap_load="YES"
#bitmap_name="/boot/freebsd-boot-alt.pcx"
rc_startmsgs="NO"
kern.vty="vt"
## Add colors to vt(4)
kern.vt.color.0.rgb="#000000"
kern.vt.color.1.rgb="#e67e80"
kern.vt.color.2.rgb="#a7c080"
kern.vt.color.3.rgb="#dbbc7f"
kern.vt.color.4.rgb="#7fbbb3"
kern.vt.color.5.rgb="#d699b6"
kern.vt.color.6.rgb="#83c092"
kern.vt.color.7.rgb="#ff0000"
kern.vt.color.8.rgb="#4b565c"
kern.vt.color.9.rgb="#e67e80"
kern.vt.color.10.rgb="#a7c080"
kern.vt.color.11.rgb="#dbbc7f"
kern.vt.color.12.rgb="#7fbbb3"
kern.vt.color.13.rgb="#d699b6"
kern.vt.color.14.rgb="#83c092"
kern.vt.color.15.rgb="#ff0000"
##########################################
# Security
## Meltdown mitigation
vm.pmap.pti=1
hw.ibrs_disable=0
###########################################
# Webcamd
cuse_load="YES"
###########################################
# Enhance shared memory X11 interface and
# other stuffs
kern.ipc.shmmax=67108864
kern.ipc.shmall=32768
kern.ipc.shmseg=1024
kern.ipc.shmmni=1024
kern.maxproc=1000000
###########################################
# Trunk Network
if_lagg_load="YES"
###########################################
# VirtualBox
#vboxdrv_load="YES"
## If you are using AIO, then increase
## these limits (PR#168298):
vfs.aio.max_buf_aio=8192
vfs.aio.max_aio_queue_per_proc=65536
vfs.aio.max_aio_per_proc=8192
vfs.aio.max_aio_queue=65536
##########################################
# Sysctl
sysctlinfo_load="YES"
sysctlbyname_load="YES"
##########################################
# Misc
libiconv_load="YES"
libmchain_load="YES"
##########################################
# Kali linux boot entry
chain_disk="disk1:"

Code:

{beastie} @ FreeBaSeD-T430 > /home/beastie
 → cat /etc/sysctl.conf                                     0 18:41:51 22-11-13
# $FreeBSD$
#
#  This file is read when going to multi-user and its contents piped thru
#  ``sysctl'' to adjust kernel values.  ``man 5 sysctl.conf'' for details.
#

# Security
## Uncomment this to prevent users from seeing information about processes that
## are being run under another UID.
security.bsd.see_other_uids=0
security.bsd.see_other_gids=0
security.bsd.see_jail_proc=0
security.bsd.unprivileged_read_msgbuf=0
security.bsd.unprivileged_proc_debug=0
## Kernel harderning
kern.randompid=1
kern.elf32.aslr.enable=1
kern.elf32.aslr.pie_enable=1
kern.elf32.aslr.honor_sbrk=0
kern.elf64.aslr.enable=1
kern.elf64.aslr.pie_enable=1
kern.elf64.aslr.honor_sbrk=0
vfs.zfs.min_auto_ashift=12
###############################################
# Boost performance
kern.sched.preempt_thresh=224
kern.ipc.shm_allow_removed=1
kern.ipc.shmmax=67108864
kern.ipc.shmall=32768
kern.threads.max_threads_per_proc=4096
#kern.sched.slice=5
kern.coredump=0
###############################################
# Network
net.local.stream.recvspace=65536
net.local.stream.sendspace=65536
net.inet.ip.forwarding=1
###############################################
# Suspend on lid close
hw.acpi.lid_switch_state=S3

Code:

{beastie} @ FreeBaSeD-T430 > /home/beastie
 → cat /etc/rc.conf                                         0 18:44:10 22-11-13
clear_tmp_enable="YES"
syslogd_flags="-ss"
sendmail_enable="NONE"
hostname="FreeBaSeD-T430"
ifconfig_em0="DHCP"
sshd_enable="YES"
moused_enable="YES"
ntpd_enable="YES"
powerd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
zfs_enable="YES"
zfskeys_enable="YES"
kld_list="i915kms linux linux64 fusefs coretemp sysctlinfo sysctlbyname_improved"
################################################################
# Boot
#allscreens_flags"-r lightred black"
################################################################
# Firewall
## IPFW
firewall_enable="YES"
firewall_quiet="YES"
firewall_type="workstation"
firewall_logdeny="YES"
firewall_myservises="80/tcp 443/tcp 22/tcp"
firewall_allowservises="any"
## PF
pf_enable="yes"
#################################################################
# Network
## Trunk
ifconfig_em0="up"
wlans_iwn0="wlan0"
ifconfig_wlan0="WPA"
create_args_wlan0="wlanaddr 00:21:cc:d9:fd:75 up"
cloned_interfaces="lagg0"
ifconfig_lagg0="up laggproto failover laggport em0 laggport wlan0 DHCP"
#ifconfig_lagg0_ipv6="inet6 accept_rtadv"
rtsold_enable="YES"
## Nat Network
gateway_enable="YES"
## VPN
#openvpn_enable="YES"
## DNSMASQ
dnsmasq_enable="YES"
#################################################################
# DBUS
dbus_enable="YES"
#################################################################
# Webcamd
webcamd_enable="YES"
#################################################################
# Enabling drive monitoring
smartd_enable="YES"
#################################################################
# Enabling Microcode updates
microcode_update_enable="YES"
acpi_ibm_load="YES"
#################################################################
# mixertui stuff
sysctlinfo_load="YES"
sysctlbyname_improved_load="YES"
#################################################################
# VM-Bhyve
vm_enable="YES"
vm_dir="zfs:zroot/vm"
ifconfig_vlan0="DHCP"
# VirtualBox
#vboxnet_enable="YES"
#ifconfig_vboxnet0="DHCP"
#################################################################
# Loading /etc/devfs.rules
devfs_system_ruleset="system"
#################################################################
# Linux Compat
linux_enable="YES"
#################################################################
# GELI
geli_devices="ada1"
geli_ada1_flags="-k /root/ada1.key"

Code:

{beastie} @ FreeBaSeD-T430 > /home/beastie
 → cat /etc/pf.conf                                         0 18:44:48 22-11-13
nat on lagg0 from {192.168.8.0/24} to any -> (lagg0)

Anthing else? (honest question)

elgrande · Nov 13, 2022

Looks not too wrong at a first glance.
I think with NAT, DHCP in the guest does not work, you need to configure the network statically in the Ubuntu network options.
I'd try to use 192.168.8.1 as gateway and e.g. 192.168.8.2 as IP for Ubuntu guest. For the DNS just try with 8.8.8.8 or any public DNS you like for a first try.

If that does not work:
What is the output of ifconfig in the host?
And the output of ip a show in the guest?

PMc · Nov 15, 2022

BobSlacker said:
Fck, so pass thru isn't a option. Back to taps and bridges.

bhyve can now attach to netgraph directly. It might suffice (didn't yet try) to attach to the lower hook of the physif, with something like virtio-net,netgraph,path=igb0:,peerhook=lower

general/other I just can't use any VM

BaronBS

scottro

BaronBS

SirDice

Administrator

BaronBS

SirDice

Administrator

Phishfry

BaronBS

Phishfry

BaronBS

jardows

BaronBS

Phishfry

BaronBS

BaronBS

elgrande

NAT Configuration

BaronBS

NAT Configuration

elgrande

BaronBS

elgrande

PMc