This is part III of my server build.. it features a 4 port Ethernet card. The purpose of this was to install a tiny ubuntu vm to run my UPS monitoring software and send alerts. I figured it would make a good example of network separation with jails and vm's.
for scope I have segmented off the network card like so.
ix0: runs several jails
ix1: bhyve instances
ix2: planning on adding a vpn
ix3: gateway
assumptions, FreeBSD 12x, updated and ready to build ports with portmaster.
Step I: Install and Reboot
# install bhyve and tmux
# add kernel modules at boot
#configure rc.conf .. note: change zroot/vm to whereever you want the default bhyve datastore
#create vm datastore default and init bhyve
reboot
for reference:
/boot/loader.conf
/etc/rc.conf
port 1 is bridged with 4 jails vnet0-4
port 2 is a virtual switch with tap0 member
Part II : Installing Ubuntu 18.04
# create switch and add interface to it .. change ix1 to your second port
# now we can build the vm .. change "vmname" to what ever you like and up 500M to 10G etc.
add: grub_run_partition="2"
change: cpu / mem
:wq!
# lets fire it up!
(you may need to press enter once or twice)
install ubuntu by pressing "next" and "yes" with your nose.. && reboot, as long as tumux is still running your fancy new vm should come right back up.
for scope I have segmented off the network card like so.
ix0: runs several jails
ix1: bhyve instances
ix2: planning on adding a vpn
ix3: gateway
assumptions, FreeBSD 12x, updated and ready to build ports with portmaster.
Step I: Install and Reboot
# install bhyve and tmux
portmaster sysutils/vm-bhyve sysutils/grub2-bhyve sysutils/tmux
# add kernel modules at boot
cat >> /boot/loader.conf << EOF
if_bridge_load="YES"
if_tap_load="YES"
nmdm_load="YES"
vmm_load="YES"
EOF
#configure rc.conf .. note: change zroot/vm to whereever you want the default bhyve datastore
sysrc vm_enable="YES"
sysrc vm_dir="zfs:zroot/vm"
sysrc vm_list=""
sysrc vm_delay="5"
#create vm datastore default and init bhyve
zfs create -o mountpoint=/vm zroot/vm
vm init
cp /usr/local/share/examples/vm-bhyve/* /vm/.templates/
reboot
for reference:
/boot/loader.conf
Code:
aesni_load="YES"
geom_eli_load="YES"
security.bsd.allow_destructive_dtrace=0
kern.geom.label.disk_ident.enable="0"
kern.geom.label.gptid.enable="0"
opensolaris_load="YES"
zfs_load="YES"
cc_htcp_load="YES"
# PF configuration
pf_load="YES"
pflog_load="YES"
net.pf.source_nodes_hashsize="1048576"
# ZFS settings
vfs.zfs.dirty_data_max_max="4359738368"
vfs.zfs.prefetch_disable="1"
# tcip settings
net.inet.tcp.hostcache.cachelimit="0"
kern.geom.label.disk_ident.enable="0"
kern.geom.label.gptid.enable="0"
kern.maxvnodes=250000
net.tcp.soreceive_stream="1"
# disable hyperthreading
machdep.hyperthreading_allowed="0"
# VM Kernel modules
if_bridge_load="YES"
if_tap_load="YES"
nmdm_load="YES"
vmm_load="YES"
/etc/rc.conf
Code:
clear_tmp_enable="YES"
ifconfig_ix0="DHCP"
ifconfig_ix1="DHCP"
ifconfig_ix2="DHCP"
ifconfig_ix3="DHCP"
syslogd_flags="-ss"
sendmail_enable="NONE"
hostname="removed"
sshd_enable="YES"
ntpdate_enable="YES"
ntpd_enable="YES"
powerd_enable="YES"
dumpdev="AUTO"
zfs_enable="YES"
keyrate="250.34"
# IOCage with VNET jails
iocage_enable="YES"
cloned_interfaces="bridge0"
ifconfig_bridge0="addm ix0 up"
# PF firewall
pf_enable="YES"
pf_rules="/etc/pf.conf"
pf_flags=""
pflog_enable="YES"
pflog_logfile="/var/log/pflog"
pflog_flags=""
# Smart CTL
smrtd_enable="YES"
kld_list="tmpfs aesni procfs"
# bhyve
vm_enable="YES"
vm_dir="zfs:zroot/vm"
smartd_enable="YES"
nmdm_load="YES"
vm_delay="5"
vm_list="vmname"
ifconfig
Code:
ix0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=a538b9<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6>
ether
inet x.x.x.x netmask 0xffffff00 broadcast x.x.x.x
media: Ethernet autoselect (10Gbase-T <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
ix1: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500 options=a538b9<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6>
ether
inet x.x.x.x netmask 0xffffff00 broadcast x.x.x.x
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
ix2: flags=8802<BROADCAST,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=e53fbb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6>
ether
inet x.x.x.x netmask 0xffffff00 broadcast x.x.x.x
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
ix3: flags=8802<BROADCAST,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=e53fbb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,WOL_UCAST,WOL_MCAST,WOL_MAGIC,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6>
ether
inet x.x.x.x netmask 0xffffff00 broadcast x.x.x.x
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
inet6
inet6 fe80:
inet 127.0.0.1 netmask 0xff000000
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>00:bd:d3:3c:be:00
pflog0: flags=141<UP,RUNNING,PROMISC> metric 0 mtu 33160
groups: pflog
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: vnet0.4 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 12 priority 128 path cost 2000
member: vnet0.3 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 11 priority 128 path cost 2000
member: vnet0.2 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 10 priority 128 path cost 2000
member: vnet0.1 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 9 priority 128 path cost 2000
member: ix0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 1 priority 128 path cost 2000
groups: bridge
nd6 options=9<PERFORMNUD,IFDISABLED>
vm-public: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
ether
id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
member: tap0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 13 priority 128 path cost 2000000
member: ix1 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
ifmaxaddr 0 port 2 priority 128 path cost 2000
groups: bridge vm-switch viid-4c918@port 3 is
nd6 options=1<PERFORMNUD>
vnet0.1: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
description: associated with jail: siren_outlands_local as nic: epair0b
options=8<VLAN_MTU>
ether
hwaddr 00:bd:d3:3c:be:00
inet6 prefixlen 64 scopeid 0x9
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
vnet0.2: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
description: associated with jail: banshee_outlands_local as nic: epair0b
options=8<VLAN_MTU>
ether
hwaddr
inet6 vnet0.2 prefixlen 64 scopeid 0xa
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
vnet0.3: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
description: associated with jail: medusa_outlands_local as nic: epair0b
options=8<VLAN_MTU>
ether
hwaddr
inet6 vnet0.3 prefixlen 64 scopeid 0xb
groups: epair
media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
vnet0.4: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
description: associated with jail: firegiant_outlands_local as nic: epair0b
options=8<VLAN_MTU>
ether
hwaddr
inet6 vnet0.4 prefixlen 64 scopeid 0xc
groups: epair
media: Ethernet 10Gbastap0:e-T (10Gbase-T <full-duplex>)
status: active
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
tap0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 mtu 1500
description: vmnet-seedbox-0-public
options=80000<LINKSTATE>
ether
groups: tap vm-port
media: Ethernet autoselect
status: active
nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
Opened by PID 16261
port 1 is bridged with 4 jails vnet0-4
port 2 is a virtual switch with tap0 member
Part II : Installing Ubuntu 18.04
# create switch and add interface to it .. change ix1 to your second port
cd /vm
vm switch create public
vm switch add public ix1
# now we can build the vm .. change "vmname" to what ever you like and up 500M to 10G etc.
fetch http://releases.ubuntu.com/18.04.3/ubuntu-18.04.3-live-server-amd64.iso
cp .templates/ubuntu.conf .
vm create -t ubuntu -s 500M vmname
vm configure vmname
add: grub_run_partition="2"
change: cpu / mem
:wq!
sysrc vm_list="vmname"
# lets fire it up!
vm install vmname ubuntu-18.04.1.0-live-server-amd64.iso
tmux
vm console vmname
(you may need to press enter once or twice)
install ubuntu by pressing "next" and "yes" with your nose.. && reboot, as long as tumux is still running your fancy new vm should come right back up.