Solved [Solved] Custom kernel and jails

Hello everyone,

I have noticed that since I have created a new custom kernel, I am no longer able to start my jails. From what I worked out it is because of the following line in /etc/fstab.webjail:
Code:
/local/jails/basejail /local/jails/webjail/basejail nullfs ro     0       0
/usr/ports     /local/jails/webjail/usr/ports  nullfs     rw       0       0
The first line above was added by the system during jail creation and I added the second to mount the host's ports tree into the jail.

The error message is:
Code:
mount_nullfs: Operation not supported by device
This is my custom kernel:
Code:
## Rebuild the kernel with the following
# kldstat
# echo "MODULES_OVERRIDE=zfs opensolaris aio accf_data accf_dns accf_http amdtemp cc/cc_htcp fdescfs geom/geom_nop usb/ums" >> /etc/make.conf
# cp /usr/src/sys/amd64/conf/GENERIC /usr/src/sys/amd64/conf/TRINITY
# vi /usr/src/sys/amd64/conf/TRINITY
# cd /usr/src/ && make clean && make buildkernel KERNCONF=TRINITY && make installkernel KERNCONF=TRINITY && make clean && sync && reboot

ident TRINITY
cpu   HAMMER

# change kernel message color to green on black
# while all other text is white on black
options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK)

# enable Pf without ALTQ (HFSC)
device pf
device pflog
device pfsync
options ALTQ
options ALTQ_HFSC
options ALTQ_NOPCC

# forward packets without decrementing
# the time to live (TTL) counter
options IPSTEALTH

# enable lagg interface
device lagg

# IPsec (Internet Protocol Security)
options IPSEC
options IPSEC_NAT_T
device  crypto

# Crypto and AES-NI support
# also add aesni_load="YES" to /boot/loader.conf
# verify AES speeds before and after with "openssl speed -evp aes-256-cbc -elapsed"
# as /dev/crypto can be slower then no /dev/crypto on some systems. Also, the
# ports version of /usr/local/bin/openssl is significantly faster then the
# default /usr/bin/openssl version which comes with freebsd.
device crypto
device cryptodev
device aesni

#############################################

makeoptions DEBUG=-g        			# Build kernel with gdb(1) debug symbols
makeoptions WITH_CTF=1      			# Run ctfconvert(1) for DTrace support

options     SCHED_ULE       			# ULE scheduler
options     PREEMPTION      			# Enable kernel thread preemption
options     INET            			# InterNETworking
options     INET6           			# IPv6 communications protocols
options     TCP_OFFLOAD     			# TCP offload
options     SCTP            			# Stream Control Transmission Protocol
options     FFS             			# Berkeley Fast Filesystem
options     SOFTUPDATES     			# Enable FFS soft updates support
options     UFS_ACL         			# Support for access control lists
options     UFS_DIRHASH     			# Improve performance on big directories
options     UFS_GJOURNAL    			# Enable gjournal-based UFS journaling
options     QUOTA           			# Enable disk quotas for UFS
options     MD_ROOT         			# MD is a potential root device
options     NFSCL           			# New Network Filesystem Client
options     NFSD            			# New Network Filesystem Server
options     NFSLOCKD        			# Network Lock Manager
options     NFS_ROOT        			# NFS usable as /, requires NFSCL
options     MSDOSFS         			# MSDOS Filesystem
options     CD9660          			# ISO 9660 Filesystem
options     PROCFS          			# Process filesystem (requires PSEUDOFS)
options     PSEUDOFS        			# Pseudo-filesystem framework
options     GEOM_PART_GPT   			# GUID Partition Tables.
options     GEOM_RAID       			# Soft RAID functionality.
options     GEOM_LABEL      			# Provides labelization
options     COMPAT_FREEBSD32    		# Compatible with i386 binaries
options     COMPAT_FREEBSD4     		# Compatible with FreeBSD4
options     COMPAT_FREEBSD5     		# Compatible with FreeBSD5
options     COMPAT_FREEBSD6     		# Compatible with FreeBSD6
options     COMPAT_FREEBSD7     		# Compatible with FreeBSD7
options     SCSI_DELAY=5000     		# Delay (in ms) before probing SCSI
options     KTRACE          			# ktrace(1) support
options     STACK           			# stack(9) support
options     SYSVSHM        				# SYSV-style shared memory
options     SYSVMSG        				# SYSV-style message queues
options     SYSVSEM         			# SYSV-style semaphores
options     _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options     PRINTF_BUFR_SIZE=128    	# Prevent printf output being interspersed.
options     KBD_INSTALL_CDEV    		# install a CDEV entry in /dev
options     HWPMC_HOOKS     			# Necessary kernel hooks for hwpmc(4)
options     AUDIT           			# Security event auditing
options     CAPABILITY_MODE     		# Capsicum capability mode
options     CAPABILITIES        		# Capsicum capabilities
options     PROCDESC        			# Support for process descriptors
options     MAC         				# TrustedBSD MAC Framework
options     KDTRACE_FRAME       		# Ensure frames are compiled in
options     KDTRACE_HOOKS       		# Kernel DTrace hooks
options     DDB_CTF         			# Kernel ELF linker loads CTF data
options     INCLUDE_CONFIG_FILE     	# Include this file in kernel

# Debugging support.  Always need this:
options     KDB             			# Enable kernel debugger support.
options     KDB_TRACE       			# Print a stack trace for a panic.

# Make an SMP-capable kernel by default
options     SMP             			# Symmetric MultiProcessor Kernel

# CPU frequency control
device      cpufreq

# Bus support.
device      acpi
device      pci

# ATA controllers
device      ahci        				# AHCI-compatible SATA controllers
device      ata     					# Legacy ATA/SATA controllers

# ATA/SCSI peripherals
device      scbus       				# SCSI bus (required for ATA/SCSI)
device      da      					# Direct Access (disks)
device      cd      					# CD

# RAID controllers
device          mfi             		# LSI MegaRAID SAS

# atkbdc0 controls both the keyboard and the PS/2 mouse
device      atkbdc      				# AT keyboard controller
device      atkbd       				# AT keyboard
device      ums     					# PS/2 mouse

device      kbdmux      				# keyboard multiplexer

device      vga     					# VGA video card driver
options     VESA        				# Add support for VESA BIOS Extensions (VBE)

device      splash      				# Splash screen and screen saver support

# syscons is the default console driver, resembling an SCO console
device      sc
options     SC_PIXEL_MODE   			# add support for the raster text mode

device      agp     					# support several AGP chipsets

# PCI Ethernet NICs.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device          miibus          		# MII bus support
#device      em      					# Intel PRO/1000 Gigabit Ethernet Family
#device      igb     					# Intel PRO/1000 PCIE Server Gigabit Family
#device      ixgbe       				# Intel PRO/10GbE PCIE Ethernet Family
device      bce     					# Broadcom NetXtreme II BCM5709 Gigabit Ethernet Family

# Pseudo devices.
device      loop        				# Network loopback
device      random      				# Entropy device
#device     padlock_rng 				# VIA Padlock RNG
device      rdrand_rng  				# Intel Bull Mountain RNG
device      ether       				# Ethernet support
device      vlan        				# 802.1Q VLAN support
device      tun     					# Packet tunnel.
device      md      					# Memory "disks"
device      gif     					# IPv6 and IPv4 tunneling
device      faith       				# IPv6-to-IPv4 relaying (translation)
device      firmware    				# firmware assist module

# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device      bpf     # Berkeley packet filter

# USB support
options     USB_DEBUG   				# enable debug msgs
device      uhci        				# UHCI PCI-USB interface
device      ohci        				# OHCI PCI-USB interface
device      ehci        				# EHCI PCI-USB interface (USB 2.0)
device      xhci        				# XHCI PCI-USB interface (USB 3.0)
device      usb     					# USB Bus (required)
device      ukbd        				# Keyboard
device      umass       				# Disks/Mass storage - Requires scbus and da
Have I removed something crucial to jails here? I looked in GENERIC and I found no reference to NULLFS.

Thank you in advance.

Fred
 
Re: Custom kernel and jails

Would I be right in assuming you're building your custom kernel with no (or only specific) kernel modules? nullfs(5) requires kernel support, but the module nullfs.ko is supposed to get loaded automagically on demand.

Do /boot/kernel/nullfs.ko and /boot/kernel/nullfs.ko.symbols exist? If you're going to be using nullfs often or even permanently, you might as well add it to the kernel anyway:
Code:
options NULLFS
If you don't fancy that, at least add nullfs to MODULES_OVERRIDE, which you seem to be using.
 
Re: Custom kernel and jails

I would suggest not using MODULES_OVERRIDE and just let the system build everything. You can still have a custom kernel though.
 
Re: Custom kernel and jails

I don't seem to have any reference to nullfs.ko at all :(
ll /boot/kernel
Code:
drwxr-xr-x  2 root  wheel        27 Jul 21 14:06 .
drwxr-xr-x  8 root  wheel        42 Jul 21 14:06 ..
-r-xr-xr-x  1 root  wheel      5928 Jul 21 14:06 accf_data.ko
-r-xr-xr-x  1 root  wheel     23048 Jul 21 14:06 accf_data.ko.symbols
-r-xr-xr-x  1 root  wheel      6552 Jul 21 14:06 accf_dns.ko
-r-xr-xr-x  1 root  wheel     28480 Jul 21 14:06 accf_dns.ko.symbols
-r-xr-xr-x  1 root  wheel     10048 Jul 21 14:06 accf_http.ko
-r-xr-xr-x  1 root  wheel     44992 Jul 21 14:06 accf_http.ko.symbols
-r-xr-xr-x  1 root  wheel     79520 Jul 21 14:06 aio.ko
-r-xr-xr-x  1 root  wheel    301696 Jul 21 14:06 aio.ko.symbols
-r-xr-xr-x  1 root  wheel     15744 Jul 21 14:06 amdtemp.ko
-r-xr-xr-x  1 root  wheel     52984 Jul 21 14:06 amdtemp.ko.symbols
-r-xr-xr-x  1 root  wheel     12304 Jul 21 14:06 cc_htcp.ko
-r-xr-xr-x  1 root  wheel     43168 Jul 21 14:06 cc_htcp.ko.symbols
-r-xr-xr-x  1 root  wheel     31128 Jul 21 14:06 fdescfs.ko
-r-xr-xr-x  1 root  wheel    220104 Jul 21 14:06 fdescfs.ko.symbols
-r-xr-xr-x  1 root  wheel     23408 Jul 21 14:06 geom_nop.ko
-r-xr-xr-x  1 root  wheel     68000 Jul 21 14:06 geom_nop.ko.symbols
-r-xr-xr-x  1 root  wheel   9657434 Jul 21 14:05 kernel
-r-xr-xr-x  1 root  wheel  32584828 Jul 21 14:05 kernel.symbols
-rw-r--r--  1 root  wheel      9560 Jul 21 14:06 linker.hints
-r-xr-xr-x  1 root  wheel     16400 Jul 21 14:06 opensolaris.ko
-r-xr-xr-x  1 root  wheel     68960 Jul 21 14:06 opensolaris.ko.symbols
-r-xr-xr-x  1 root  wheel     26928 Jul 21 14:06 ums.ko
-r-xr-xr-x  1 root  wheel     71824 Jul 21 14:06 ums.ko.symbols
-r-xr-xr-x  1 root  wheel   2147040 Jul 21 14:06 zfs.ko
-r-xr-xr-x  1 root  wheel  16898872 Jul 21 14:06 zfs.ko.symbols

@SirDice, why is using MODULES_OVERRIDE not a good idea? Would I have avoided my current problem if I didn't use it?

Thank you.
 
Last edited by a moderator:
Re: Custom kernel and jails

fred974 said:
@SirDice, why using MODULES_OVERRIDE is not a good idea?
Because you're bound to forget to build some module or a module depends on another module you didn't build. Or the system may leave behind some modules that were previously built for an older kernel. Loading them in your current kernel could cause problems. Just build everything, they're not that big and compared to building the rest of the kernel and world the extra build time is negligible.

Would I have avoided my current problem if I didn't use it?
Indeed.
 
Last edited by a moderator:
Re: Custom kernel and jails

Lesson learned! Thank you!

On my way to recompile my kernel.
 
Back
Top