I have 3 jail using VNET for networking. The setup is mostly following from Jail handbook guide.
The problem is, after reboot 2 jail has no access to internet and cannot accessed from outside, restarting all jail fix this problem.
here is some /var/log/messages
(host) /etc/rc.conf
/etc/jail.conf
Each jail configuration
/etc/jail.conf.d/postgres.conf
/etc/pf.conf
When starting jail manually the networking works as expected, but when the host reboot and jail started by rc, jail caddy and nginx cannot be accessed from outside (port mapped in pf).
so after reboot, all jail should be restarted manually to make it works.
on /var/log/message has message: "arp: ae:34:e7:1c:fc:14 is using my IP address 10.10.1.2 on epair2b!" could it be ip conflict? but why no conflict when restart jail manually (service jail restart)
The problem is, after reboot 2 jail has no access to internet and cannot accessed from outside, restarting all jail fix this problem.
here is some /var/log/messages
Code:
Dec 27 11:06:00 freebsd kernel: epair2a: Ethernet address: 02:4c:69:01:e2:0a
Dec 27 11:06:00 freebsd kernel: epair2b: Ethernet address: 02:4c:69:01:e2:0b
Dec 27 11:06:00 freebsd kernel: epair2a: link state changed to UP
Dec 27 11:06:00 freebsd kernel: epair2b: link state changed to UP
Dec 27 11:06:01 freebsd kernel: epair2a: promiscuous mode enabled
Dec 27 11:06:01 freebsd kernel: lo0: link state changed to UP
Dec 27 11:06:01 freebsd kernel:
Dec 27 11:06:01 freebsd kernel: epair3a: Ethernet address: 02:d1:ec:53:bb:0a
Dec 27 11:06:01 freebsd kernel: epair3b: Ethernet address: 02:d1:ec:53:bb:0b
Dec 27 11:06:01 freebsd kernel: epair3a: link state changed to UP
Dec 27 11:06:01 freebsd kernel: epair3b: link state changed to UP
Dec 27 11:06:01 freebsd kernel: epair3a: promiscuous mode enabled
Dec 27 11:06:02 freebsd kernel: arp: ae:34:e7:1c:fc:14 is using my IP address 10.10.1.2 on epair2b!
Dec 27 11:06:02 freebsd kernel: lo0: link state changed to UP
Dec 27 11:06:03 freebsd kernel:
Dec 27 11:06:03 freebsd kernel: epair4a: Ethernet address: 02:d2:07:ca:35:0a
Dec 27 11:06:03 freebsd kernel: epair4b: Ethernet address: 02:d2:07:ca:35:0b
Dec 27 11:06:03 freebsd kernel: epair4a: link state changed to UP
Dec 27 11:06:03 freebsd kernel: epair4b: link state changed to UP
Dec 27 11:06:03 freebsd kernel: epair4a: promiscuous mode enabled
Dec 27 11:06:03 freebsd kernel: arp: ae:34:e7:1c:fc:14 is using my IP address 10.10.1.4 on epair4b!
Dec 27 11:06:03 freebsd kernel: lo0: link state changed to UP
Dec 27 11:10:38 freebsd kernel: arp: 10.10.1.3 moved from 02:d1:ec:53:bb:0b to ae:34:e7:1c:fc:14 on bridge0
Dec 27 11:16:03 freebsd kernel: arp: 10.10.1.4 moved from 02:d2:07:ca:35:0b to ae:34:e7:1c:fc:14 on bridge0
Dec 27 11:25:33 freebsd kernel: arp: 10.10.1.1 moved from 58:9c:fc:00:6b:19 to 02:4c:69:01:e2:0a on epair2b(host) /etc/rc.conf
Code:
# Networking
hostname="freebsd.prod"
ifconfig_vtnet0="DHCP"
ifconfig_vtnet0_ipv6="inet6 accept_rtadv"
# Service
sshd_enable="YES"
ntpd_enable="YES"
zfs_enable="YES"
# Jail
jail_enable="YES"
jail_parallel_start="NO"
jail_list="postgres caddy nginx"
jail_reverse_stop="YES"
# Jail VNET
cloned_interfaces="bridge0"
create_args_bridge0="inet 10.10.1.1/24 addm vtnet0 up"
# Firewall
gateway_enable="YES"
pf_enable="YES"
pflog_enable="yes"
blacklistd_enable="yes"/etc/jail.conf
Code:
# PERMISSIONS
allow.raw_sockets;
allow.set_hostname;
exec.clean;
mount.devfs;
devfs_ruleset=4;
# HOSTNAME/PATH
host.hostname = "${name}";
path = "/usr/local/jails/${name}";
# STARTUP/LOGGING
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
exec.consolelog = "/var/log/jail_${name}_console.log";
.include "/etc/jail.conf.d/*.conf";Each jail configuration
Code:
postgres: $id = "2";
caddy: $id = "3";
nginx: $id = "4";/etc/jail.conf.d/postgres.conf
Code:
postgres {
allow.sysvipc = 1;
devfs_ruleset = 5;
# VNET/VIMAGE
vnet;
vnet.interface = "${epair}b";
# NETWORKS/INTERFACES
$id = "2";
$ip = "10.10.1.${id}/24";
$gateway = "10.10.1.1";
$bridge = "bridge0";
$epair = "epair${id}";
# ADD TO bridge INTERFACE
exec.prestart = "/sbin/ifconfig ${epair} create up";
exec.prestart += "/sbin/ifconfig ${epair}a up descr jail:${name}";
exec.prestart += "/sbin/ifconfig ${bridge} addm ${epair}a up";
exec.start += "/sbin/ifconfig ${epair}b ${ip} up";
exec.start += "/sbin/route add default ${gateway}";
exec.poststop = "/sbin/ifconfig ${bridge} deletem ${epair}a";
exec.poststop += "/sbin/ifconfig ${epair}a destroy";
}/etc/pf.conf
Code:
# External interface
ext_if = "vtnet0"
allowed_ports = "{ 22, 80, 443, 8080 }"
# Options
set skip on lo
set skip on bridge
set skip on epair
# Scrubbing
scrub in on $ext_if all fragment reassemble
# Translation rules (NAT and redirects)
nat on $ext_if inet from 10.10.1.0/24 to any -> ($ext_if)
rdr on $ext_if proto tcp from any to ($ext_if) port {80, 443} -> 10.10.1.3
rdr on $ext_if proto tcp from any to ($ext_if) port 8080 -> 10.10.1.4 port 80
# Default block policy
block log all
# Pass rules
pass out keep state
# Allow jail traffic
pass in on $ext_if inet proto tcp from any to any port $allowed_ports flags S/SA keep state
#pass in on $ext_if inet proto tcp from any to 10.10.1.3 port {80, 443} flags S/SA keep state
# ICMP
pass inet proto icmp all keep state
pass inet6 proto icmp6 all keep state
# Blacklist anchor
anchor "blacklistd/*" in on $ext_ifWhen starting jail manually the networking works as expected, but when the host reboot and jail started by rc, jail caddy and nginx cannot be accessed from outside (port mapped in pf).
so after reboot, all jail should be restarted manually to make it works.
on /var/log/message has message: "arp: ae:34:e7:1c:fc:14 is using my IP address 10.10.1.2 on epair2b!" could it be ip conflict? but why no conflict when restart jail manually (service jail restart)