UPDATE:
I may have actually solved this, by swapping the order so that the jail creates the RFC1918 address on that interface FIRST, and then creates the loopback interface second. Now this confuses me even more, since I cannot route using ipv4 from the host system on that FIB, but it is routing inside the jail. Very strange. Would still appreciate any comments on best practices for this sort of use case.
ORIGINAL:
Hi,
Hoping someone has some ideas about what I might be missing. I have IPv4 connectivity issues in jails using non-0 FIBs that appear to be related to how the IPv4 routing is handled. I'll set up the details and it should become clear what the problem is.
Host and jail versions, respectively.
I have a FIB that is used by my jail called netbox:
And from inside the jail, the routing table:
FWIW, I am testing using the setting
IPv6 connectivity works fine in this jail.
But the IPv4 source address is not selected as I would expect it to be.
Works fine if I specify the source address.
The network configuration is
Even stranger to me is that from the host system, I can't route using this FIB at all.
When I set net.add_addr_allfibs=1 this has allowed me to route using the FIB from the host system, but the same behaviour persists on the jail system.
I'm not really worried about attackers hitting my firewall, so I'll share my configurations.
jail.conf
rc.conf
I'm at a bit of a loss. Any ideas? Thanks.
I may have actually solved this, by swapping the order so that the jail creates the RFC1918 address on that interface FIRST, and then creates the loopback interface second. Now this confuses me even more, since I cannot route using ipv4 from the host system on that FIB, but it is routing inside the jail. Very strange. Would still appreciate any comments on best practices for this sort of use case.
ORIGINAL:
Hi,
Hoping someone has some ideas about what I might be missing. I have IPv4 connectivity issues in jails using non-0 FIBs that appear to be related to how the IPv4 routing is handled. I'll set up the details and it should become clear what the problem is.
Host and jail versions, respectively.
Code:
root@barge-1:~ # freebsd-version
13.1-RELEASE-p5
Code:
root@netbox:~ # freebsd-version
13.1-RELEASE-p9I have a FIB that is used by my jail called netbox:
Code:
root@barge-1:~ # setfib 1 netstat -rn
Routing tables (fib: 1)
Internet:
Destination Gateway Flags Netif Expire
default 192.168.7.1 UGS igb1
127.0.0.1 link#5 UHS lo0
127.0.1.10 link#6 UH lo1
127.0.1.109 link#6 UH lo1
127.0.1.203 link#6 UH lo1
192.168.6.0/24 link#1 U igb0
192.168.7.0/24 link#2 U igb1
192.168.8.0/24 link#3 U igb2
192.168.10.0/24 link#4 U igb3
Internet6:
Destination Gateway Flags Netif Expire
::/96 ::1 UGRS lo0
default 2603:3015:1003:5627::cab:0 UGS igb1
::1 link#5 UHS lo0
::ffff:0.0.0.0/96 ::1 UGRS lo0
2603:3015:1003:5626::/64 link#1 U igb0
2603:3015:1003:5627::/64 link#2 U igb1
2603:3015:1003:5628::/64 link#3 U igb2
2603:3015:1003:562a::/64 link#4 U igb3
fe80::/10 ::1 UGRS lo0
fe80::%igb0/64 link#1 U igb0
fe80::%igb1/64 link#2 U igb1
fe80::%igb2/64 link#3 U igb2
fe80::%igb3/64 link#4 U igb3
fe80::%lo0/64 link#5 U lo0
fe80::%lo1/64 link#6 U lo1
ff02::/16 ::1 UGRS lo0And from inside the jail, the routing table:
Code:
root@netbox:~ # netstat -rn
Routing tables
Internet:
Destination Gateway Flags Netif Expire
127.0.1.203 link#6 UH lo1
192.168.7.203 link#2 UHS lo0
Internet6:
Destination Gateway Flags Netif Expire
2603:3015:1003:5627:bed:1:feed:203 link#2 UHS lo0
root@netbox:~ # setfib 1 netstat -rn
Routing tables (fib: 1)
Internet:
Destination Gateway Flags Netif Expire
127.0.1.203 link#6 UH lo1
root@netbox:~ #FWIW, I am testing using the setting
Code:
root@barge-1:~ # sysctl net.add_addr_allfibs
net.add_addr_allfibs: 0IPv6 connectivity works fine in this jail.
Code:
root@netbox:~ # ping google.com
PING6(56=40+8+8 bytes) 2603:3015:1003:5627:bed:1:feed:203 --> 2607:f8b0:4009:809::200e
16 bytes from 2607:f8b0:4009:809::200e, icmp_seq=0 hlim=54 time=24.494 msBut the IPv4 source address is not selected as I would expect it to be.
Code:
root@netbox:~ # ping -4 google.com
PING google.com (142.250.190.14): 56 data bytes
ping: sendto: Can't assign requested addressWorks fine if I specify the source address.
Code:
root@netbox:~ # ping -S 192.168.7.203 -4 google.com
PING google.com (142.250.190.14) from 192.168.7.203: 56 data bytes
64 bytes from 142.250.190.14: icmp_seq=0 ttl=54 time=19.371 ms
64 bytes from 142.250.190.14: icmp_seq=1 ttl=54 time=23.174 ms
64 bytes from 142.250.190.14: icmp_seq=2 ttl=54 time=27.399 msThe network configuration is
Code:
root@netbox:~ # ifconfig
igb0: flags=8863<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=4e507bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,NOMAP>
ether b8:ca:3a:67:87:c8
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
igb1: flags=8863<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=4e507bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,NOMAP>
ether b8:ca:3a:67:87:c9
inet 192.168.7.203 netmask 0xffffff00 broadcast 192.168.7.255
inet6 2603:3015:1003:5627:bed:1:feed:203 prefixlen 64
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
igb2: flags=8863<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=4e507bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,NOMAP>
ether b8:ca:3a:67:87:ca
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
igb3: flags=8863<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
options=4e507bb<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,JUMBO_MTU,VLAN_HWCSUM,TSO4,TSO6,LRO,VLAN_HWFILTER,VLAN_HWTSO,RXCSUM_IPV6,TXCSUM_IPV6,NOMAP>
ether b8:ca:3a:67:87:cb
media: Ethernet autoselect (1000baseT <full-duplex>)
status: active
nd6 options=23<PERFORMNUD,ACCEPT_RTADV,AUTO_LINKLOCAL>
lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
lo1: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
options=680003<RXCSUM,TXCSUM,LINKSTATE,RXCSUM_IPV6,TXCSUM_IPV6>
inet 127.0.1.203 netmask 0xffffffff
groups: lo
nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>Even stranger to me is that from the host system, I can't route using this FIB at all.
Code:
root@barge-1:~ # setfib -F1 traceroute -i igb1 google.com
traceroute to google.com (172.217.1.110), 64 hops max, 40 byte packets
^C
root@barge-1:~ # setfib -F1 ping -4 -S 192.168.7.203 google.com
PING google.com (172.217.1.110) from 192.168.7.203: 56 data bytes
^C
--- google.com ping statistics ---
2 packets transmitted, 0 packets received, 100.0% packet loss
root@barge-1:~ # setfib 1 ping -4 google.com
PING google.com (172.217.1.110): 56 data bytes
^CWhen I set net.add_addr_allfibs=1 this has allowed me to route using the FIB from the host system, but the same behaviour persists on the jail system.
I'm not really worried about attackers hitting my firewall, so I'll share my configurations.
jail.conf
Code:
$barge_id = "1";
$domain = "delhi.o4data.net";
$jail_zfs = "tank/jail";
$jail_path = "/$jail_zfs";
path = "$jail_path/$name";
host.hostname = "$name.$domain";
$hostsrv_if = "igb1";
$hostsrv_fib = "1";
$hostsrv_cidr = "64";
$hostsrv_subnet = "2603:3015:1003:5627";
$hostsrv_addr = "$hostsrv_subnet:bed:$barge_id:feed:$guest_id/$hostsrv_cidr";
$hostsrv_cidr4 = "24";
$hostsrv_subnet4 = "192.168.7";
$hostsrv_cast4 = "$hostsrv_subnet4.255";
$hostsrv_addr4 = "$hostsrv_subnet4.$guest_id/$hostsrv_cidr4";
$hostsrv_conf = "$hostsrv_if|$hostsrv_addr";
$hostsrv_conf4 = "$hostsrv_if|$hostsrv_addr4 $hostsrv_cast4";
$loop_if = "lo1";
$loop_subnet4 = "127.0.$barge_id";
$loop_cast4 = "127.255.255.255";
$loop_addr4 = "$loop_subnet4.$guest_id";
$loop_conf4 = "$loop_if|$loop_addr4 $loop_cast4";
children.max = 0;
securelevel = 2;
enforce_statfs = 2;
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
exec.clean;
netbox {
$guest_id = "203";
exec.fib = "$hostsrv_fib";
ip6.addr = "$hostsrv_conf";
ip4.addr = "$loop_conf4";
ip4.addr += "$hostsrv_conf4";
allow.raw_sockets;
}rc.conf
Code:
clear_tmp_enable="YES"
hostname="barge-1.delhi.o4data.net"
ifconfig_igb0="inet 192.168.6.11 netmask 255.255.255.0"
ifconfig_igb0_ipv6="inet6 accept_rtadv"
ifconfig_igb0_alias0="inet6 2603:3015:1003:5626::bed:1 prefixlen 64"
ifconfig_igb1="inet 192.168.7.11 netmask 255.255.255.0"
ifconfig_igb1_ipv6="inet6 accept_rtadv"
ifconfig_igb1_alias0="inet6 2603:3015:1003:5627::bed:1 prefixlen 64"
ifconfig_igb2="inet 192.168.8.11 netmask 255.255.255.0"
ifconfig_igb2_ipv6="inet6 accept_rtadv"
ifconfig_igb2_alias0="inet6 2603:3015:1003:5628::bed:1 prefixlen 64"
ifconfig_igb3="inet 192.168.10.11 netmask 255.255.255.0"
ifconfig_igb3_ipv6="inet6 accept_rtadv"
ifconfig_igb3_alias0="inet6 2603:3015:1003:562a::bed:1 prefixlen 64"
static_routes="hdef4 hdmz4 hsrv4 hctl4 gdmz4"
ipv6_static_routes="hdef hdmz hsrv hctl gdmz"
route_hdef4="-net 0.0.0.0/0 192.168.6.1 -fib 0"
ipv6_route_hdef="-net ::/0 2603:3015:1003:5626::cab:0 -fib 0"
route_hsrv4="-net 0.0.0.0/0 192.168.7.1 -fib 1"
ipv6_route_hsrv="-net ::/0 2603:3015:1003:5627::cab:0 -fib 1"
route_hctl4="-net 0.0.0.0/0 192.168.8.1 -fib 2"
ipv6_route_hctl="-net ::/0 2603:3015:1003:5628::cab:0 -fib 2"
route_gdmz4="-net 0.0.0.0/0 192.168.10.1 -fib 3"
ipv6_route_gdmz="-net ::/0 2603:3015:1003:562a::cab:0 -fib 3"
route_hdmz4="-net 0.0.0.0/0 192.168.6.1 -fib 4"
ipv6_route_hdmz="-net ::/0 2603:3015:1003:5626::cab:0 -fib 4"
sshd_enable="YES"
moused_enable="YES"
ntpdate_enable="YES"
ntpd_enable="YES"
dumpdev="AUTO"
zfs_enable="YES"
nullfs_load="YES"
jail_enable="YES"
cloned_interfaces="lo1"
ezjail_enable="NO"I'm at a bit of a loss. Any ideas? Thanks.