Hey All,
Need you assistance in solving this networking issue.
I am running FreeBSD 11-STABLE on my server. Use this machine as a router / NAS. I am investing sometime in setting up jails. Brief overview of what my setup is, what I am trying to accomplish and what the issue is.
System is running 3 fibs.Fib one gets default route from ISP and fib 1 & fib 2 use vpn as default route. I've setup two jails one in fib0 and one in fib2. My issue is that I am unable to ping jail1 (jdev1) from 10.10.10.0/24, but I can ping it from 10.10.20.0/24. Jail2 (jdev2) does not suffer from this issue I can ping it from both network ranges.
Relevant rc.conf section
pf.conf file
jail.conf file
netstat -r4 output below
sysctl net.add_addr_allfibs: 1, so system adds routes automatically across all fibs.
I feel there is something really basic that I am missing. Would appreciate if someone can have a quick glance at the issue and give me pointers.
Need you assistance in solving this networking issue.
I am running FreeBSD 11-STABLE on my server. Use this machine as a router / NAS. I am investing sometime in setting up jails. Brief overview of what my setup is, what I am trying to accomplish and what the issue is.
System is running 3 fibs.Fib one gets default route from ISP and fib 1 & fib 2 use vpn as default route. I've setup two jails one in fib0 and one in fib2. My issue is that I am unable to ping jail1 (jdev1) from 10.10.10.0/24, but I can ping it from 10.10.20.0/24. Jail2 (jdev2) does not suffer from this issue I can ping it from both network ranges.
Relevant rc.conf section
Code:
# Hostname for computer
hostname="bnx"
# WAN ip config
#ifconfig_igb1="dhcp"
ifconfig_igb0="up"
gateway_enable="YES"
# NAT ip config
# lagg lacp setup
ifconfig_ix0="up"
ifconfig_ix1="up"
cloned_interfaces="lagg0 vlan1 vlan2 vlan3 lo1 lo2 lo3"
ifconfig_lagg0="laggproto lacp laggport ix0 laggport ix1"
ifconfig_vlan1="inet 10.10.10.10 netmask 255.255.255.0 vlan 1 vlandev lagg0"
ifconfig_vlan2="inet 10.10.20.20 netmask 255.255.255.0 vlan 2 vlandev lagg0 fib 1"
ifconfig_vlan3="inet 10.10.30.30 netmask 255.255.255.0 vlan 3 vlandev lagg0 fib 2"
ifconfig_lo1="inet 10.10.30.10 netmask 255.255.255.255 fib 2"
ifconfig_lo2="inet 10.10.10.20 netmask 255.255.255.255"
ifconfig_lo3="inet 10.10.30.20 netmask 255.255.255.255 fib 2"pf.conf file
Code:
## Macros
# Interfaces
ext = "tun0"
int1 = "vlan1"
int2 = "vlan2"
int3 = "vlan3"
vpn = "tun1"
v1p = "tun2"
v2p = "tun3"
localnet = $int1:network
vps = "10.10.50.0/24"
vpc = "10.10.20.0/24"
broken="224.0.0.22 127.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12, 169.254.0.0/16, 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24, 169.254.0.0/16, 240.0.0.0/4, 255.255.255.255/32"
## Tables
# Open ports
tcps = "{22,22000,32400}"
udps = "{1194}"
p2p = "{51266}"
table <fail2ban> persist
## Options
set limit states 20000
set block-policy drop
set state-policy if-bound
set loginterface $ext
#set skip on {lo0,lo1,lo2,lo3}
set skip on lo
## Traffic Normalization
#scrub in all no-df max-mss 1440
## Translation
# NAT
nat on $ext from $localnet to any -> ($ext)
nat on $v1p from $vpc to any -> ($v1p)
nat on $ext from $vps to any -> ($ext)
nat on $v2p from 10.10.30.10 to any -> ($v2p)
nat on $ext from 10.10.10.20 to any -> ($ext)
nat on $v2p from 10.10.30.20 to any -> ($v2p)
#nat on $v1p from $vps to any -> ($v1p) rtable 1
# Filters
## Block rules for external interfaces inbound/outbound
antispoof quick for {$ext, $v1p, $v2p}
block drop in log quick on {$ext, $v1p, $v2p} from {$broken no-route} to any
block drop in log quick on {$ext, $v1p, $v2p} from <fail2ban> to any
block drop in quick on {$ext, $v1p, $v2p} inet6
block drop in log on {$ext, $v1p, $v2p} # Default policy
#block out quick on {$ext, $v2p} inet6
## Pass rules for external interfaces inbound/outbound
pass inet proto icmp icmp-type echoreq keep state rtable 0
pass inet proto icmp icmp-type echoreq keep state rtable 1
pass inet proto icmp icmp-type echoreq keep state rtable 2
pass in on $ext inet proto tcp from any to ($ext) port $tcps rtable 0
pass in on $ext inet proto udp from any to ($ext) port $udps rtable 0
pass in on $v2p inet proto {tcp, udp} from any to any port $p2p rtable 2
## Pass rules on internal interfaces indound/outbound
pass in on $int1 all rtable 0
pass in on $int2 all rtable 1
pass in on $int3 all rtable 2
pass in on $vpn all rtable 0
#pass out all rtable 0
#pass out all rtable 1
pass out inet alljail.conf file
Code:
allow.raw_sockets = "1";
allow.set_hostname = "1";
allow.sysvipc = "1";
allow.mount.devfs;
host.hostname = "${name}";
path = "/usr/jails/${name}";
mount.fstab = "/etc/jails/fstab.${name}";
mount.devfs = "1";
devfs_ruleset = "4";
exec.start = "/bin/sh /etc/rc";
exec.stop = "/bin/sh /etc/rc.shutdown";
exec.clean;
# Jail definitions
base {
exec.fib = "2";
ip4.addr = "10.10.30.10";
}
jdev1 {
ip4.addr = "10.10.10.20";
}
jdev2 {
exec.fib = "2";
ip4.addr = "10.10.30.20";
}netstat -r4 output below
Code:
netstat -r4 [0:18:15]
Routing tables
Internet:
Destination Gateway Flags Netif Expire
default 10.20.25.96 US tun0
10.4.0.0/16 10.4.0.1 UGS tun3
10.4.0.1 link#16 UH tun3
10.4.71.179 link#16 UHS lo0
10.10.10.0/24 link#8 U vlan1
bnx link#8 UHS lo0
jdev1 link#12 UH lo2
10.10.20.0/24 link#9 U vlan2
10.10.30.0/24 link#10 U vlan3
10.10.30.10 link#11 UH lo1
jdev2 link#13 UH lo3
10.10.50.0/24 10.10.50.2 UGS tun1
10.10.50.1 link#15 UHS lo0
10.10.50.2 link#15 UH tun1
10.20.25.96 link#14 UHS tun0
localhost link#5 UH lo0
xx.xx.xx.xx link#14 UHS lo0
Code:
setfib 1 netstat -r4 [0:35:41]
Routing tables (fib: 1)
Internet:
Destination Gateway Flags Netif Expire
10.4.0.1 link#16 UH tun3
10.10.10.0/24 link#8 U vlan1
jdev1 link#12 UH lo2
10.10.20.0/24 link#9 U vlan2
nix link#9 UHS lo0
10.10.30.0/24 link#10 U vlan3
10.10.30.10 link#11 UH lo1
jdev2 link#13 UH lo3
10.10.50.0/24 tun1 US tun1
10.10.50.2 link#15 UH tun1
10.20.25.96 link#14 UH tun0
localhost link#5 UH lo0
Code:
setfib 2 netstat -r4 [0:39:02]
Routing tables (fib: 2)
Internet:
Destination Gateway Flags Netif Expire
default 10.4.0.1 UGS tun3
10.4.0.1 link#16 UH tun3
10.10.10.0/24 link#8 U vlan1
jdev1 link#12 UH lo2
10.10.20.0/24 link#9 U vlan2
10.10.30.0/24 link#10 U vlan3
10.10.30.10 link#11 UH lo1
jdev2 link#13 UH lo3
hax link#10 UHS lo0
10.10.50.0/24 tun1 US tun1
10.10.50.2 link#15 UH tun1
10.20.25.96 link#14 UH tun0
localhost link#5 UH lo0sysctl net.add_addr_allfibs: 1, so system adds routes automatically across all fibs.
I feel there is something really basic that I am missing. Would appreciate if someone can have a quick glance at the issue and give me pointers.
