FreeBSD and transparent Squid. Again...

[Mack]

Hello!
I know, that there were similar topics already here, but they didn't help me. Let me describe why.
I have FreeBSD 10.1 with 3 interfaces. 2 of them don't have any IP assigned and are combined into a bridge. 3rd has an IP and is used for O&M purposes. There's also Squid 3.5.4, working in TPROXY mode. The problem is that I cannot forward http traffic to it.
I use following ipfw rules:

ipfw add 1100 fwd 127.0.0.1,8080 log ip from any to any 80 in recv bge0
ipfw add 1101 fwd 127.0.0.1 log ip from any 80 to any in recv bge1

And the counters are growing:

01100  56 10370 fwd 127.0.0.1,8080 log logamount 50 ip from any to any dst-port 80 in recv bge0
01101  44 33438 fwd 127.0.0.1 log logamount 50 ip from any 80 to any in recv bge1

Logs seem to be OK also:

May 17 19:46:02 squid kernel: ipfw: 1100 Forward to 127.0.0.1:8080 TCP 192.168.1.10:13944 192.168.1.20:80 in via bge0
May 17 19:46:02 squid kernel: ipfw: 1101 Forward to 127.0.0.1 TCP 192.168.1.20:80 192.168.1.10:13944 in via bge1

And Squid has got nothing, cache.log is empty! If I stop Squid, nothing changes, so the traffic is bypassing it for some reason. That is the problem!

From the beginning I was trying to make Squid work, using intercept mode with the help of pf, and it worked after I've assigned an address to the bridge (interfaces were still without IPs). But it was modifying source address in this mode, which is not needed to me, so I've moved to tproxy mode.

So, here's Squid config:

acl localnet src 192.168.0.0/16 # RFC1918 possible internal network

acl SSL_ports port 443
acl Safe_ports port 80 # http
acl Safe_ports port 21 # ftp
acl Safe_ports port 443  # https
acl Safe_ports port 70 # gopher
acl Safe_ports port 210  # wais
acl Safe_ports port 1025-65535 # unregistered ports
acl Safe_ports port 280  # http-mgmt
acl Safe_ports port 488  # gss-http
acl Safe_ports port 591  # filemaker
acl Safe_ports port 777  # multiling http
acl CONNECT method CONNECT

http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports

acl manager url_regex -i ^cache_object:// /squid-internal-mgr/
http_access allow localhost manager
http_access deny manager

http_access allow localnet
http_access allow localhost
http_access deny all

http_port 3128
http_port 127.0.0.1:8080 tproxy

refresh_pattern ^ftp: &n... 1440 20%  10080
refresh_pattern ^gopher: 1440 0% 1440
refresh_pattern -i (/cgi-bin/|\?) 0  0% 0
#refresh_pattern .  0  20%  43200 override-lastmod reload-into-ims
refresh_pattern .  0  20%  4320

debug_options ALL,1 0,9 5,9 11,9 17,9 73,9 76,9 89,9 22,3 3,3 28,9

logformat squid %{%Y.%m.%d %H:%M:%S %z}tl %6tr %>a %Ss/%03>Hs %<st %rm %ru %un %Sh/%<A %mt
access_log /var/log/squid/access.log
cache_log /var/log/squid/cache.log

After it starts I can see in cache.log:

2015/05/17 20:03:49.015 kid1| Accepting TPROXY intercepted HTTP Socket connections at local=127.0.0.1:8080 remote=[::] FD 17 flags=25

squid -v:

configure options: '--with-default-user=squid' '--bindir=/usr/local/sbin' '--sbindir=/usr/local/sbin' '--datadir=/usr/local/etc/squid' '--libexecdir=/usr/local/libexec/squid' '--localstatedir=/var' '--sysconfdir=/usr/local/etc/squid' '--with-logdir=/var/log/squid' '--with-pidfile=/var/run/squid/squid.pid' '--with-swapdir=/var/squid/cache' '--without-gnutls' '--enable-auth' '--enable-build-info' '--enable-loadable-modules' '--enable-removal-policies=lru heap' '--disable-epoll' '--disable-linux-netfilter' '--disable-linux-tproxy' '--disable-translation' '--disable-arch-native' '--disable-eui' '--disable-cache-digests' '--disable-delay-pools' '--disable-ecap' '--disable-esi' '--disable-follow-x-forwarded-for' '--enable-htcp' '--disable-icap-client' '--disable-icmp' '--enable-ident-lookups' '--disable-ipv6' '--enable-kqueue' '--with-large-files' '--disable-http-violations' '--without-nettle' '--enable-snmp' '--disable-ssl' '--disable-ssl-crtd' '--disable-stacktraces' '--disable-ipf-transparent' '--enable-ipfw-transparent' '--disable-pf-transparent' '--with-nat-devpf' '--disable-forw-via-db' '--disable-wccp' '--disable-wccpv2' '--with-heimdal-krb5=/usr' 'CFLAGS=-I/usr/include -O2 -pipe -fstack-protector -fno-strict-aliasing' 'LDFLAGS=-L/usr/lib -pthread -fstack-protector' 'LIBS=-lkrb5 -lgssapi -lgssapi_krb5 ' '--enable-auth-basic=DB SMB_LM MSNT-multi-domain NCSA PAM POP3 RADIUS fake getpwnam' '--enable-auth-digest=file' '--enable-external-acl-helpers=file_userip time_quota unix_group' '--enable-auth-negotiate=kerberos wrapper' '--enable-auth-ntlm=fake smb_lm' '--enable-storeio=ufs aufs diskd' '--enable-disk-io=AIO Blocking IpcIo Mmapped DiskThreads DiskDaemon' '--enable-log-daemon-helpers=file' '--enable-url-rewrite-helpers=fake' '--enable-storeid-rewrite-helpers=file' '--prefix=/usr/local' '--mandir=/usr/local/man' '--infodir=/usr/local/info/' '--build=amd64-portbld-freebsd10.1' 'build_alias=amd64-portbld-freebsd10.1' 'CC=cc' 'CPPFLAGS=' 'CXX=c++' 'CXXFLAGS=-O2 -pipe -fstack-protector -fno-strict-aliasing ' 'CPP=cpp' '--enable-ltdl-convenience'

/etc/rc.conf:

hostname="squid"
sshd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
zfs_enable="YES"

background_fsck="NO"

defaultrouter="10.16.0.1"
ifconfig_bge3="inet 10.16.2.165/22"

cloned_interfaces="bridge0"
ifconfig_bridge0="addm bge0 addm bge1 up"
ifconfig_bge0="up"
ifconfig_bge1="up"

firewall_enable="YES"
firewall_type="open"
firewall_logging="YES"

squid_enable="YES"
squid_user="root"

During the tests I add ipfw rules manually. So it looks like this:

00100  0  0 allow ip from any to any via lo0
00200  0  0 deny ip from any to 127.0.0.0/8
00300  0  0 deny ip from 127.0.0.0/8 to any
00400  0  0 deny ip from any to ::1
00500  0  0 deny ip from ::1 to any
00600  0  0 allow ipv6-icmp from :: to ff02::/16
00700  0  0 allow ipv6-icmp from fe80::/10 to fe80::/10
00800  0  0 allow ipv6-icmp from fe80::/10 to ff02::/16
00900  0  0 allow ipv6-icmp from any to any ip6 icmp6types 1
01000  0  0 allow ipv6-icmp from any to any ip6 icmp6types 2,135,136
01100 82 8498 fwd 127.0.0.1,8080 log logamount 50 ip from any to any dst-port 80 in recv bge0
01101 56  30724 fwd 127.0.0.1 log logamount 50 ip from any 80 to any in recv bge1
65000 17790 1820248 allow ip from any to any
65535  0  0 deny ip from any to any

sysctl -a | grep forward:

kern.smp.forward_signal_enabled: 1
net.inet.ip.forwarding: 1
net.inet.ip.fastforwarding: 0
net.inet6.ip6.forwarding: 0

kldstat:

Id Refs Address Size  Name
1  16 0xffffffff80200000 17627c8 kernel
2 1 0xffffffff81963000 2f94a0  zfs.ko
3 2 0xffffffff81c5d000 6040  opensolaris.ko
4 1 0xffffffff81e11000 939c  if_bridge.ko
5 1 0xffffffff81e1b000 53f8  bridgestp.ko
6 1 0xffffffff81e21000 11460 ipfw.ko

Please, help me to find out what am I doing wrong! Using ipfw is not mandatory for me.

 

[obsigna]

I got a similar setup running with squid as a transparent proxy. I got one interface re0, which is connected to the internet, and two interfaces em0+em1 forming bridge0 that connects the LAN. NAT and firewall is provided by ipfw(8). Your ipfw fwd rules are completely questionable - bge0/bge1 aren't configured for IP on there own, these interfaces are members of bridge0. I got only one rule:

/sbin/ipfw -q add 80 fwd 127.0.0.1,3128 tcp from 192.168.0.0/16 to any dst-port 80

I could have limited this rule to the bridge interface bridge0 but it is not exactly necessary.

With that ipfw rule in place, my file /usr/local/etc/squid/squid.conf got the following two related entries:

...
http_port        192.168.0.1:3127
http_port        127.0.0.1:3128 intercept
...

That's all. Good luck.

 

[Mack]

Thank you!
My 1100 rule is almost the same. As soon as you have NAT and Squid intercept mode, you don't need "backward" rule (like my 1101), and it's working for you.
I was trying to assign this rules to the bridge, but it also didn't work:

01100  19  1239 fwd 127.0.0.1,8080 log logamount 50 ip from 192.168.1.10 to any dst-port 80 via bridge0
01101  71 105027 fwd 127.0.0.1 log logamount 50 ip from any 80 to 192.168.1.10 via bridge0

 

[Mack]

Just tried another way with netgraph:

#!/bin/sh
kldload -v ng_ether
kldload -v ng_bpf
kldload -v ng_eiface
ngctl mkpeer bge0: bpf lower bge0-lower
ngctl name bge0:lower br-bpf
ngctl connect bge0: br-bpf: upper bge0-upper
ngctl connect bge1: br-bpf: lower bge1-lower
ngctl connect bge1: br-bpf: upper bge1-upper
ngctl mkpeer br-bpf: eiface squid ether

  PATTERN="tcp dst port 80"
  INHOOK="bge0-lower"
  MATCHHOOK="squid"
  NOTMATCHHOOK="bge1-lower"

  BPFPROG=$( tcpdump -s 8192 -ddd ${PATTERN} | grep -v WARNING | \
( read len ; \
echo -n "bpf_prog_len=$len" ; \
echo -n " bpf_prog=[" ; \
while read code jt jf k ; do \
echo -n " { code=$code jt=$jt jf=$jf k=$k }" ; \
done ; \
echo " ]" ) )

  ngctl -d msg br-bpf: setprogram { thisHook=\"${INHOOK}\" \
  ifMatch=\"${MATCHHOOK}\" \
  ifNotMatch=\"${NOTMATCHHOOK}\" \
  ${BPFPROG} }

  PATTERN="tcp src port 80"
  INHOOK="bge1-lower"
  MATCHHOOK="squid"
  NOTMATCHHOOK="bge0-lower"

  BPFPROG=$( tcpdump -s 8192 -ddd ${PATTERN} | \
( read len ; \
echo -n "bpf_prog_len=$len" ; \
echo -n " bpf_prog=[" ; \
while read code jt jf k ; do \
echo -n " { code=$code jt=$jt jf=$jf k=$k }" ; \
done ; \
echo " ]" ) )

  ngctl msg br-bpf: setprogram { thisHook=\"${INHOOK}\" \
  ifMatch=\"${MATCHHOOK}\" \
  ifNotMatch=\"${NOTMATCHHOOK}\" \
  ${BPFPROG} }

ngctl msg bge0: setpromisc 1
ngctl msg bge1: setpromisc 1
ngctl msg bge0: setautosrc 0
ngctl msg bge1: setautosrc 0

Squid was reconfigured to listen to port 80:

2015/05/26 16:19:04.349 kid1| Accepting TPROXY intercepted HTTP Socket connections at local=0.0.0.0:80 remote=[::] FD 17 flags=25

So, I can see:

tcpdump -i ngeth0 -vvv -s0 tcp dst port 80
tcpdump: listening on ngeth0, link-type EN10MB (Ethernet), capture size 65535 bytes
17:17:09.767118 IP (tos 0x0, ttl 128, id 921, offset 0, flags [DF], proto TCP (6), length 52)
    192.168.1.10.25055 > 192.168.1.20.http: Flags , cksum 0x8521 (correct), seq 2363512694, win 8192, options [mss 1460,nop,wscale 2,nop,nop,sackOK], length 0
17:17:12.231562 IP (tos 0x0, ttl 128, id 924, offset 0, flags [DF], proto TCP (6), length 52)
    192.168.1.10.25055 > 192.168.1.20.http: Flags , cksum 0x8521 (correct), seq 2363512694, win 8192, options [mss 1460,nop,wscale 2,nop,nop,sackOK], length 0

But cache.log is still empty. So, the result looks similar. Is smth wrong with my Squid?