Solved How to fix Bug 185967 on FreeBSD 10.1

fred974

fred974

Hello all,

I updated my FreeBSD 10.0-RELEASE-p12 to FreeBSD 10.1 and since then the network is down. I use lagg(4) with the lacp aggregation protocols.

As lagg is not enabled in the GENERIC kernel, I recompiled the kernel with the following command: make clean && make buildkernel KERNCONF=CUSTOMKERN && make installkernel KERNCONF=CUSTOMKERN && make clean && sync

CUSTOMKERN file is:
Code: 
ident CUSTOMKERN
cpu   HAMMER

# change kernel message color to green on black
# while all other text is white on black
options SC_KERNEL_CONS_ATTR=(FG_GREEN|BG_BLACK)

# enable Pf without ALTQ (HFSC)
device pf
device pflog
device pfsync
options ALTQ
options ALTQ_HFSC
options ALTQ_NOPCC

# forward packets without decrementing
# the time to live (TTL) counter
options IPSTEALTH

# enable lagg interface
device lagg

# IPsec (Internet Protocol Security)
options IPSEC
options IPSEC_NAT_T
device  crypto

# Crypto and AES-NI support
# also add aesni_load="YES" to /boot/loader.conf
# verify AES speeds before and after with "openssl speed -evp aes-256-cbc -elapsed"
# as /dev/crypto can be slower then no /dev/crypto on some systems. Also, the
# ports version of /usr/local/bin/openssl is significantly faster then the
# default /usr/bin/openssl version which comes with freebsd.
device crypto
device cryptodev
device aesni

#############################################

makeoptions DEBUG=-g        # Build kernel with gdb(1) debug symbols
makeoptions WITH_CTF=1      # Run ctfconvert(1) for DTrace support

options     SCHED_ULE       # ULE scheduler
options     PREEMPTION      # Enable kernel thread preemption
options     INET            # InterNETworking
options     INET6           # IPv6 communications protocols
options     TCP_OFFLOAD     # TCP offload
options     SCTP            # Stream Control Transmission Protocol
options     FFS             # Berkeley Fast Filesystem
options     SOFTUPDATES     # Enable FFS soft updates support
options     UFS_ACL         # Support for access control lists
options     UFS_DIRHASH     # Improve performance on big directories
options     UFS_GJOURNAL    # Enable gjournal-based UFS journaling
options     QUOTA           # Enable disk quotas for UFS
options     MD_ROOT         # MD is a potential root device
options     NFSCL           # New Network Filesystem Client
options     NFSD            # New Network Filesystem Server
options     NFSLOCKD        # Network Lock Manager
options     NFS_ROOT        # NFS usable as /, requires NFSCL
options     MSDOSFS         # MSDOS Filesystem
options     CD9660          # ISO 9660 Filesystem
options     PROCFS          # Process filesystem (requires PSEUDOFS)
options     PSEUDOFS        # Pseudo-filesystem framework
options     GEOM_PART_GPT   # GUID Partition Tables.
options     GEOM_RAID       # Soft RAID functionality.
options     GEOM_LABEL      # Provides labelization
options     COMPAT_FREEBSD32    # Compatible with i386 binaries
options     COMPAT_FREEBSD4     # Compatible with FreeBSD4
options     COMPAT_FREEBSD5     # Compatible with FreeBSD5
options     COMPAT_FREEBSD6     # Compatible with FreeBSD6
options     COMPAT_FREEBSD7     # Compatible with FreeBSD7
options     SCSI_DELAY=5000     # Delay (in ms) before probing SCSI
options     KTRACE          # ktrace(1) support
options     STACK           # stack(9) support
options     SYSVSHM         # SYSV-style shared memory
options     SYSVMSG         # SYSV-style message queues
options     SYSVSEM         # SYSV-style semaphores
options     _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options     PRINTF_BUFR_SIZE=128    # Prevent printf output being interspersed.
options     KBD_INSTALL_CDEV    # install a CDEV entry in /dev
options     HWPMC_HOOKS     # Necessary kernel hooks for hwpmc(4)
options     AUDIT           # Security event auditing
options     CAPABILITY_MODE     # Capsicum capability mode
options     CAPABILITIES        # Capsicum capabilities
options     PROCDESC        # Support for process descriptors
options     MAC         # TrustedBSD MAC Framework
options     KDTRACE_FRAME       # Ensure frames are compiled in
options     KDTRACE_HOOKS       # Kernel DTrace hooks
options     DDB_CTF         # Kernel ELF linker loads CTF data
options     INCLUDE_CONFIG_FILE     # Include this file in kernel

# Debugging support.  Always need this:
options     KDB             # Enable kernel debugger support.
options     KDB_TRACE       # Print a stack trace for a panic.

# Make an SMP-capable kernel by default
options     SMP             # Symmetric MultiProcessor Kernel

# CPU frequency control
device      cpufreq

# Bus support.
device      acpi
device      pci

# ATA controllers
device      ahci        # AHCI-compatible SATA controllers
device      ata     # Legacy ATA/SATA controllers

# ATA/SCSI peripherals
device      scbus       # SCSI bus (required for ATA/SCSI)
device      da      # Direct Access (disks)
device      cd      # CD

# RAID controllers
device          mfi             # LSI MegaRAID SAS

# atkbdc0 controls both the keyboard and the PS/2 mouse
device      atkbdc      # AT keyboard controller
device      atkbd       # AT keyboard
device      ums     # PS/2 mouse

device      kbdmux      # keyboard multiplexer

device      vga     # VGA video card driver
options     VESA        # Add support for VESA BIOS Extensions (VBE)

device      splash      # Splash screen and screen saver support

# syscons is the default console driver, resembling an SCO console
device      sc
options     SC_PIXEL_MODE   # add support for the raster text mode

device      agp     # support several AGP chipsets

# PCI Ethernet NICs.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device          miibus          # MII bus support
#device      em      # Intel PRO/1000 Gigabit Ethernet Family
#device      igb     # Intel PRO/1000 PCIE Server Gigabit Family
#device      ixgbe       # Intel PRO/10GbE PCIE Ethernet Family
device      bce     # Broadcom NetXtreme II BCM5709 Gigabit Ethernet Family

# Pseudo devices.
device      loop        # Network loopback
device      random      # Entropy device
#device     padlock_rng # VIA Padlock RNG
device      rdrand_rng  # Intel Bull Mountain RNG
device      ether       # Ethernet support
device      vlan        # 802.1Q VLAN support
device      tun     # Packet tunnel.
device      md      # Memory "disks"
device      gif     # IPv6 and IPv4 tunneling
device      faith       # IPv6-to-IPv4 relaying (translation)
device      firmware    # firmware assist module

# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device      bpf     # Berkeley packet filter

# USB support
options     USB_DEBUG   # enable debug msgs
device      uhci        # UHCI PCI-USB interface
device      ohci        # OHCI PCI-USB interface
device      ehci        # EHCI PCI-USB interface (USB 2.0)
device      xhci        # XHCI PCI-USB interface (USB 3.0)
device      usb     # USB Bus (required)
device      ukbd        # Keyboard
device      umass       # Disks/Mass storage - Requires scbus and da
If I remove the lagg and simply set the following then it works ok
Code: 
ifconfig_bce0="inet 192.168.0.175 netmask 255.255.255.0"
defaultrouter="192.168.1.1"
This is my existing setup
Code: 
ifconfig_bce0="up"
ifconfig_bce1="up"
defaultrouter="192.168.1.1"
cloned_interfaces="lagg0"
ifconfig_lagg0="inet 192.168.0.175 netmask 255.255.255.0 laggproto lacp laggport bce0 laggport bce1"
 
Hi everyone,

It look like bug Bug 185967 still exists in version 10.1. On FreeBSD 10.0, I resolved the problem with the following patch but it doesn't work on FreeBSD 10.1:
Code: 
Index: ieee8023ad_lacp.c
===================================================================
--- ieee8023ad_lacp.c    (revision 261432)
+++ ieee8023ad_lacp.c    (working copy)
@@ -192,6 +192,11 @@
SYSCTL_INT(_net_link_lagg_lacp, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_TUN,
     &lacp_debug, 0, "Enable LACP debug logging (1=debug, 2=trace)");
TUNABLE_INT("net.link.lagg.lacp.debug", &lacp_debug);
+static int lacp_strict = 0;
+SYSCTL_INT(_net_link_lagg_lacp, OID_AUTO, lacp_strict_mode,
+    CTLFLAG_RW | CTLFLAG_TUN, &lacp_strict, 0,
+    "Enable LACP strict protocol compliance");
+TUNABLE_INT("net.link.lagg.lacp.lacp_strict_mode", &lacp_strict);

#define LACP_DPRINTF(a) if (lacp_debug & 0x01) { lacp_dprintf a ; }
#define LACP_TRACE(a) if (lacp_debug & 0x02) { lacp_dprintf(a,"%s\n",__func__); }
@@ -791,7 +796,7 @@

     lsc->lsc_hashkey = arc4random();
     lsc->lsc_active_aggregator = NULL;
-    lsc->lsc_strict_mode = 1;
+    lsc->lsc_strict_mode = lacp_strict;
     LACP_LOCK_INIT(lsc);
     TAILQ_INIT(&lsc->lsc_aggregators);
     LIST_INIT(&lsc->lsc_ports);
 
You don't really need to go through building a new kernel just to add lagg support. Just add the following to /boot/loader.conf:
Code: 
if_lagg_load="YES"
When you say the patch didn't work in 10.1, did it just not apply, or did you make the change, recompile, and it still wouldn't work correctly.
 
Hi usdmatt,
Thank you for your reply. I already have lagg support in my /boot/loader.conf

usdmatt said:
When you say the patch didn't work in 10.1, did it just not apply, or did you make the change, recompile, and it still wouldn't work correctly.
Click to expand...

Basically when I rebooted the system, I realized that my web server stopped working and SSH to the server was impossible. On my FreeBSD 10.0-RELEASE-p12 version, I was able to use lagg(4)() with failover but not with lacp support until I apply the patch I mentioned in my first post. So after updating FreeBSD I have applied the patch again as root using
Code: 
cd /usr/src/sys/net/
patch -C < /home/suadmin/lacp.patch
And then recompiled my kernel.

I am still unable to do reach the server via SSH or OpenVPN. The only think that allows me to debug the server is that I have remote access to the Dell IDRAC.

Any idea on how to solve my problem?

Thank you
 
1) As lagg is not enabled in the GENERIC kernel, I recompiled the kernel with the following command.
2) I already have lagg support in my /boot/loader.conf
Click to expand...

Why did you recompile then? To add lagg support, you either need to add the device to the kernel (which I think is a waste of time), or load the module dynamically on boot. You don't need to do both.

Did the patch definitely apply without error? Can you confirm that after applying the patch, the below line has actually changed in your source file:

Code: 
-    lsc->lsc_strict_mode = 1;
+    lsc->lsc_strict_mode = lacp_strict;

This line is around ~768 by the look of it in recent copies of that file. As your patch is looking around line 791, it might not be applying correctly. I don't use patch much, but I'd expect it to complain if the lines it's looking for are not where the patch file says they should be. Usually you'd want the patch to be made for the specific version of the file you're using, whereas in this case the patch is made for the file from 10.0, and you're trying to apply to 10.1.

All it's really doing is turning strict mode off by changing that 1 to a 0. However, instead of hard coding 0, it's using a variable and allowing you to modify that variable at runtime using the sysctl capability.

Of course as mentioned in the bug details, strict mode should work. The only difference is it causes FreeBSD to wait for a response from the device on the other end of the cable before bringing the lacp link up. If configured correctly, your switch (or whatever's on the other end) should really be sending that response.
 
usdmatt, it looks like you were right. The patch didn't apply. This is what I have in the /usr/src/sys/net/ieee8023ad_lacp.c file
Code: 
 794  lsc->lsc_strict_mode = 1;
Now that we identify that the patch is not working for this version of FreeBSD, do you have any idea how to adapt the patch to make it work? This is now way out of my league, as my knowledge of coding is really poor.

Should I do a hard coding 0?
Code: 
lsc->lsc_strict_mode = 0;

Thank you,

Fred
 
Yeah, just setting that to 0 should do what you want. Of course you'll have to remember to do the same thing in future if you ever want to upgrade.

Have you definitely made sure that there's no setting on the other end of the link that will make it work with FreeBSD's strict mode? That would be the ideal way to fix it as it would require no change to FreeBSD source. Other than that you'll just have to hope that the developers decide to actually make this strict mode a tuneable in future.
 
Changed the code from
Code: 
lsc->lsc_strict_mode = 1;
to
Code: 
lsc->lsc_strict_mode = 0;
but the network is still down :(
 
If you've rebuilt the kernel with that setting changed and still have the problem then the issue in the bug report isn't your problem (or not your only problem at least).

What does ifconfig look like when the machine is up, but not accessible over the network?
 
This is a stupid mistake that cost me 2 days...
Code: 
lagg Failover Mode
ifconfig_bce0="up"
ifconfig_bce1="up"
idefaultrouter="192.168.1.1"
cloned_interfaces="lagg0"
# LACP
ifconfig_lagg0="inet 192.168.1.185 netmask 255.255.255.0 laggproto lacp laggport bce0 laggport bce1"

When using the vi editor I inserted 'i' in the front of idefaultrouter="192.168.1.1".

Thank you for all your help usdmatt
 
You must log in or register to reply here.
Back
Top