Postfix + Dovecot + Rspamd not learning SPAM / HAM

Remington

Well-Known Member

Reaction score: 145
Messages: 489

I rebuilt my mail server using this guide https://www.c0ffee.net/blog/mail-server-guide/ but it seems that it's not learning SPAM / HAM manually when I mark or unmark email as spam as the spam score has not changed. Everything else works as it can detect email spam and move it into spam folder automatically. Has anyone run into this problem with Rspamd not learning manually?

I'm running FreeBSD 11.2 and mail server is in Jail.
 
OP
OP
Remington

Remington

Well-Known Member

Reaction score: 145
Messages: 489

How and where are you marking this?
When I move email from Inbox to Junk folder the spam score doesn't change. I checked Dovecot and Rspamd log and the imapsieve correctly executed the learn-spam.sh and Rspamd said email was learned as spam but the score didn't change. My friend is using the same configuration on Linux server and it works fine. Not sure why its not working in my case with FreeBSD server.
 
OP
OP
Remington

Remington

Well-Known Member

Reaction score: 145
Messages: 489

Here's my Dovecot and Rspamd confs:

Dovecot.conf
Code:
# 2.3.4 (0ecbaf23d): /usr/local/etc/dovecot/dovecot.conf
# Pigeonhole version 0.5.4 (60b0f48d)
# OS: FreeBSD 11.2-RELEASE amd64
# Hostname: localhost
auth_cache_size = 10 M
auth_verbose = yes
first_valid_uid = 5000
imap_idle_notify_interval = 29 mins
info_log_path = /var/log/dovecot-info.log
last_valid_uid = 5000
log_path = /var/log/dovecot.log
mail_fsync = never
mail_home = /usr/local/vmail/%d/%n
mail_location = maildir:~
mail_privileged_group = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy include variables body enotify environment mailbox date index ihave duplicate mime foreverypart extracttext imapsieve vnd.dovecot.imapsieve
namespace inbox {
  inbox = yes
  location =
  mailbox Archive {
    auto = subscribe
    special_use = \Archive
  }
  mailbox Drafts {
    auto = subscribe
    special_use = \Drafts
  }
  mailbox Junk {
    auto = create
    special_use = \Junk
  }
  mailbox Sent {
    auto = subscribe
    special_use = \Sent
  }
  mailbox Trash {
    auto = create
    special_use = \Trash
  }
  prefix =
  separator = /
}
passdb {
  args = /usr/local/etc/dovecot/dovecot-sql.conf.ext
  driver = sql
}
plugin {
  imapsieve_mailbox1_before = file:/usr/local/vmail/sieve/global/learn-spam.sieve
  imapsieve_mailbox1_causes = COPY
  imapsieve_mailbox1_name = Junk
  imapsieve_mailbox2_before = file:/usr/local/vmail/sieve/global/learn-ham.sieve
  imapsieve_mailbox2_causes = COPY
  imapsieve_mailbox2_from = Junk
  imapsieve_mailbox2_name = *
  recipient_delimiter = +
  sieve = file:~/sieve;active=~/sieve/.dovecot.sieve
  sieve_before = /usr/local/vmail/sieve/global/spam-global.sieve
  sieve_global_extensions = +vnd.dovecot.pipe
  sieve_pipe_bin_dir = /usr/local/vmail/sieve/global
  sieve_plugins = sieve_imapsieve sieve_extprograms
  sieve_quota_max_storage = 50M
}
protocols = imap lmtp sieve
service auth-worker {
  user = vmail
}
service auth {
  client_limit = 2000
  unix_listener /var/spool/postfix/private/auth {
    group = postfix
    mode = 0660
    user = postfix
  }
}
service imap-login {
  inet_listener imap {
    address = 192.168.8.3, ::1
  }
  process_min_avail = 3
  service_count = 0
  vsz_limit = 1 G
}
service imap {
  process_min_avail = 3
  service_count = 256
}
service lmtp {
  unix_listener /var/spool/postfix/private/dovecot-lmtp {
    group = postfix
    mode = 0600
    user = postfix
  }
}
service managesieve-login {
  inet_listener sieve {
    port = 4190
  }
  process_min_avail = 0
  service_count = 1
  vsz_limit = 64 M
}
service managesieve {
  process_limit = 1024
}
service pop3-login {
  inet_listener pop3 {
    port = 0
  }
  inet_listener pop3s {
    port = 0
  }
}
ssl = required
ssl_cert = </etc/ssl/acme/star_acme_com.crt
ssl_dh = # hidden, use -P to show it
ssl_key = # hidden, use -P to show it
ssl_prefer_server_ciphers = yes
userdb {
  args = /usr/local/etc/dovecot/dovecot-sql.conf.ext
  driver = sql
}
protocol lmtp {
  mail_fsync = optimized
  mail_plugins = $mail_plugins sieve
}
protocol lda {
  mail_fsync = optimized
  mail_plugins = $mail_plugins sieve
}
protocol sieve {
  mail_max_userip_connections = 10
  managesieve_implementation_string = Dovecot Pigeonhole
  managesieve_max_compile_errors = 5
  managesieve_max_line_length = 65536
}
protocol imap {
  mail_max_userip_connections = 50
  mail_plugins = $mail_plugins imap_sieve
}
Rspamd worker config
Code:
*** Section worker ***
normal {
    task_timeout = 8;
    enabled = false;
    bind_socket = "192.168.8.3:11333";
    mime = true;
}

*** End of section worker ***
*** Section worker ***
controller {
    password = "*** deleted ***";
    secure_ip = "192.168.8.3";
    enable_password = "*** deleted ***";
    static_dir = "/usr/local/share/rspamd/www";
    count = 1;
    bind_socket = "/var/run/rspamd/rspamd.sock mode=0666 owner=nobody";
    bind_socket = "192.168.8.3:11334";
}

*** End of section worker ***
*** Section worker ***
rspamd_proxy {
    max_retries = 5;
    timeout = 120;
    spam_header = "X-Spam";
    quarantine_on_reject = false;
    reject_message = "Spam message rejected";
    discard_on_reject = false;
    milter = true;
    upstream {
        local {
            hosts = "localhost";
            self_scan = true;
            default = true;
        }
    }
    bind_socket = "/var/run/rspamd/milter.sock mode=0666 owner=nobody";
    count = 1;
}

*** End of section worker ***
*** Section worker ***
fuzzy {
    backend = "redis";
    allow_update [
        "localhost",
    ]
    count = -1;
    bind_socket = "192.168.8.3:11335";
    expire = 7776000;
}

*** End of section worker ***
Rspamd classifier config
Code:
*** Section classifier ***
bayes {
    backend = "redis";
    min_tokens = 11;
    languages_enabled = true;
    cache {
        path = "/var/db/rspamd/learn_cache.sqlite";
    }
    expire = 31536000;
    new_schema = true;
    statfile {
        path = "/var/db/rspamd/bayes.ham.sqlite";
        spam = false;
        symbol = "BAYES_HAM";
    }
    statfile {
        path = "/var/db/rspamd/bayes.spam.sqlite";
        spam = true;
        symbol = "BAYES_SPAM";
    }
    autolearn = true;
    tokenizer {
        name = "osb";
    }
    learn_condition = <<EOD
return function(task, is_spam, is_unlearn)
  local learn_type = task:get_request_header('Learn-Type')

  if not (learn_type and tostring(learn_type) == 'bulk') then
    local prob = task:get_mempool():get_variable('bayes_prob', 'double')

    if prob then
      local in_class = false
      local cl
      if is_spam then
        cl = 'spam'
        in_class = prob >= 0.95
      else
        cl = 'ham'
        in_class = prob <= 0.05
      end

      if in_class then
        return false,string.format('already in class %s; probability %.2f%%',
          cl, math.abs((prob - 0.5) * 200.0))
      end
    end
  end

  return true
end
EOD;
    min_learns = 200;
}

*** End of section classifier ***
learn-spam.sieve
Code:
require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];

if environment :matches "imap.email" "*" {
  set "email" "${1}";
}

pipe :copy "learn-spam.sh" [ "${email}" ];
learn-ham.sieve
Code:
require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];

if environment :matches "imap.mailbox" "*" {
  set "mailbox" "${1}";
}

if string "${mailbox}" "Trash" {
  stop;
}

if environment :matches "imap.email" "*" {
  set "email" "${1}";
}

pipe :copy "learn-ham.sh" [ "${email}" ];
learn-spam.sh
Code:
exec /usr/local/bin/rspamc -h /var/run/rspamd/rspamd.sock learn_spam
learn-ham.sh
Code:
exec /usr/local/bin/rspamc -h /var/run/rspamd/rspamd.sock learn_ham
 

xtaz

Well-Known Member

Reaction score: 104
Messages: 409

I looked at doing this on my own server and left out the configuration for doing this specifically because I didn't believe it would work on FreeBSD. From memory I think the problem is that this depends on having the pipe extension installed in dovecot-pigeonhole which isn't a standard extension, and when I looked in to it there was no pkg or port that could be installed to add it. That may have changed, but I suspect this is the problem. The pipe is simply being ignored.

Basically you need this: https://wiki2.dovecot.org/Pigeonhole/Sieve/Plugins/Pipe , although that page now says that it's been superseded by this: https://wiki2.dovecot.org/Pigeonhole/Sieve/Plugins/Extprograms which interestingly says that it's part of the v0.4 release of pigeonhole.
 
OP
OP
Remington

Remington

Well-Known Member

Reaction score: 145
Messages: 489

Manual HAM/SPAM learning is working now after 2 months of trainings due to low volume of emails I received.
 
Top