Difference between revisions of "Rspamd"

m (Spam Training)
m (Spam Training)
Line 172: Line 172:
 
The zmtrainsa script has been modified to remove (some of) the DSPAM & SA training, I've added a section for training the rspamd system from the same spam/ham accounts in ZCS so users continue using the 'Spam' & 'Not Spam' function in the ZCS Web UI.  
 
The zmtrainsa script has been modified to remove (some of) the DSPAM & SA training, I've added a section for training the rspamd system from the same spam/ham accounts in ZCS so users continue using the 'Spam' & 'Not Spam' function in the ZCS Web UI.  
  
This is my modified 'new' zmtrainsa script below:
+
  #
 +
# This is my modified zmtrainsa script which I currently use.
 +
#
 +
 +
# Make a copy the original zmtrainsa script then replace the origonal with this modified script,
 +
# that means you don't have to modify the scheduled cron jobs for zmtrainsa.
 +
# You will need to disable the cron jobs for SA learning and rules update
 
   
 
   
 +
# This script is also run after the training to clean-up the directories and we'll
 +
# use the new script for that as well
 
  #
 
  #
  # This is my modified file which seems to work, so far.
+
  # eventually we can move the ZCS zmtrainsa to a backup then use our 'new' training script
  # This has further SA & DSPAM bits disabled in here.
+
  # which must be updated after every ZCS update
 
  #
 
  #
 +
# NB DSPAM is no longer supported or installed,
 +
# I have removed the parts of this script that relate to DSPAM & Spamassassin.
 +
 
  #!/bin/bash
 
  #!/bin/bash
  #
+
  #  
 
  # ***** BEGIN LICENSE BLOCK *****
 
  # ***** BEGIN LICENSE BLOCK *****
 
  # Zimbra Collaboration Suite Server
 
  # Zimbra Collaboration Suite Server
Line 192: Line 203:
 
  # See the GNU General Public License for more details.
 
  # See the GNU General Public License for more details.
 
  # You should have received a copy of the GNU General Public License along with this program.
 
  # You should have received a copy of the GNU General Public License along with this program.
  # If not, see [add license address].
+
  # If not, see <gnu dot org  licenses >.
 
  # ***** END LICENSE BLOCK *****
 
  # ***** END LICENSE BLOCK *****
  #
+
  #  
 
+
  autoTrainSystem() {  
+
  autoTrainSystem() {  
 
+
 
   timestampit "Starting spam/ham extraction from system accounts."
 
   timestampit "Starting spam/ham extraction from system accounts."
 
    
 
    
 
  # This is the section for extracting the email to a
 
  # This is the section for extracting the email to a
 
  # couple of temp directories for spam & ham
 
  # couple of temp directories for spam & ham
  #
+
   
 
   spamdir=`mktemp -d -t spam.XXXXXXX` || exit 1
 
   spamdir=`mktemp -d -t spam.XXXXXXX` || exit 1
 
   hamdir=`mktemp -d -t ham.XXXXXXX` || exit 1
 
   hamdir=`mktemp -d -t ham.XXXXXXX` || exit 1
Line 209: Line 220:
 
   timestampit "Finished extracting spam/ham from system accounts."
 
   timestampit "Finished extracting spam/ham from system accounts."
 
   
 
   
# timestampit "Starting spamassassin training."
 
 
# This is the actual 'training' section for spamassassin
 
#
 
#  /opt/zimbra/common/bin/sa-learn \
 
#    --dbpath=${db_path} -L --no-sync \
 
#    --spam ${spamdir}
 
# /opt/zimbra/common/bin/sa-learn \
 
#  --dbpath=${db_path} -L --no-sync \
 
#  --ham ${hamdir}
 
 
 
# /opt/zimbra/common/bin/sa-learn \
 
#  --dbpath=${db_path} --sync
 
# end of the spamassassin 'training' section
 
#
 
# timestampit "Finished spamassassin training."
 
#
 
# NB DSPAM is no longer supported or installed
 
# if [ x"$amavis_dspam_enabled" = "xtrue" ]; then
 
#  timestampit "Starting dspam training"
 
#  /opt/zimbra/dspam/bin/dspam_train zimbra ${spamdir} ${hamdir}
 
#  #for f in ${spamdir}/*; do
 
#    #test -f ${f} && /opt/zimbra/dspam/bin/dspam_corpus --addspam zimbra ${f}
 
#  #done
 
#  #for f in ${hamdir}/*; do
 
#    #test -f ${f} && /opt/zimbra/dspam/bin/dspam_corpus zimbra ${f}
 
#  #done
 
 
#  /opt/zimbra/dspam/bin/dspam_clean -p0 $USER
 
#  timestampit "Finished dspam training"
 
 
# fi
 
# end of the DSPAM training session
 
#
 
 
# This is the actual section for rspamd training
 
    timestampit "Starting rspamd training."
 
# Let's do a test here to see if rspamc is doing it's thing!
 
#
 
# List some stats before training
 
# the passwords for these rspamc commands need to
 
# be changed for your server
 
 
 
  timestampit "List rspam stats before training."
 
  timestampit "List rspam stats before training."
 
  rspamc -h 192.168.1.30:11334 -P apassword stat
 
  rspamc -h 192.168.1.30:11334 -P apassword stat
Line 264: Line 232:
 
  timestampit "List rspam stats after training."
 
  timestampit "List rspam stats after training."
 
  rspamc -h 192.168.1.30:11334 -P apassword stat
 
  rspamc -h 192.168.1.30:11334 -P apassword stat
    timestampit "Finished rspamd training."
+
timestampit "Finished rspamd training."
 
  # End of the rspamd training section
 
  # End of the rspamd training section
 
+
 
  # the following is the rest of the original file
 
  # the following is the rest of the original file
 
   /bin/rm -rf ${spamdir} ${hamdir}
 
   /bin/rm -rf ${spamdir} ${hamdir}
 
 
}
 
 
trainAccountFolder() {
 
 
  if [ "x${MODE}" = "xspam" ]; then
 
    tempdir=`mktemp -d -t spam.XXXXXX` || exit 1
 
    FOLDER=${FOLDER:=junk}
 
  elif [ "x${MODE}" = "xham" ]; then
 
    tempdir=`mktemp -d -t ham.XXXXXX` || exit 1
 
    FOLDER=${FOLDER:=inbox}
 
  fi
 
 
  timestampit  "Starting spamassassin $MODE training for $USER using folder $FOLDER"
 
  /opt/zimbra/libexec/zmspamextract -r -m $USER -o ${tempdir} -q in:${FOLDER} 
 
 
  /opt/zimbra/common/bin/sa-learn \
 
    --dbpath=${db_path} -L --no-sync \
 
    --${MODE} ${tempdir}
 
 
  /opt/zimbra/common/bin/sa-learn \
 
    --dbpath=${db_path} --sync
 
  timestampit  "Finished spamassassin $MODE training for $USER using folder $FOLDER"
 
 
  if [ x"$amavis_dspam_enabled" = "xtrue" ]; then
 
    timestampit  "Starting dspam $MODE training for $USER using folder $FOLDER"
 
    if [ $MODE == "ham" ]; then
 
      MODE="innocent"
 
    fi
 
 
    /opt/zimbra/dspam/bin/dspam --user zimbra --class=${MODE} --source=corpus --mode=teft \
 
      --feature=noise --stdout
 
 
    /opt/zimbra/dspam/bin/dspam_clean -p0 $USER
 
    timestampit  "Finished dspam $MODE training for $USER using folder $FOLDER"
 
  fi
 
 
  /bin/rm -rf ${tempdir}
 
 
   
 
   
 
  }
 
  }
Line 314: Line 244:
 
   echo "$SIMPLE_DATE $1"
 
   echo "$SIMPLE_DATE $1"
 
  }
 
  }
 
usage() {
 
  echo "Usage: $0 <user> <spam|ham> [folder]"
 
  exit 1
 
}
 
 
if [ x`whoami` != xzimbra ]; then
 
    echo Error: must be run as zimbra user
 
  exit 1
 
fi
 
 
if [ ! -x "/opt/zimbra/common/sbin/amavisd" ]; then
 
    echo "Error: SpamAssassin not installed"
 
    exit 1
 
fi
 
 
source `dirname $0`/zmshutil || exit 1
 
zmsetvars
 
 
amavis_dspam_enabled=`/opt/zimbra/bin/zmprov -l gs ${zimbra_server_hostname} zimbraAmavisDSPAMEnabled | grep zimbraAmavisDSPAMEnabled: | awk  '{print $2}'`
 
amavis_dspam_enabled=$(echo $amavis_dspam_enabled | tr A-Z a-z)
 
antispam_mysql_enabled=$(echo $antispam_mysql_enabled | tr A-Z a-z)
 
zmtrainsa_cleanup_host=$(echo $zmtrainsa_cleanup_host | tr A-Z a-z)
 
 
if [ "x${zimbra_spam_externalIsSpamAccount}" = "x" ]; then
 
  spam_account="-s"
 
else
 
  spam_account="-m ${zimbra_spam_externalIsSpamAccount}"
 
fi
 
 
if [ "x${zimbra_spam_externalIsNotSpamAccount}" = "x" ]; then
 
  ham_account="-n"
 
else
 
  ham_account="-m ${zimbra_spam_externalIsNotSpamAccount}"
 
fi
 
 
# Set db_path
 
if [ x"$antispam_mysql_enabled" = "xtrue" ]; then
 
  db_path='/opt/zimbra/data/amavisd/mysql/data'
 
else
 
  db_path='/opt/zimbra/data/amavisd/.spamassassin'
 
fi
 
 
   
 
   
 
  # No argument mode uses zmspamextract for auto-training.
 
  # No argument mode uses zmspamextract for auto-training.
Line 371: Line 259:
 
     /bin/rm -rf ${mydir}
 
     /bin/rm -rf ${mydir}
 
     timestampit "Finished spam/ham cleanup"
 
     timestampit "Finished spam/ham cleanup"
   else
+
   else  
 
     timestampit "Cleanup skipped: $zimbra_server_hostname is not a spam/ham cleanup host."
 
     timestampit "Cleanup skipped: $zimbra_server_hostname is not a spam/ham cleanup host."
 
   fi
 
   fi
Line 377: Line 265:
 
  fi
 
  fi
 
   
 
   
  USER=$1  
+
  USER=$1
  MODE=`echo $2 | tr A-Z a-z`  
+
  MODE=`echo $2 | tr A-Z a-z`
 
  FOLDER=$3
 
  FOLDER=$3
 
   
 
   
Line 386: Line 274:
 
   
 
   
 
  if [ "x${USER}" = "x" ]; then
 
  if [ "x${USER}" = "x" ]; then
  usage
+
  usage
 
  fi
 
  fi
 
   
 
   

Revision as of 10:03, 27 October 2017

RSPAMD - Fast, free and open-source spam filtering system

   KB 23536        Last updated on 2017-10-27  




0.00
(0 votes)


Introduction

I've been looking for a suitable replacement for DSPAM, a project that was good but died and became unmaintained then totally removed from Zimbra. In recent searches I came across Rspamd which does all that DSPAM did and more and it can completely replace spamassassin - it's lighter, more configurable and more performant than spamassassin. These notes are a brief explanation of how to easily integrate rspamd into a ZCS server and disable spamassassin completely.

You should make these changes on a test server and backup any ZCS & rspamd configuration files that you are going to modify before you modify them - when you're happy that it works you can try it on a live server but again, make sure you backup any modified configuration files.

These are notes for installing rspamd on ZCS 8.7.x (my current version is 8.7.11), I use CentOS and these instructions are for that distribution and the equivalent version of RHEL although installing rspamd on Ubuntu should not be that different.

NB: Rmilter from the Rspamd project and documented on their web site is deprecated and won't be available as a separate package in future, do not use it.

[EDIT] As of today (2017-09-06) I have updated my server to the latest ZCS 8.8.3GA release and these instructions work and are valid for that version as well.

I've been running various versions of Rspamd for the last eight months and have not had any ill effects, although I only run this on my home server there has been a huge reduction in CPU usage from 15-20% to less than 5% for the same levels of inbound/outbound email. I have also removed all DNS & Protocol checks from within the ZCS configuration and the Postscreen settings have been set to their default values – this leads to rspamd processing all inbound email and also further CPU usage reductions.

The current rspamd install now uses an inbuilt milter compatible process for the mail that's sent to it, the anti-spam etc. components of rspamd for processing the mail through it's various modules and pass it back to postfix via the milter process. Do note that this configuration still uses the ZCS inbuilt amavisd & clamav for anti-virus processing.

The multiple-milter function in Postfix is broken by ZCS and only allows one milter to be functional (bugzilla report 97706 ), although that bug seems to require a trivial fix it’s still languishing in bugzilla without any attention for the past two-plus years - please add your comments and vote if you feel it should be implemented.

Configuring Rspamd on the Zimbra Server

You will need to enable the rspamd & epel repositories, further details on the rspamd wiki:

Install Rspamd on Red Hat Enterprise Linux

yum install rspamd redis

Once you've tested and confirmed rspamd is running correctly you can activate redis as it's cache server is used as an efficient key-value storage by many Rspamd modules, further details on the Rspamd wiki.

Modify the ZCS and rspamd config files as shown below

  • Make a copy of the ZCS /opt/zimbra/common/conf/main.cf & /opt/zimbra/conf/zmconfigd.cf files (just in case).
  • We need to set the zimbraMtaSmtpdMilters parameter in LDAP (this change will survive upgrades):
$ su - zimbra
$ zmprov ms $(hostname) zimbraMtaSmtpdMilters "smtpd_milters=inet:localhost:11332"

Check it's been set with this:

$ zmprov gs $(hostname) zimbraMtaSmtpdMilters 

That should give the following result:

zimbraMtaSmtpdMilters: smtpd_milters=inet:localhost:11332

As zmconfigd regularly rebuilds the main.cf file you'll have to make a small hack to the zmconfigd file to stop our changes being overwritten (as mentioned above, details in the ZCS bugzilla report 97706):

Edit the zmconfigd.cf configuration file (still as the zimbra user):

$ vi /opt/zimbra/conf/zmconfigd.cf

Locate and remove the following four lines:

POSTCONF smtpd_milters
if VAR zimbraMtaSmtpdMilters
    POSTCONF smtpd_milters VAR zimbraMtaSmtpdMilters
fi

Save the file and exit, next we need to modify the main.cf file:

vi /opt/zimbra/common/conf/main.cf

Find the following line:

smtpd_milters =

Replace that line with the following lines:

# rspamd changes 
# 
# use a TCP socket
smtpd_milters = inet:localhost:11332
milter_protocol = 6
milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen}
milter_rcpt_macros = i {rcpt_addr}
# skip mail without checks if milter dies  
# milter_default_action = tempfail 
#
# use accept just in case rspamd fails
milter_default_action = accept
#
# end of rspamd changes

Save and exit that file.

The aim is to not use spamassassin so we need to disable it (still as the zimbra user), check the current status of the enabled services:

$ zmprov gs $(zmhostname) | grep zimbraServiceEnabled

The above command will list all the services that are enabled on your server. Next, modify the antispam service with the following commands to disable and stop antispam service:

$ zmprov ms $(zmhostname) -zimbraServiceEnabled antispam
$ zmantispamctl stop
$ zmprov gs $(zmhostname) | grep zimbraServiceEnabled

The antispam service should not now be shown in the list of enabled services.

The LDAP entry for smtpd_milters that we modified earlier survives an upgrade but the following one using postconf does not survive an upgrade nor do the changes to the zmconfigd.cf file that we made earlier, they need to be set after after every upgrade. The postconf setting is this :

$ postconf smtpd_milters=inet:localhost:11332

You can check the current status of the smtpd_milters entry with:

$ postconf | grep smtpd_milters

These are the basic changes you need to get rspamd working and the only rspamd config files you should need to create and modify are the ones below (these are not set by default):

vi /etc/rspamd/local.d/worker-proxy.inc
 
    upstream "local" {
           self_scan = true;
       }

If you want extended headers in messages you need to add just one line to the config file to enable them:

vi /etc/rspamd/local.d/milter_headers.conf

   extended_spam_headers = true;

To send rspamd log message to a file we need to add a local file to the logging:

vi /etc/rspamd/local.d/logging.inc
# Included from top-level .conf file
# enable systemd & console logging
 
#   type = "console";
#   systemd = true;

   type  = "file";
   level = "info";
   filename = "$LOGDIR/rspamd.log";

Save and exit that file.

There is also a web ui available for monitoring and configuration of rspamd, check the web site for further documentation on this feature.

vi /etc/rspamd/local.d/worker-controller.inc
# Included from top-level .conf file
# all we need in here is the user and admin passwords
 
# password for normal user
password = "apassword";    # obviously, your password of choice
  
# password for 'admin'
enable_password = "a-secret-password";  # create this admin password with the rspamd utilities

At this point we can start the rspamd daemon:

systemctl restart rspamd ; systemctl status rspamd -l

Check the log in /var/log/rspamd/rspamd.log to verify all is well.

NB: because of the high number of RBL (and other) DNS lookups, it's advisable to have a caching namesever on your lan (either the ZCS dnscache or your own DNS server, I use PowerDNS), if you don't do this you may get your server IP blacklisted with the RBL sites.

To summarise, you should now have a working rspamd daemon running with your ZCS server and mail passing between them both for processing. The only modification you should need to make after each upgrade are the ones to the zmconfigd.cf file and the postconf command for setting the smtpd_milters socket.

Do not modify any of the config files in the /etc/rspamd directory, please read the Rspamd documentation on the web site for details on which files to modify and how to add config override files to make further modifications to the rspamd config.

What I haven't mentioned is the current feature set in Rspamd and the fact that some of it's other modules could also replace their ZCS equivalents, for more details take a look at the Rspamd web site features page:

Spam Training

Users will still be able to use the Junk & Not Junk button in the Web UI to train the antispam system, the messages in the Junk folder will be passed to rspamd by a slightly modified zmtrainsa script. I suggest that the modified zmtrainsa script is added as a new cron job for training ZCS otherwise you will have to replace the original ZCS supplied script after every update.

The zmtrainsa script has been modified to remove (some of) the DSPAM & SA training, I've added a section for training the rspamd system from the same spam/ham accounts in ZCS so users continue using the 'Spam' & 'Not Spam' function in the ZCS Web UI.

 #
# This is my modified zmtrainsa script which I currently use.
#

# Make a copy the original zmtrainsa script then replace the origonal with this modified script, 
# that means you don't have to modify the scheduled cron jobs for zmtrainsa.
# You will need to disable the cron jobs for SA learning and rules update

# This script is also run after the training to clean-up the directories and we'll
# use the new script for that as well
#
# eventually we can move the ZCS zmtrainsa to a backup then use our 'new' training script
# which must be updated after every ZCS update
#
# NB DSPAM is no longer supported or installed,
# I have removed the parts of this script that relate to DSPAM & Spamassassin.

#!/bin/bash
# 
# ***** BEGIN LICENSE BLOCK *****
# Zimbra Collaboration Suite Server
# Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2012, 2013, 2014, 2015, 2016 Synacor, Inc.
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software Foundation,
# version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
# You should have received a copy of the GNU General Public License along with this program.
# If not, see <gnu dot org  licenses >.
# ***** END LICENSE BLOCK *****
# 

autoTrainSystem() {    

  timestampit "Starting spam/ham extraction from system accounts."
 
# This is the section for extracting the email to a
# couple of temp directories for spam & ham

  spamdir=`mktemp -d -t spam.XXXXXXX` || exit 1
  hamdir=`mktemp -d -t ham.XXXXXXX` || exit 1
  /opt/zimbra/libexec/zmspamextract ${spam_account} -o ${spamdir}
  /opt/zimbra/libexec/zmspamextract ${ham_account} -o ${hamdir}
  timestampit "Finished extracting spam/ham from system accounts."

timestampit "List rspam stats before training."
rspamc -h 192.168.1.30:11334 -P apassword stat

# do the spam directory
rspamc -h 192.168.1.30:11334 -P apassword learn_spam  ${spamdir}

#do the ham directory
rspamc -h 192.168.1.30:11334 -P apassword learn_ham  ${hamdir}

# List some stats after training
timestampit "List rspam stats after training."
rspamc -h 192.168.1.30:11334 -P apassword stat
timestampit "Finished rspamd training."
# End of the rspamd training section

# the following is the rest of the original file
  /bin/rm -rf ${spamdir} ${hamdir}

}

timestampit() {
  SIMPLE_DATE=`date +%Y%m%d%H%M%S`
  echo "$SIMPLE_DATE $1"
}

# No argument mode uses zmspamextract for auto-training.
if [ x$1 = "x" ]; then
  autoTrainSystem
  exit
fi

if [ x$1 = "x--cleanup" ]; then
  if [ x${zmtrainsa_cleanup_host} = "xtrue" ]; then
    timestampit "Starting spam/ham cleanup"
    mydir=`mktemp -d -t cleanup.XXXXXX` || exit 1
    /opt/zimbra/libexec/zmspamextract ${spam_account} -o ${mydir} -d
    /opt/zimbra/libexec/zmspamextract ${ham_account} -o ${mydir} -d
    /bin/rm -rf ${mydir}
    timestampit "Finished spam/ham cleanup"
  else 
    timestampit "Cleanup skipped: $zimbra_server_hostname is not a spam/ham cleanup host."
  fi
  exit
fi

USER=$1
MODE=`echo $2 | tr A-Z a-z`
FOLDER=$3

if [ "x${MODE}" != "xspam" -a "x${MODE}" != "xham" ]; then
  usage
fi

if [ "x${USER}" = "x" ]; then
  usage
fi

trainAccountFolder

exit 0

You can take a look at the processing of this script by installing the following (make a copy of your original file first) modified zmtrainsa files and running the following command:

sh -x /opt/zimbra/bin/zmtrainsa_test2 &>>/opt/zimbra/log/zmtrainsa_test.log  <-- check the output to see if it's working correctly.

Feedback

Feedback on your experience with rspamd, tweaks, configuration of rspamd itself and any questions/problems are always welcome in the forum thread on this topic.

If you think this is a worthwhile change for Zimbra then feel free to add you comments and votes to the RFE I' ve filed for removing spamassassin and adding this as a replacement in ZCS: bugzilla id 108168

Verified Against: Zimbra Collaboration Suite 8.8, 8.7 Date Created: 09/09/2017
Article ID: https://wiki.zimbra.com/index.php?title=Rspamd Date Modified: 2017-10-27



Try Zimbra

Try Zimbra Collaboration with a 60-day free trial.
Get it now »

Want to get involved?

You can contribute in the Community, Wiki, Code, or development of Zimlets.
Find out more. »

Looking for a Video?

Visit our YouTube channel to get the latest webinars, technology news, product overviews, and so much more.
Go to the YouTube channel »


Jump to: navigation, search