diff -Nur -x '*.orig' -x '*.rej' smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl mezzanine_patched_smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl --- smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl 2007-04-11 16:48:31.000000000 -0400 +++ mezzanine_patched_smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl 2007-09-10 17:06:26.000000000 -0400 @@ -1,774 +1,831 @@ -#!/usr/bin/perl -w - -############################################################################# -# -# This script provides daily SpamFilter statistics and deletes all users -# junkmails. Configuration of the script is done by the Spam Filter -# Server-Manager module -# -# April 2006 - no longer controlled by server manager, and does not delete files -# -# This script has been developed -# by Jesper Knudsen at http://sme.swerts-knudsen.dk -# -# Revision History: -# -# August 13, 2003: Initial version -# August 25, 2004: fixed problem when hostname had no-ASCII chars -# March 23, 2006 Revised for sme7 RM -# March 27, 2006 ditto BJR (http://www.abandonmicrosoft.co.uk) -# - Merged Clamav and SA stats -# - Moved all analysis to qsmtpd log -# - Removed parameterised interval (for simplicity - not sure of format anyway) -# - add in archived log files for people who have high turnover -# - Alter labels to be more accurate -# - Detect deleted spam (over threshold) without using spam score -# - Detect RBL rejections -# - Detect pattern (executible) rejections -# - Look for the DENY labels - add in Miscellaneous category -# April 6, 2006 - check qpsmtp log level and also DNS enable properties -# - Average spam scores for under and over threshold seperatly -# - Log tag and Reject levels -# - TBD - check that RBL DENY are being detected (I have no date to check this) -# April 7, 2007 - re-written by Charlie Brady totally in Perl -# April 16, 2006 - move warnings to report -# - Spot fetchmail deliveries -# - Spot Internal connections from client PCs -# - TBD check that RBL DENY are being detected (I have no data to check this) -# April 30, 2006 - Pascal Schirrmann Start Time and End Time to noon - should be a param -# so the script can be run at any time in the day. -# - adds 'by recipients domains' stats Useful for MX-Backup or multi domains hosts -# - Add a 'recipients per mail' stat. Useful : until now the sums are correct :-) -# - Correct some messages about rbl who can led to wrong entry in the config database -# ( and without expected results, of course !) -# - improve a regexp in the SPAM detection -# May 1, 2006 - BJR - Fix situation where mxbackup prop is not defined -# - fix a spelling and minor format of domain report -# May 9, 2006 - bjr - Make RBL percentage a percentage of total connections (else it >100%) -# May 9, 2006 - ps - some 'sanity check' in the 'per domains part of the stats (to avoid / 0) -# May 12, 2006 - ps - some cleanup in the 'per domains' stats -# - Add a version number, logged in the mail -# June 20, 2006 - bjr - Minor change to RBL instructions, and adjust domain table format -# Feb 19, 2007 - bjr - Adjust table lines oin a couple of places -# - bjr - and add documentation details about percentages etc -# - bjr - Alter misc to "non conforming" anmd accumulated these hourly -# - bjr - Express change over tag count to exclude spam rejected over threshold -# - bjr - Change "processsed" to "fully downloaded" -# - bjr - Change percentages so that they are all a percetnage of the total emails received -# 0.6.1 - bjr - Change to use output from the logterse qpsmtpd plugin -# 0.6.2 - bjr - Fix fetchmail tests -# 0.6.3 - bjr - adjust for log-items change in order -# 0.6.4&5 - bjr - Adjust table formatting -# 0.6.6 - bjr - Take outgoing emails out of "others", add "Outgoing" and "Internal" -# 0.6.7 - bjr - Fix missing plugins/wrong names. pull invalid recipient out of deny msg for goodrcptto -# 0.6.8 - bjr - catch a few more plugin name failures -############################################################################# - - -# internal modules (part of core perl distribution) -use strict; -use warnings; -use Getopt::Long; -use Pod::Usage; -use POSIX qw/strftime floor/; -use Time::Local; -use Date::Manip; -use Time::TAI64; -use esmith::ConfigDB; -use esmith::DomainsDB; -use Sys::Hostname; -use Switch; - -my $hostname = hostname(); - -#Configuration section -my %opt = (); - -$opt{'version'} = '0.6.8'; # please update at each change. -$opt{'debug'} = 0; # guess what ? -$opt{'sendmail'} = '/usr/sbin/sendmail'; # Path to sendmail stub -$opt{'from'} = 'spamfilter-stats'; # Who is the mail from -$opt{'end'} = `date --iso-8601`; # midnight today -my $yesterday = $opt{ 'end' }; -$yesterday =~ s/\-//g ; -$yesterday--; -$opt{'start'} = `date --iso-8601 -d $yesterday`; # midnight yesterday -$opt{'mail'} = "admin"; -$opt{'timezone'} = `date +%z`; -Date_Init("TZ=$opt{'timezone'}"); -my $FetchmailIP = '127.0.0.200'; #Apparent Ip address of fetchmail deliveries - -my $disabled = 0; - -my $tstart = time; - -#Local variables -my $YEAR = ( localtime(time) )[5]; # this is years since 1900 - -my $total = 0; -my $spamcount = 0; -my $spamavg = 0; -my $hamcount = 0; -my $hamavg = 0; -my $rejectspamavg = 0; - -my $Accepttotal = 0; -my $localAccepttotal = 0; #Fetchmail connections -my $localsendtotal = 0; #Connections from local PCs -my $totalexamined = 0; #total download + RBL etc -my %sendtotalbyhour = (); -my %localLANbyhour = (); -my %localacceptbyhour = (); - -my $above15 = 0; -my %above15byhour = (); - -my $RBLcount = 0; -my %RBLbyhour = (); - -my $MiscDenyCount = 0; -my %MiscDenybyhour = (); - -my $PatternFilterCount = 0; -my %patternfilterbyhour = (); - -my $noninfectedcount = 0; -my $okemailcount = 0; - -my $infectedcount = 0; -my %infectedbyhour = (); -my %found_viruses = (); - -my %spambyhour = (); -my %hambyhour = (); - -my $warnnoreject = " "; -my $rblnotset = ' '; - -my $FS = "\t"; # field separator used by logterse plugin -my %log_items = (); -my $score; -my %timestamp_items = (); -my $localflag = 0; #indicate if current email is local or not - -# some storage for by recipient domains stats (PS) -# my bad : I have to deal with multiple simoultaneous connections -# will play with the process number. -# my $currentrcptdomain = '' ; -my %currentrcptdomain ; # temporay store the recipient domain until end of mail processing -my %byrcptdomain ; # Store 'by domains stats' -my @extdomain ; # only useful in some MX-Backup case, when any subdomains are allowed -my $morethanonercpt = 0 ; # count every 'second' recipients for a mail. - -# store the domain of interest. Every other records are stored in a 'Other' zone -my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n"; -foreach my $domain( $ddb->get_all_by_prop( type => "domain" ) ) { - $byrcptdomain{ $domain->key }{ 'type' }='local'; -} -$byrcptdomain{ esmith::ConfigDB->open_ro->get('SystemName')->value . "." - . esmith::ConfigDB->open_ro->get('DomainName')->value }{ 'type' } = 'local'; - -# is this system a MX-Backup ? -if (esmith::ConfigDB->open_ro->get('mxbackup')){ - if ( ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('status') || 'disabled' ) eq 'enabled' ) { - my %MXValues = split( /,/, ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('name') || '' ) ) ; - foreach my $data ( keys %MXValues ) { - $byrcptdomain{ $data }{ 'type' } = "mxbackup-$MXValues{ $data }" ; - if ( $MXValues{ $data } == 1 ) { # subdomains allowed, must take care of this - push @extdomain, $data ; - } - } - } -} - -my ( $start, $end ) = parse_arg( $opt{'start'}, $opt{'end'} ); - -# -# First check current configuration for logging, DNS enable and Max threshold for spamassassin -# - -#my $LogLevel = esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('LogLevel'); -#my $LowLogLevel = ( $LogLevel < 8 ); - -my $RHSenabled = - ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('RHSBL') eq 'enabled' ); -my $DNSenabled = - ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('DNSBL') eq 'enabled' ); -my $SARejectLevel = - esmith::ConfigDB->open_ro->get('spamassassin')->prop('RejectLevel'); -my $SATagLevel = - esmith::ConfigDB->open_ro->get('spamassassin')->prop('TagLevel'); -my $DomainName = - esmith::ConfigDB->open_ro->get('DomainName')->value; - -# check that logterse is in use -#my pluginfile = '/var/service/qpsmtpd/config/peers/0'; - - - -if ( !$RHSenabled || !$DNSenabled ) { - $rblnotset = '*'; -} - -if ( $SARejectLevel == 0 ) { - - $warnnoreject = "(*Warning* 0 = no reject)"; - -} - -# -#--------------------------------------- -# Scan the qpsmtpd log file -#--------------------------------------- - - -# Init the hashes -my $nhour = floor( $start / 3600 ); -while ( $nhour < $end / 3600 ) { - $MiscDenybyhour{$nhour}=0; - $RBLbyhour{$nhour}=0; - $above15byhour{$nhour}=0; - $patternfilterbyhour{$nhour}=0; - $nhour++; -} - -my $starttai = Time::TAI64::unixtai64n($start); -my $endtai = Time::TAI64::unixtai64n($end); - -LINE: while (<>) { - my($tai,$log) = split(' ',$_,2); - - - #If date specified, only process lines matching date - next LINE if ( $tai lt $starttai ); - last if ( $tai gt $endtai ); - - #only select Logterse output - next LINE unless m/terse plugin/; - - - my $abstime = Time::TAI64::tai2unix($tai); - my $abshour = floor( $abstime / 3600 ); # Hours since the epoch - - - my ($timestamp_part, $log_part) = split '`'; - my (@log_items) = split $FS, $log_part; - - my (@timestamp_items) = split(' ',$timestamp_part); - - # we store the more recent recipient domain, for domain statistics - # in fact, we only store the first recipient. Could be sort of headhache - # to obtain precise stats with many recipients on more than one domain ! - my $proc = $timestamp_items[1] ; #numeric Id for the email - - $totalexamined++; - - # first spot the fetchmail and local deliveries. - - # print '<'.$log_items[1].'><'.$log_items[5].'><'.$log_items[8].">\n"; - - if ($log_items[1] =~ m/.*$DomainName.*/) {$localsendtotal++;$localLANbyhour{$abshour}++;$localflag=1} - else {$localflag=0} - - if ($log_items[0] =~ m/.*$FetchmailIP.*/) {$localAccepttotal++;$localacceptbyhour{$abshour}++} - - - # and adjust for recipient field if not set-up by denying plugin - extract from deny msg - - if (length($log_items[4])==0) { - if ($log_items[5] eq 'check_goodrcptto') { - if ($log_items[7] gt "invalid recipient") { - $log_items[4] = substr($log_items[7],18) #Leave only email address - } - } - } - - - - if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) { - $currentrcptdomain{ $proc } = lc($log_items[4]) ; - $currentrcptdomain{ $proc } =~ s/.*@//; - - - -# print $proc,$log_items[4]."\n"; - - $currentrcptdomain{ $proc } =~ s/[^\w\-\.]//g ; - -# print $currentrcptdomain{ $proc }."\n"; - - my $NotableDomain = 0 ; - if ( defined ( $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'type' } ) ) { - $NotableDomain = 1 ; - } else { - foreach ( @extdomain ) { - if ( $currentrcptdomain{ $proc } =~ m/$_$/ ) { - $NotableDomain = 1 ; - last ; - } - } - } - if ( !$NotableDomain ) { - # check for outgoing email - if ($localflag==1) {$currentrcptdomain{ $proc } = 'Outgoing'} - else {$currentrcptdomain{ $proc } = 'Others'} - } else { - if ($localflag==1) {$currentrcptdomain{ $proc } = 'Internal'} - } - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'total' }++ ; - } else { - # there more than a recipient for a mail, how many daily ? - $morethanonercpt++; - } - - # then categorise the result - if (exists $log_items[5]) { - #Check for badly formed lines (from earlier testing) - - if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'rhsbl') { $RBLcount++;$RBLbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'dnsbl') { $RBLcount++;$RBLbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$patternfilterbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$patternfilterbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} - - if ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one. - - if ($log_items[5] eq 'spamassassin') { $above15++;$above15byhour{$abshour}++; - # and extract the spam score - if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1} - mark_domain_rejected($proc); - next LINE - } - - if ($log_items[5] eq 'virus::clamav') { $infectedcount++;$infectedbyhour{$abshour}++; - #extract the virus name - if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;} - mark_domain_rejected($proc); - next LINE - } - - if ($log_items[5] eq 'queued') { $Accepttotal++; - #extract the spam score - if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") { - $score = $1; - if ($score < $SATagLevel) { $hamcount++;$hamavg += $score} - else {$spamcount++;$spamavg += $score} - } - if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; - $currentrcptdomain{ $proc } = '' ; - } - next LINE - } - - print $log_items[5]."\n"; #Not detected - - } - - - -} - -my $QueryNoLogTerse = ($totalexamined==0); #might indicate logterse not installed in qpsmtpd plugins - -#Calculate some numbers - -$spamavg = $spamavg / $spamcount if $spamcount; -$rejectspamavg = $rejectspamavg / $above15 if $above15; -$hamavg = $hamavg / $hamcount if $hamcount; - -# RBL etc percent of total SMTP sessions - -my $rblpercent = ( ( $RBLcount / $totalexamined ) * 100 ) if $totalexamined; -my $PatternFilterpercent = ( ( $PatternFilterCount / $totalexamined ) * 100 ) if $totalexamined; -my $Miscpercent = ( ( $MiscDenyCount / $totalexamined ) * 100 ) if $totalexamined; - -#Spam and virus percent of total email downloaded -#Expressed as a % of total examined -my $spampercent = ( ( $spamcount / $totalexamined ) * 100 ) if $totalexamined; -my $hampercent = ( ( $hamcount / $totalexamined ) * 100 ) if $totalexamined; -my $hrsinperiod = ( ( $end - $start ) / 3600 ); -my $emailperhour = ( $totalexamined / $hrsinperiod ) if $totalexamined; -my $above15percent = ( $above15 / $totalexamined * 100 ) if $totalexamined; -my $infectedpercent = ( ( $infectedcount / ($totalexamined) ) * 100 ) if $totalexamined; -my $AcceptPercent = ( ( $Accepttotal / ($totalexamined) ) * 100 ) if $totalexamined; - -my $oldfh; -#Open Sendmail if we are mailing it -if ( $opt{'mail'} && !$disabled ) { - open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" ) - or die "Can't open sendmail: $!\n"; - print SENDMAIL "From: $opt{'from'}\n"; - print SENDMAIL "To: $opt{'mail'}\n"; - print SENDMAIL "Subject: Spam Filter Statistics from $hostname - ", - strftime( "%F", localtime($start) ), "\n\n"; - $oldfh = select SENDMAIL; -} - -my $telapsed = time - $tstart; - -if ( !$disabled ) { - - #Output results - print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n"; - print "----------------------------------------------------", "\n\n"; - - print "$0 Version : $opt{'version'}", "\n\n"; - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n"; - print "Period Ending : ", strftime( "%c", localtime($end) ), "\n"; - print "\n"; - - print "Clam Version : ", `freshclam -V`; - print "SpamAssassin Version : ", `spamassassin -V`; - printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel, - $SARejectLevel; - - print "\n"; - printf "Reporting Period : %.2f hrs\n", $hrsinperiod; - print "----------------------------\n"; - print "\n"; - - printf "All SMTP connections accepted : %8d \n", $totalexamined; - - if ($localAccepttotal>0) { - printf "Connections from Fetchmail : %8d \n", - $localAccepttotal; - } - printf "SMTP from local workstations : %8d \n\n", $localsendtotal; - - - printf "RBL rejected : %8d (%6.2f%%)\n", $RBLcount, - $rblpercent || 0; - printf "Pattern filter rejected : %8d (%6.2f%%)\n", - $PatternFilterCount, $PatternFilterpercent || 0; - printf "Rejected due to non conformance : %8d (%6.2f%%)\n", $MiscDenyCount, - $Miscpercent || 0; - - printf "Infected by Virus : %8d (%6.2f%%)\n", $infectedcount, - $infectedpercent || 0; - - printf "Spam rejected (over reject level): %8d (%6.2f%%)\n", $above15, - $above15percent || 0; - printf "Spam detected (over tag level) : %8d (%6.2f%%)\n", $spamcount, - $spampercent || 0; - printf "Ham detected (under tag level) : %8d (%6.2f%%)\n", $hamcount, - $hampercent || 0; - print " --------------------\n"; - printf "Total emails accepted : %8d (%6.2f%%)\n", $Accepttotal, - $AcceptPercent || 0; - - printf "Emails per hour : (%8.1f/hr)\n", $emailperhour || 0; - print "\n"; - printf "Average spam score (accepted): %11.2f\n", $spamavg || 0; - printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0; - printf "Average ham score : %11.2f\n", $hamavg || 0; - print "\n"; - print "Statistics by Hour\n"; - if ($localAccepttotal>0) { - print "---------------------------------------------------------------------------------------- \n"; - print - "Hour Fetchml Local Virus Spam Ham RBL/DNS$rblnotset Execut. Non.Conf.\n"; - print "-------------- -------- -------- -------- -------- -------- -------- -------- ---------\n"; - - my $hour = floor( $start / 3600 ); - while ( $hour < $end / 3600 ) { - printf( - "%s %8d %8d %8d %8d %8d %8d %8d %8d\n", - strftime( "%F, %H", localtime( $hour * 3600 ) ), - $localacceptbyhour{$hour} || 0, - $localLANbyhour{$hour} || 0, - $infectedbyhour{$hour} || 0, - $spambyhour{$hour} || 0, - $hambyhour{$hour} || 0, - $RBLbyhour{$hour} || 0, - $patternfilterbyhour{$hour} || 0, - $MiscDenybyhour{$hour} || 0 - ); - $hour++; - } - print "---------------------------------------------------------------------------------------- \n"; - - } else { - print "------------------------------------------------------------------------------- \n"; - print - "Hour Local Virus Spam Ham RBL/DNS$rblnotset Execut. Non.Conf.\n"; - print "--------------- -------- -------- -------- -------- -------- -------- --------- \n"; - - my $hour = floor( $start / 3600 ); - while ( $hour < $end / 3600 ) { - printf( - "%s %8d %8d %8d %8d %8d %8d %8d\n", - strftime( "%F, %H", localtime( $hour * 3600 ) ), - $localLANbyhour{$hour} || 0, - $infectedbyhour{$hour} || 0, - $spambyhour{$hour} || 0, - $hambyhour{$hour} || 0, - $RBLbyhour{$hour} || 0, - $patternfilterbyhour{$hour} || 0, - $MiscDenybyhour{$hour} || 0 - ); - $hour++; - } - - print "------------------------------------------------------------------------------ \n"; - - } - if ($localAccepttotal>0) { - print "*Fetchml* means connections from Fetchmail delivering email\n"; - } - print "*Local* means connections from workstations on local LAN.\n"; - print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol.\n"; - print " or email was to non existant address.\n"; - print "\n"; - - if ($QueryNoLogTerse) { - print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; - print " to enable it follow the instructions at .............................\n"; - } - - - if ( !$RHSenabled || !$DNSenabled ) { - - # comment about RBL not set - print -"* - This means that one or more of the possible spam black listing services\n that are available have not been enabled.\n"; - print " You have not enabled:\n"; - - if ( !$RHSenabled ) { - print " RHSBL\n"; - } - - if ( !$DNSenabled ) { - print " DNSBL\n"; - } - - - print " To enable these you can use the following commands:\n"; - if ( !$RHSenabled ) { - print " config setprop qpsmtpd RHSBL enabled\n"; - } - - if ( !$DNSenabled ) { - print " config setprop qpsmtpd DNSBL enabled\n"; - } - - # there so much templates to expand... (PS) - print " Followed by:\n signal-event email-update and\n svc -t /service/qpsmtpd\n\n"; - } - - - # time to do a 'by recipient domain' report - print "\nIncoming mails by recipient domains usage\n"; - print "-----------------------------------------\n"; - print - "Domains Type Total Denied XferErr Accept \%accept\n"; - print - "---------------------------- ---------- ------ ------ ------- ------ -------\n"; - my %total = ( - total => 0, - deny => 0, - xfer => 0, - accept => 0, - ); - foreach my $domain ( - sort { - join( "\.", reverse( split /\./, $a ) ) cmp - join( "\.", reverse( split /\./, $b ) ) - } keys %byrcptdomain - ) - { - next if ( ( $byrcptdomain{$domain}{'total'} || 0 ) == 0 ); - my $tp = $byrcptdomain{$domain}{'type'} || 'other'; - my $to = $byrcptdomain{$domain}{'total'} || 0; - my $de = $byrcptdomain{$domain}{'deny'} || 0; - my $xr = $byrcptdomain{$domain}{'xfer'} || 0; - my $ac = $byrcptdomain{$domain}{'accept'} || 0; - printf "%-28s %-10s %6d %6d %7d %6d %6.2f%%\n", $domain, $tp, $to, - $de, $xr, $ac, $ac * 100 / $to; - $total{'total'} += $to; - $total{'deny'} += $de; - $total{'xfer'} += $xr; - $total{'accept'} += $ac; - } - print - "---------------------------- ---------- ------ ------- ------ ------ -------\n"; - - # $total{ 'total' } can be equal to 0, bad for divisions... - my $perc1 = 0; - my $perc2 = 0; - if ( $total{'total'} != 0 ) { - $perc1 = $total{'accept'} * 100 / $total{'total'}; - $perc2 = ( ( $total{'total'} + $morethanonercpt ) / $total{'total'} ); - } - printf - "Total %6d %6d %7d %6d %6.2f%%\n\n", - $total{'total'}, $total{'deny'}, $total{'xfer'}, $total{'accept'}, - $perc1; - printf - "%d mails were processed for %d Recipients\nThe average recipients by mail is %4.2f\n\n", - $total{'total'}, ( $total{'total'} + $morethanonercpt ), $perc2; - - if ( $infectedcount > 0 ) { - show_virus_variants(); - } - -} # not disabled - -List_Junkmail(); - -if ( !$disabled ) { - - print "\nDone. Report generated in $telapsed sec.\n\n"; - - #Close Senmdmail if it was opened - if ( $opt{'mail'} ) { - select $oldfh; - close(SENDMAIL); - } - -} - -#All done -exit 0; - -############################################################################# -# Subroutines ############################################################### -############################################################################# - - -######################################## -# Process parms # -######################################## -sub parse_arg { - my $startdate = shift; - my $enddate = shift; - - my $secsinday = 86400; - my $time = 0; - - my $start = UnixDate( $startdate, "%s" ); - my $end = UnixDate( $enddate, "%s" ); - - if ( !$start && !$end ) { - $end = time; - $start = $end - $secsinday; - return ( $start, $end ); - } - - if ( !$start ) { - $start = $end - $secsinday; - return ( $start, $end ); - } - - if ( !$end ) { - $end = $start + $secsinday; - return ( $start, $end ); - } - - if ( $start > $end ) { - return ( $end, $start ); - } - - return ( $start, $end ); - -} - -sub dbg { - my $msg = shift; - - if ( $opt{debug} ) { - print STDERR $msg; - } -} - -sub List_Junkmail { - - # - # Show how many junkmails in each user's junkmail folder. - # - use esmith::AccountsDB; - my $adb = esmith::AccountsDB->open_ro; - my $entry; - foreach my $user ($adb->users) { - my $found = 0; - my $junkmail_dir = "/home/e-smith/files/users/" . - $user->key . "/Maildir/.junkmail"; -# print $user->key; - foreach my $dir (qw(new cur)) { - # Now get the content list for the directory. - if (opendir( QDIR, "$junkmail_dir/$dir" )) { - while ($entry=readdir(QDIR) ) { - next if $entry =~ /^\./; - $found++; - } - - closedir(QDIR); - } - } - if ( !$disabled ) { - printf "User \"%s\" ", $user->key; - printf "- %d email(s) left in junkmail folder\n", $found; - } - } -} - -sub show_virus_variants - -# -# Show a league table of the different virus types found today -# - -{ - - print("Virus Statistics by name:\n"); - print("---------------------------------------------\n"); - foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} } - keys %found_viruses) - { - print "Rejected $found_viruses{$virus}\t$virus\n"; - } - print("---------------------------------------------\n"); -} - -sub mark_domain_rejected - -# -# Tag domain as having a rejected email -# -{ -my ($proc) = @_; -if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'deny' }++ ; - $currentrcptdomain{ $proc } = '' ; - } +#!/usr/bin/perl -w + +############################################################################# +# +# This script provides daily SpamFilter statistics and deletes all users +# junkmails. Configuration of the script is done by the Spam Filter +# Server-Manager module +# +# April 2006 - no longer controlled by server manager, and does not delete files +# +# This script has been developed +# by Jesper Knudsen at http://sme.swerts-knudsen.dk +# +# Revision History: +# +# August 13, 2003: Initial version +# August 25, 2004: fixed problem when hostname had no-ASCII chars +# March 23, 2006 Revised for sme7 RM +# March 27, 2006 ditto BJR (http://www.abandonmicrosoft.co.uk) +# - Merged Clamav and SA stats +# - Moved all analysis to qsmtpd log +# - Removed parameterised interval (for simplicity - not sure of format anyway) +# - add in archived log files for people who have high turnover +# - Alter labels to be more accurate +# - Detect deleted spam (over threshold) without using spam score +# - Detect RBL rejections +# - Detect pattern (executible) rejections +# - Look for the DENY labels - add in Miscellaneous category +# April 6, 2006 - check qpsmtp log level and also DNS enable properties +# - Average spam scores for under and over threshold seperatly +# - Log tag and Reject levels +# - TBD - check that RBL DENY are being detected (I have no date to check this) +# April 7, 2007 - re-written by Charlie Brady totally in Perl +# April 16, 2006 - move warnings to report +# - Spot fetchmail deliveries +# - Spot Internal connections from client PCs +# - TBD check that RBL DENY are being detected (I have no data to check this) +# April 30, 2006 - Pascal Schirrmann Start Time and End Time to noon - should be a param +# so the script can be run at any time in the day. +# - adds 'by recipients domains' stats Useful for MX-Backup or multi domains hosts +# - Add a 'recipients per mail' stat. Useful : until now the sums are correct :-) +# - Correct some messages about rbl who can led to wrong entry in the config database +# ( and without expected results, of course !) +# - improve a regexp in the SPAM detection +# May 1, 2006 - BJR - Fix situation where mxbackup prop is not defined +# - fix a spelling and minor format of domain report +# May 9, 2006 - bjr - Make RBL percentage a percentage of total connections (else it >100%) +# May 9, 2006 - ps - some 'sanity check' in the 'per domains part of the stats (to avoid / 0) +# May 12, 2006 - ps - some cleanup in the 'per domains' stats +# - Add a version number, logged in the mail +# June 20, 2006 - bjr - Minor change to RBL instructions, and adjust domain table format +# Feb 19, 2007 - bjr - Adjust table lines oin a couple of places +# - bjr - and add documentation details about percentages etc +# - bjr - Alter misc to "non conforming" anmd accumulated these hourly +# - bjr - Express change over tag count to exclude spam rejected over threshold +# - bjr - Change "processsed" to "fully downloaded" +# - bjr - Change percentages so that they are all a percetnage of the total emails received +# 0.6.1 - bjr - Change to use output from the logterse qpsmtpd plugin +# 0.6.2 - bjr - Fix fetchmail tests +# 0.6.3 - bjr - adjust for log-items change in order +# 0.6.4&5 - bjr - Adjust table formatting +# 0.6.6 - bjr - Take outgoing emails out of "others", add "Outgoing" and "Internal" +# 0.6.7 - bjr - Fix missing plugins/wrong names. pull invalid recipient out of deny msg for goodrcptto +# 0.6.8 - bjr - catch a few more plugin name failures +# 0.6.9 - bjr - Catch webmail and mailman +# 0.6.10 - bjr - Refine Webmail identification +# 0.6.11 - bjr - Fix Webmail identification +# +# +# +# TODO +# ---- +# +# 1. Re-Write the table so that it does not show certain columns if they are zero, this will allow EXTRA +# columns to be shown (e.g Mailman, fetchmail, webmail) only if there is some activity +# +############################################################################# + + +# internal modules (part of core perl distribution) +use strict; +use warnings; +use Getopt::Long; +use Pod::Usage; +use POSIX qw/strftime floor/; +use Time::Local; +use Date::Manip; +use Time::TAI64; +use esmith::ConfigDB; +use esmith::DomainsDB; +use Sys::Hostname; +use Switch; + +my $hostname = hostname(); + +#Configuration section +my %opt = (); + +$opt{'version'} = '0.6.11'; # please update at each change. +$opt{'debug'} = 0; # guess what ? +$opt{'sendmail'} = '/usr/sbin/sendmail'; # Path to sendmail stub +$opt{'from'} = 'spamfilter-stats'; # Who is the mail from +$opt{'end'} = `date --iso-8601`; # midnight today +my $yesterday = $opt{ 'end' }; +$yesterday =~ s/\-//g ; +$yesterday--; +$opt{'start'} = `date --iso-8601 -d $yesterday`; # midnight yesterday +$opt{'mail'} = "admin"; +$opt{'timezone'} = `date +%z`; +Date_Init("TZ=$opt{'timezone'}"); +my $FetchmailIP = '127.0.0.200'; #Apparent Ip address of fetchmail deliveries +my $WebmailIP = '127.0.0.1'; #Apparent Ip of Webmail sender +my $localhost = 'localhost'; #Apparent sender for webmail +my $FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email +my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost + +my $disabled = 0; + +my $tstart = time; + +#Local variables +my $YEAR = ( localtime(time) )[5]; # this is years since 1900 + +my $total = 0; +my $spamcount = 0; +my $spamavg = 0; +my $hamcount = 0; +my $hamavg = 0; +my $rejectspamavg = 0; + +my $Accepttotal = 0; +my $localAccepttotal = 0; #Fetchmail connections +my $localsendtotal = 0; #Connections from local PCs +my $totalexamined = 0; #total download + RBL etc +my $WebMailsendtotal = 0; #total from Webmail +my $mailmansendcount = 0; #total from mailman +my %sendtotalbyhour = (); +my %localLANbyhour = (); +my %localacceptbyhour = (); +my %WebMailbyhour = (); + +my $above15 = 0; +my %above15byhour = (); + +my $RBLcount = 0; +my %RBLbyhour = (); + +my $MiscDenyCount = 0; +my %MiscDenybyhour = (); + +my $PatternFilterCount = 0; +my %patternfilterbyhour = (); + +my $noninfectedcount = 0; +my $okemailcount = 0; + +my $infectedcount = 0; +my %infectedbyhour = (); +my %found_viruses = (); + +my %spambyhour = (); +my %hambyhour = (); + +my $warnnoreject = " "; +my $rblnotset = ' '; + +my $FS = "\t"; # field separator used by logterse plugin +my %log_items = (); +my $score; +my %timestamp_items = (); +my $localflag = 0; #indicate if current email is local or not +my $WebMailflag = 0; #indicate if current mail is send from webmail + +# some storage for by recipient domains stats (PS) +# my bad : I have to deal with multiple simoultaneous connections +# will play with the process number. +# my $currentrcptdomain = '' ; +my %currentrcptdomain ; # temporay store the recipient domain until end of mail processing +my %byrcptdomain ; # Store 'by domains stats' +my @extdomain ; # only useful in some MX-Backup case, when any subdomains are allowed +my $morethanonercpt = 0 ; # count every 'second' recipients for a mail. + +# store the domain of interest. Every other records are stored in a 'Other' zone +my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n"; +foreach my $domain( $ddb->get_all_by_prop( type => "domain" ) ) { + $byrcptdomain{ $domain->key }{ 'type' }='local'; +} +$byrcptdomain{ esmith::ConfigDB->open_ro->get('SystemName')->value . "." + . esmith::ConfigDB->open_ro->get('DomainName')->value }{ 'type' } = 'local'; + +# is this system a MX-Backup ? +if (esmith::ConfigDB->open_ro->get('mxbackup')){ + if ( ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('status') || 'disabled' ) eq 'enabled' ) { + my %MXValues = split( /,/, ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('name') || '' ) ) ; + foreach my $data ( keys %MXValues ) { + $byrcptdomain{ $data }{ 'type' } = "mxbackup-$MXValues{ $data }" ; + if ( $MXValues{ $data } == 1 ) { # subdomains allowed, must take care of this + push @extdomain, $data ; + } + } + } +} + +my ( $start, $end ) = parse_arg( $opt{'start'}, $opt{'end'} ); + +# +# First check current configuration for logging, DNS enable and Max threshold for spamassassin +# + +#my $LogLevel = esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('LogLevel'); +#my $LowLogLevel = ( $LogLevel < 8 ); + +my $RHSenabled = + ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('RHSBL') eq 'enabled' ); +my $DNSenabled = + ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('DNSBL') eq 'enabled' ); +my $SARejectLevel = + esmith::ConfigDB->open_ro->get('spamassassin')->prop('RejectLevel'); +my $SATagLevel = + esmith::ConfigDB->open_ro->get('spamassassin')->prop('TagLevel'); +my $DomainName = + esmith::ConfigDB->open_ro->get('DomainName')->value; + +# check that logterse is in use +#my pluginfile = '/var/service/qpsmtpd/config/peers/0'; + + + +if ( !$RHSenabled || !$DNSenabled ) { + $rblnotset = '*'; +} + +if ( $SARejectLevel == 0 ) { + + $warnnoreject = "(*Warning* 0 = no reject)"; + +} + +# +#--------------------------------------- +# Scan the qpsmtpd log file +#--------------------------------------- + + +# Init the hashes +my $nhour = floor( $start / 3600 ); +while ( $nhour < $end / 3600 ) { + $MiscDenybyhour{$nhour}=0; + $RBLbyhour{$nhour}=0; + $above15byhour{$nhour}=0; + $patternfilterbyhour{$nhour}=0; + $WebMailbyhour{$nhour}=0; + $nhour++; +} + +my $starttai = Time::TAI64::unixtai64n($start); +my $endtai = Time::TAI64::unixtai64n($end); + +LINE: while (<>) { + my($tai,$log) = split(' ',$_,2); + + + #If date specified, only process lines matching date + next LINE if ( $tai lt $starttai ); + last if ( $tai gt $endtai ); + + #only select Logterse output + next LINE unless m/terse plugin/; + + + my $abstime = Time::TAI64::tai2unix($tai); + my $abshour = floor( $abstime / 3600 ); # Hours since the epoch + + + my ($timestamp_part, $log_part) = split '`'; + my (@log_items) = split $FS, $log_part; + + my (@timestamp_items) = split(' ',$timestamp_part); + + # we store the more recent recipient domain, for domain statistics + # in fact, we only store the first recipient. Could be sort of headhache + # to obtain precise stats with many recipients on more than one domain ! + my $proc = $timestamp_items[1] ; #numeric Id for the email + + $totalexamined++; + + # first spot the fetchmail and local deliveries. + + # print '<'.$log_items[1].'><'.$log_items[5].'><'.$log_items[8].">\n"; + + # Spot from local workstation + $localflag = 0; + $WebMailflag=0; + if ($log_items[1] =~ m/.*$DomainName.*/) {$localsendtotal++;$localLANbyhour{$abshour}++;$localflag=1} + # see if from localhost + elsif ($log_items[1] =~ m/.*$localhost.*/){ + # but not if it comes from fetchmail +# print $log_items[3]."\n"; + if ($log_items[3] =~ m/.*$FETCHMAIL.*/){} + else { + # might still be from mailman here +# print "got webmail\n"; + if ($log_items[3] =~ m/.*$MAILMAN.*/){$mailmansendcount++;$localsendtotal++;$localLANbyhour{$abshour}++;$localflag=1} + else { + # eliminate incoming localhost spoofs + if ($log_items[8] =~ m/.*msg denied before queued.*/){} + else {$localflag = 1;$WebMailsendtotal++;$WebMailbyhour{$abshour}++;$WebMailflag=1} + } + } + } + + # try to spot fetchmail emails + if ($log_items[0] =~ m/.*$FetchmailIP.*/) {$localAccepttotal++;$localacceptbyhour{$abshour}++} + elsif ($log_items[3] =~ m/.*$FETCHMAIL.*/) {$localAccepttotal++;$localacceptbyhour{$abshour}++} + + + # and adjust for recipient field if not set-up by denying plugin - extract from deny msg + + if (length($log_items[4])==0) { + if ($log_items[5] eq 'check_goodrcptto') { + if ($log_items[7] gt "invalid recipient") { + $log_items[4] = substr($log_items[7],18) #Leave only email address + } + } + } + + + + if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) { + $currentrcptdomain{ $proc } = lc($log_items[4]) ; + $currentrcptdomain{ $proc } =~ s/.*@//; + + + +# print $proc,$log_items[4]."\n"; + + $currentrcptdomain{ $proc } =~ s/[^\w\-\.]//g ; + +# print $currentrcptdomain{ $proc }."\n"; + + my $NotableDomain = 0 ; + if ( defined ( $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'type' } ) ) { + $NotableDomain = 1 ; + } else { + foreach ( @extdomain ) { + if ( $currentrcptdomain{ $proc } =~ m/$_$/ ) { + $NotableDomain = 1 ; + last ; + } + } + } + if ( !$NotableDomain ) { + # check for outgoing email + if ($localflag==1) {$currentrcptdomain{ $proc } = 'Outgoing'} + else {$currentrcptdomain{ $proc } = 'Others'} + } else { + if ($localflag==1) {$currentrcptdomain{ $proc } = 'Internal'} + } + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'total' }++ ; + } else { + # there more than a recipient for a mail, how many daily ? + $morethanonercpt++; + } + + # then categorise the result + if (exists $log_items[5]) { + #Check for badly formed lines (from earlier testing) + + if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'rhsbl') { $RBLcount++;$RBLbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'dnsbl') { $RBLcount++;$RBLbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$patternfilterbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$patternfilterbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} + + if ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one. + + if ($log_items[5] eq 'spamassassin') { $above15++;$above15byhour{$abshour}++; + # and extract the spam score + if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1} + mark_domain_rejected($proc); + next LINE + } + + if ($log_items[5] eq 'virus::clamav') { $infectedcount++;$infectedbyhour{$abshour}++; + #extract the virus name + if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;} + mark_domain_rejected($proc); + next LINE + } + + if ($log_items[5] eq 'queued') { $Accepttotal++; + #extract the spam score + if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") { + $score = $1; +# print $log_items[8]."<".$score.">\n"; + if ($score < $SATagLevel) { $hamcount++;$hambyhour{$abshour}++;$hamavg += $score} + else {$spamcount++;$spambyhour{$abshour}++;$spamavg += $score} + } + if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; + $currentrcptdomain{ $proc } = '' ; + } + next LINE + } + + print $log_items[5]."\n"; #Not detected + + } + + + +} + +my $QueryNoLogTerse = ($totalexamined==0); #might indicate logterse not installed in qpsmtpd plugins + +#Calculate some numbers + +$spamavg = $spamavg / $spamcount if $spamcount; +$rejectspamavg = $rejectspamavg / $above15 if $above15; +$hamavg = $hamavg / $hamcount if $hamcount; + +# RBL etc percent of total SMTP sessions + +my $rblpercent = ( ( $RBLcount / $totalexamined ) * 100 ) if $totalexamined; +my $PatternFilterpercent = ( ( $PatternFilterCount / $totalexamined ) * 100 ) if $totalexamined; +my $Miscpercent = ( ( $MiscDenyCount / $totalexamined ) * 100 ) if $totalexamined; + +#Spam and virus percent of total email downloaded +#Expressed as a % of total examined +my $spampercent = ( ( $spamcount / $totalexamined ) * 100 ) if $totalexamined; +my $hampercent = ( ( $hamcount / $totalexamined ) * 100 ) if $totalexamined; +my $hrsinperiod = ( ( $end - $start ) / 3600 ); +my $emailperhour = ( $totalexamined / $hrsinperiod ) if $totalexamined; +my $above15percent = ( $above15 / $totalexamined * 100 ) if $totalexamined; +my $infectedpercent = ( ( $infectedcount / ($totalexamined) ) * 100 ) if $totalexamined; +my $AcceptPercent = ( ( $Accepttotal / ($totalexamined) ) * 100 ) if $totalexamined; + +my $oldfh; + +#Open Sendmail if we are mailing it +if ( $opt{'mail'} && !$disabled ) { + open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" ) + or die "Can't open sendmail: $!\n"; + print SENDMAIL "From: $opt{'from'}\n"; + print SENDMAIL "To: $opt{'mail'}\n"; + print SENDMAIL "Subject: Spam Filter Statistics from $hostname - ", + strftime( "%F", localtime($start) ), "\n\n"; + $oldfh = select SENDMAIL; +} + +my $telapsed = time - $tstart; + +if ( !$disabled ) { + + #Output results + print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n"; + print "----------------------------------------------------", "\n\n"; + + print "$0 Version : $opt{'version'}", "\n\n"; + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n"; + print "Period Ending : ", strftime( "%c", localtime($end) ), "\n"; + print "\n"; + + print "Clam Version : ", `freshclam -V`; + print "SpamAssassin Version : ", `spamassassin -V`; + printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel, + $SARejectLevel; + + print "\n"; + printf "Reporting Period : %.2f hrs\n", $hrsinperiod; + print "----------------------------\n"; + print "\n"; + + printf "All SMTP connections accepted : %8d \n", $totalexamined; + + if ($localAccepttotal>0) { + printf "Connections from Fetchmail : %8d \n", + $localAccepttotal; + } + + if ($WebMailsendtotal>0) { + printf "Emails sent from WebMail : %8d \n", + $WebMailsendtotal; + } + + if ($mailmansendcount > 0) { + printf "Emails sent from Mailman : %8d \n", + $mailmansendcount; + } + + printf "SMTP from local workstations : %8d \n\n", $localsendtotal; + + + printf "RBL rejected : %8d (%6.2f%%)\n", $RBLcount, + $rblpercent || 0; + printf "Pattern filter rejected : %8d (%6.2f%%)\n", + $PatternFilterCount, $PatternFilterpercent || 0; + printf "Rejected due to non conformance : %8d (%6.2f%%)\n", $MiscDenyCount, + $Miscpercent || 0; + + printf "Infected by Virus : %8d (%6.2f%%)\n", $infectedcount, + $infectedpercent || 0; + + printf "Spam rejected (over reject level): %8d (%6.2f%%)\n", $above15, + $above15percent || 0; + printf "Spam detected (over tag level) : %8d (%6.2f%%)\n", $spamcount, + $spampercent || 0; + printf "Ham detected (under tag level) : %8d (%6.2f%%)\n", $hamcount, + $hampercent || 0; + print " --------------------\n"; + printf "Total emails accepted : %8d (%6.2f%%)\n", $Accepttotal, + $AcceptPercent || 0; + + printf "Emails per hour : (%8.1f/hr)\n", $emailperhour || 0; + print "\n"; + printf "Average spam score (accepted): %11.2f\n", $spamavg || 0; + printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0; + printf "Average ham score : %11.2f\n", $hamavg || 0; + print "\n"; + print "Statistics by Hour\n"; + if ($localAccepttotal>0) { + print "------------------------------------------------------------------------------------------------- \n"; + print + "Hour Fetchml Local WebMail Virus Spam Ham RBL/DNS$rblnotset Execut. Non.Conf.\n"; + print "-------------- -------- -------- -------- -------- -------- -------- -------- -------- ---------\n"; + + my $hour = floor( $start / 3600 ); + while ( $hour < $end / 3600 ) { + printf( + "%s %8d %8d %8d %8d %8d %8d %8d %8d %8d\n", + strftime( "%F, %H", localtime( $hour * 3600 ) ), + $localacceptbyhour{$hour} || 0, + $localLANbyhour{$hour} || 0, + $WebMailbyhour{$hour} || 0, + $infectedbyhour{$hour} || 0, + $spambyhour{$hour} || 0, + $hambyhour{$hour} || 0, + $RBLbyhour{$hour} || 0, + $patternfilterbyhour{$hour} || 0, + $MiscDenybyhour{$hour} || 0 + ); + $hour++; + } + print "------------------------------------------------------------------------------------------------- \n"; + + } else { + print "---------------------------------------------------------------------------------------- \n"; + print + "Hour Local WebMail Virus Spam Ham RBL/DNS$rblnotset Execut. Non.Conf.\n"; + print "--------------- -------- -------- -------- -------- -------- -------- -------- --------- \n"; + + my $hour = floor( $start / 3600 ); + while ( $hour < $end / 3600 ) { + printf( + "%s %8d %8d %8d %8d %8d %8d %8d %8d\n", + strftime( "%F, %H", localtime( $hour * 3600 ) ), + $localLANbyhour{$hour} || 0, + $WebMailbyhour{$hour} || 0, + $infectedbyhour{$hour} || 0, + $spambyhour{$hour} || 0, + $hambyhour{$hour} || 0, + $RBLbyhour{$hour} || 0, + $patternfilterbyhour{$hour} || 0, + $MiscDenybyhour{$hour} || 0 + ); + $hour++; + } + + print "---------------------------------------------------------------------------------------- \n"; + + } + if ($localAccepttotal>0) { + print "*Fetchml* means connections from Fetchmail delivering email\n"; + } + print "*Local* means connections from workstations on local LAN.\n"; + print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol.\n"; + print " or email was to non existant address.\n"; + print "\n"; + + if ($QueryNoLogTerse) { + print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; +# print " to enable it follow the instructions at .............................\n"; + } + + + if ( !$RHSenabled || !$DNSenabled ) { + + # comment about RBL not set + print +"* - This means that one or more of the possible spam black listing services\n that are available have not been enabled.\n"; + print " You have not enabled:\n"; + + if ( !$RHSenabled ) { + print " RHSBL\n"; + } + + if ( !$DNSenabled ) { + print " DNSBL\n"; + } + + + print " To enable these you can use the following commands:\n"; + if ( !$RHSenabled ) { + print " config setprop qpsmtpd RHSBL enabled\n"; + } + + if ( !$DNSenabled ) { + print " config setprop qpsmtpd DNSBL enabled\n"; + } + + # there so much templates to expand... (PS) + print " Followed by:\n signal-event email-update and\n svc -t /service/qpsmtpd\n\n"; + } + +# if ($Webmailsendtotal > 0) {print "If you have the mailman contrib installed, then the webmail totals might include some mailman emails\n"} + + # time to do a 'by recipient domain' report + print "\nIncoming mails by recipient domains usage\n"; + print "-----------------------------------------\n"; + print + "Domains Type Total Denied XferErr Accept \%accept\n"; + print + "---------------------------- ---------- ------ ------ ------- ------ -------\n"; + my %total = ( + total => 0, + deny => 0, + xfer => 0, + accept => 0, + ); + foreach my $domain ( + sort { + join( "\.", reverse( split /\./, $a ) ) cmp + join( "\.", reverse( split /\./, $b ) ) + } keys %byrcptdomain + ) + { + next if ( ( $byrcptdomain{$domain}{'total'} || 0 ) == 0 ); + my $tp = $byrcptdomain{$domain}{'type'} || 'other'; + my $to = $byrcptdomain{$domain}{'total'} || 0; + my $de = $byrcptdomain{$domain}{'deny'} || 0; + my $xr = $byrcptdomain{$domain}{'xfer'} || 0; + my $ac = $byrcptdomain{$domain}{'accept'} || 0; + printf "%-28s %-10s %6d %6d %7d %6d %6.2f%%\n", $domain, $tp, $to, + $de, $xr, $ac, $ac * 100 / $to; + $total{'total'} += $to; + $total{'deny'} += $de; + $total{'xfer'} += $xr; + $total{'accept'} += $ac; + } + print + "---------------------------- ---------- ------ ------- ------ ------ -------\n"; + + # $total{ 'total' } can be equal to 0, bad for divisions... + my $perc1 = 0; + my $perc2 = 0; + if ( $total{'total'} != 0 ) { + $perc1 = $total{'accept'} * 100 / $total{'total'}; + $perc2 = ( ( $total{'total'} + $morethanonercpt ) / $total{'total'} ); + } + printf + "Total %6d %6d %7d %6d %6.2f%%\n\n", + $total{'total'}, $total{'deny'}, $total{'xfer'}, $total{'accept'}, + $perc1; + printf + "%d mails were processed for %d Recipients\nThe average recipients by mail is %4.2f\n\n", + $total{'total'}, ( $total{'total'} + $morethanonercpt ), $perc2; + + if ( $infectedcount > 0 ) { + show_virus_variants(); + } + +} # not disabled + +List_Junkmail(); + +if ( !$disabled ) { + + print "\nDone. Report generated in $telapsed sec.\n\n"; + + #Close Senmdmail if it was opened + if ( $opt{'mail'} ) { + select $oldfh; + close(SENDMAIL); + } + +} + +#All done +exit 0; + +############################################################################# +# Subroutines ############################################################### +############################################################################# + + +######################################## +# Process parms # +######################################## +sub parse_arg { + my $startdate = shift; + my $enddate = shift; + + my $secsinday = 86400; + my $time = 0; + + my $start = UnixDate( $startdate, "%s" ); + my $end = UnixDate( $enddate, "%s" ); + + if ( !$start && !$end ) { + $end = time; + $start = $end - $secsinday; + return ( $start, $end ); + } + + if ( !$start ) { + $start = $end - $secsinday; + return ( $start, $end ); + } + + if ( !$end ) { + $end = $start + $secsinday; + return ( $start, $end ); + } + + if ( $start > $end ) { + return ( $end, $start ); + } + + return ( $start, $end ); + +} + +sub dbg { + my $msg = shift; + + if ( $opt{debug} ) { + print STDERR $msg; + } +} + +sub List_Junkmail { + + # + # Show how many junkmails in each user's junkmail folder. + # + use esmith::AccountsDB; + my $adb = esmith::AccountsDB->open_ro; + my $entry; + foreach my $user ($adb->users) { + my $found = 0; + my $junkmail_dir = "/home/e-smith/files/users/" . + $user->key . "/Maildir/.junkmail"; +# print $user->key; + foreach my $dir (qw(new cur)) { + # Now get the content list for the directory. + if (opendir( QDIR, "$junkmail_dir/$dir" )) { + while ($entry=readdir(QDIR) ) { + next if $entry =~ /^\./; + $found++; + } + + closedir(QDIR); + } + } + if ( !$disabled ) { + printf "User \"%s\" ", $user->key; + printf "- %d email(s) left in junkmail folder\n", $found; + } + } +} + +sub show_virus_variants + +# +# Show a league table of the different virus types found today +# + +{ + + print("Virus Statistics by name:\n"); + print("---------------------------------------------\n"); + foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} } + keys %found_viruses) + { + print "Rejected $found_viruses{$virus}\t$virus\n"; + } + print("---------------------------------------------\n"); +} + +sub mark_domain_rejected + +# +# Tag domain as having a rejected email +# +{ +my ($proc) = @_; +if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'deny' }++ ; + $currentrcptdomain{ $proc } = '' ; + } } \ No newline at end of file