| 1 |
diff -Nur -x '*.orig' -x '*.rej' smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl mezzanine_patched_smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl |
| 2 |
--- smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl 2007-10-11 16:11:29.000000000 -0400 |
| 3 |
+++ mezzanine_patched_smeserver-mailstats-0.0.2/root/usr/bin/spamfilter-stats-7.pl 2007-10-11 14:39:51.000000000 -0400 |
| 4 |
@@ -1,829 +1,1109 @@ |
| 5 |
-#!/usr/bin/perl -w |
| 6 |
- |
| 7 |
-############################################################################# |
| 8 |
-# |
| 9 |
-# This script provides daily SpamFilter statistics and deletes all users |
| 10 |
-# junkmails. Configuration of the script is done by the Spam Filter |
| 11 |
-# Server-Manager module |
| 12 |
-# |
| 13 |
-# April 2006 - no longer controlled by server manager, and does not delete files |
| 14 |
-# |
| 15 |
-# This script has been developed |
| 16 |
-# by Jesper Knudsen at http://sme.swerts-knudsen.dk |
| 17 |
-# |
| 18 |
-# Revision History: |
| 19 |
-# |
| 20 |
-# August 13, 2003: Initial version |
| 21 |
-# August 25, 2004: fixed problem when hostname had no-ASCII chars |
| 22 |
-# March 23, 2006 Revised for sme7 RM |
| 23 |
-# March 27, 2006 ditto BJR (http://www.abandonmicrosoft.co.uk) |
| 24 |
-# - Merged Clamav and SA stats |
| 25 |
-# - Moved all analysis to qsmtpd log |
| 26 |
-# - Removed parameterised interval (for simplicity - not sure of format anyway) |
| 27 |
-# - add in archived log files for people who have high turnover |
| 28 |
-# - Alter labels to be more accurate |
| 29 |
-# - Detect deleted spam (over threshold) without using spam score |
| 30 |
-# - Detect RBL rejections |
| 31 |
-# - Detect pattern (executible) rejections |
| 32 |
-# - Look for the DENY labels - add in Miscellaneous category |
| 33 |
-# April 6, 2006 - check qpsmtp log level and also DNS enable properties |
| 34 |
-# - Average spam scores for under and over threshold seperatly |
| 35 |
-# - Log tag and Reject levels |
| 36 |
-# - TBD - check that RBL DENY are being detected (I have no date to check this) |
| 37 |
-# April 7, 2007 - re-written by Charlie Brady totally in Perl |
| 38 |
-# April 16, 2006 - move warnings to report |
| 39 |
-# - Spot fetchmail deliveries |
| 40 |
-# - Spot Internal connections from client PCs |
| 41 |
-# - TBD check that RBL DENY are being detected (I have no data to check this) |
| 42 |
-# April 30, 2006 - Pascal Schirrmann Start Time and End Time to noon - should be a param |
| 43 |
-# so the script can be run at any time in the day. |
| 44 |
-# - adds 'by recipients domains' stats Useful for MX-Backup or multi domains hosts |
| 45 |
-# - Add a 'recipients per mail' stat. Useful : until now the sums are correct :-) |
| 46 |
-# - Correct some messages about rbl who can led to wrong entry in the config database |
| 47 |
-# ( and without expected results, of course !) |
| 48 |
-# - improve a regexp in the SPAM detection |
| 49 |
-# May 1, 2006 - BJR - Fix situation where mxbackup prop is not defined |
| 50 |
-# - fix a spelling and minor format of domain report |
| 51 |
-# May 9, 2006 - bjr - Make RBL percentage a percentage of total connections (else it >100%) |
| 52 |
-# May 9, 2006 - ps - some 'sanity check' in the 'per domains part of the stats (to avoid / 0) |
| 53 |
-# May 12, 2006 - ps - some cleanup in the 'per domains' stats |
| 54 |
-# - Add a version number, logged in the mail |
| 55 |
-# June 20, 2006 - bjr - Minor change to RBL instructions, and adjust domain table format |
| 56 |
-# Feb 19, 2007 - bjr - Adjust table lines oin a couple of places |
| 57 |
-# - bjr - and add documentation details about percentages etc |
| 58 |
-# - bjr - Alter misc to "non conforming" anmd accumulated these hourly |
| 59 |
-# - bjr - Express change over tag count to exclude spam rejected over threshold |
| 60 |
-# - bjr - Change "processsed" to "fully downloaded" |
| 61 |
-# - bjr - Change percentages so that they are all a percetnage of the total emails received |
| 62 |
-# 0.6.1 - bjr - Change to use output from the logterse qpsmtpd plugin |
| 63 |
-# 0.6.2 - bjr - Fix fetchmail tests |
| 64 |
-# 0.6.3 - bjr - adjust for log-items change in order |
| 65 |
-# 0.6.4&5 - bjr - Adjust table formatting |
| 66 |
-# 0.6.6 - bjr - Take outgoing emails out of "others", add "Outgoing" and "Internal" |
| 67 |
-# 0.6.7 - bjr - Fix missing plugins/wrong names. pull invalid recipient out of deny msg for goodrcptto |
| 68 |
-# 0.6.8 - bjr - catch a few more plugin name failures |
| 69 |
-# 0.6.9 - bjr - Catch webmail and mailman |
| 70 |
-# 0.6.10 - bjr - Refine Webmail identification |
| 71 |
-# 0.6.11 - bjr - Fix Webmail identification |
| 72 |
-# 0.6.12 - cfc - Fixed issue with previous day for the first day of the month. Initialized an array with empty values so that the script wouldn't issue warning messages when not all the fields were used. |
| 73 |
-# |
| 74 |
-# |
| 75 |
-# |
| 76 |
-# TODO |
| 77 |
-# ---- |
| 78 |
-# |
| 79 |
-# 1. Re-Write the table so that it does not show certain columns if they are zero, this will allow EXTRA |
| 80 |
-# columns to be shown (e.g Mailman, fetchmail, webmail) only if there is some activity |
| 81 |
-# |
| 82 |
-############################################################################# |
| 83 |
- |
| 84 |
- |
| 85 |
-# internal modules (part of core perl distribution) |
| 86 |
-use strict; |
| 87 |
-use warnings; |
| 88 |
-use Getopt::Long; |
| 89 |
-use Pod::Usage; |
| 90 |
-use POSIX qw/strftime floor/; |
| 91 |
-use Time::Local; |
| 92 |
-use Date::Manip; |
| 93 |
-use Time::TAI64; |
| 94 |
-use esmith::ConfigDB; |
| 95 |
-use esmith::DomainsDB; |
| 96 |
-use Sys::Hostname; |
| 97 |
-use Switch; |
| 98 |
- |
| 99 |
-my $hostname = hostname(); |
| 100 |
- |
| 101 |
-#Configuration section |
| 102 |
-my %opt = (); |
| 103 |
- |
| 104 |
-$opt{'version'} = '0.6.12'; # please update at each change. |
| 105 |
-$opt{'debug'} = 0; # guess what ? |
| 106 |
-$opt{'sendmail'} = '/usr/sbin/sendmail'; # Path to sendmail stub |
| 107 |
-$opt{'from'} = 'spamfilter-stats'; # Who is the mail from |
| 108 |
-$opt{'end'} = `date --iso-8601`; # midnight today |
| 109 |
-$opt{'start'} = UnixDate(DateCalc($opt{'end'},"- 24hours"),"%Y-%m-%d"); |
| 110 |
-$opt{'mail'} = "admin"; |
| 111 |
-$opt{'timezone'} = `date +%z`; |
| 112 |
-Date_Init("TZ=$opt{'timezone'}"); |
| 113 |
-my $FetchmailIP = '127.0.0.200'; #Apparent Ip address of fetchmail deliveries |
| 114 |
-my $WebmailIP = '127.0.0.1'; #Apparent Ip of Webmail sender |
| 115 |
-my $localhost = 'localhost'; #Apparent sender for webmail |
| 116 |
-my $FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email |
| 117 |
-my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost |
| 118 |
- |
| 119 |
-my $disabled = 0; |
| 120 |
- |
| 121 |
-my $tstart = time; |
| 122 |
- |
| 123 |
-#Local variables |
| 124 |
-my $YEAR = ( localtime(time) )[5]; # this is years since 1900 |
| 125 |
- |
| 126 |
-my $total = 0; |
| 127 |
-my $spamcount = 0; |
| 128 |
-my $spamavg = 0; |
| 129 |
-my $hamcount = 0; |
| 130 |
-my $hamavg = 0; |
| 131 |
-my $rejectspamavg = 0; |
| 132 |
- |
| 133 |
-my $Accepttotal = 0; |
| 134 |
-my $localAccepttotal = 0; #Fetchmail connections |
| 135 |
-my $localsendtotal = 0; #Connections from local PCs |
| 136 |
-my $totalexamined = 0; #total download + RBL etc |
| 137 |
-my $WebMailsendtotal = 0; #total from Webmail |
| 138 |
-my $mailmansendcount = 0; #total from mailman |
| 139 |
-my %sendtotalbyhour = (); |
| 140 |
-my %localLANbyhour = (); |
| 141 |
-my %localacceptbyhour = (); |
| 142 |
-my %WebMailbyhour = (); |
| 143 |
- |
| 144 |
-my $above15 = 0; |
| 145 |
-my %above15byhour = (); |
| 146 |
- |
| 147 |
-my $RBLcount = 0; |
| 148 |
-my %RBLbyhour = (); |
| 149 |
- |
| 150 |
-my $MiscDenyCount = 0; |
| 151 |
-my %MiscDenybyhour = (); |
| 152 |
- |
| 153 |
-my $PatternFilterCount = 0; |
| 154 |
-my %patternfilterbyhour = (); |
| 155 |
- |
| 156 |
-my $noninfectedcount = 0; |
| 157 |
-my $okemailcount = 0; |
| 158 |
- |
| 159 |
-my $infectedcount = 0; |
| 160 |
-my %infectedbyhour = (); |
| 161 |
-my %found_viruses = (); |
| 162 |
- |
| 163 |
-my %spambyhour = (); |
| 164 |
-my %hambyhour = (); |
| 165 |
- |
| 166 |
-my $warnnoreject = " "; |
| 167 |
-my $rblnotset = ' '; |
| 168 |
- |
| 169 |
-my $FS = "\t"; # field separator used by logterse plugin |
| 170 |
-my %log_items = ("","","","","","","",""); |
| 171 |
-my $score; |
| 172 |
-my %timestamp_items = (); |
| 173 |
-my $localflag = 0; #indicate if current email is local or not |
| 174 |
-my $WebMailflag = 0; #indicate if current mail is send from webmail |
| 175 |
- |
| 176 |
-# some storage for by recipient domains stats (PS) |
| 177 |
-# my bad : I have to deal with multiple simoultaneous connections |
| 178 |
-# will play with the process number. |
| 179 |
-# my $currentrcptdomain = '' ; |
| 180 |
-my %currentrcptdomain ; # temporay store the recipient domain until end of mail processing |
| 181 |
-my %byrcptdomain ; # Store 'by domains stats' |
| 182 |
-my @extdomain ; # only useful in some MX-Backup case, when any subdomains are allowed |
| 183 |
-my $morethanonercpt = 0 ; # count every 'second' recipients for a mail. |
| 184 |
- |
| 185 |
-# store the domain of interest. Every other records are stored in a 'Other' zone |
| 186 |
-my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n"; |
| 187 |
-foreach my $domain( $ddb->get_all_by_prop( type => "domain" ) ) { |
| 188 |
- $byrcptdomain{ $domain->key }{ 'type' }='local'; |
| 189 |
-} |
| 190 |
-$byrcptdomain{ esmith::ConfigDB->open_ro->get('SystemName')->value . "." |
| 191 |
- . esmith::ConfigDB->open_ro->get('DomainName')->value }{ 'type' } = 'local'; |
| 192 |
- |
| 193 |
-# is this system a MX-Backup ? |
| 194 |
-if (esmith::ConfigDB->open_ro->get('mxbackup')){ |
| 195 |
- if ( ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('status') || 'disabled' ) eq 'enabled' ) { |
| 196 |
- my %MXValues = split( /,/, ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('name') || '' ) ) ; |
| 197 |
- foreach my $data ( keys %MXValues ) { |
| 198 |
- $byrcptdomain{ $data }{ 'type' } = "mxbackup-$MXValues{ $data }" ; |
| 199 |
- if ( $MXValues{ $data } == 1 ) { # subdomains allowed, must take care of this |
| 200 |
- push @extdomain, $data ; |
| 201 |
- } |
| 202 |
- } |
| 203 |
- } |
| 204 |
-} |
| 205 |
- |
| 206 |
-my ( $start, $end ) = parse_arg( $opt{'start'}, $opt{'end'} ); |
| 207 |
- |
| 208 |
-# |
| 209 |
-# First check current configuration for logging, DNS enable and Max threshold for spamassassin |
| 210 |
-# |
| 211 |
- |
| 212 |
-#my $LogLevel = esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('LogLevel'); |
| 213 |
-#my $LowLogLevel = ( $LogLevel < 8 ); |
| 214 |
- |
| 215 |
-my $RHSenabled = |
| 216 |
- ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('RHSBL') eq 'enabled' ); |
| 217 |
-my $DNSenabled = |
| 218 |
- ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('DNSBL') eq 'enabled' ); |
| 219 |
-my $SARejectLevel = |
| 220 |
- esmith::ConfigDB->open_ro->get('spamassassin')->prop('RejectLevel'); |
| 221 |
-my $SATagLevel = |
| 222 |
- esmith::ConfigDB->open_ro->get('spamassassin')->prop('TagLevel'); |
| 223 |
-my $DomainName = |
| 224 |
- esmith::ConfigDB->open_ro->get('DomainName')->value; |
| 225 |
- |
| 226 |
-# check that logterse is in use |
| 227 |
-#my pluginfile = '/var/service/qpsmtpd/config/peers/0'; |
| 228 |
- |
| 229 |
- |
| 230 |
- |
| 231 |
-if ( !$RHSenabled || !$DNSenabled ) { |
| 232 |
- $rblnotset = '*'; |
| 233 |
-} |
| 234 |
- |
| 235 |
-if ( $SARejectLevel == 0 ) { |
| 236 |
- |
| 237 |
- $warnnoreject = "(*Warning* 0 = no reject)"; |
| 238 |
- |
| 239 |
-} |
| 240 |
- |
| 241 |
-# |
| 242 |
-#--------------------------------------- |
| 243 |
-# Scan the qpsmtpd log file |
| 244 |
-#--------------------------------------- |
| 245 |
- |
| 246 |
- |
| 247 |
-# Init the hashes |
| 248 |
-my $nhour = floor( $start / 3600 ); |
| 249 |
-while ( $nhour < $end / 3600 ) { |
| 250 |
- $MiscDenybyhour{$nhour}=0; |
| 251 |
- $RBLbyhour{$nhour}=0; |
| 252 |
- $above15byhour{$nhour}=0; |
| 253 |
- $patternfilterbyhour{$nhour}=0; |
| 254 |
- $WebMailbyhour{$nhour}=0; |
| 255 |
- $nhour++; |
| 256 |
-} |
| 257 |
- |
| 258 |
-my $starttai = Time::TAI64::unixtai64n($start); |
| 259 |
-my $endtai = Time::TAI64::unixtai64n($end); |
| 260 |
- |
| 261 |
-LINE: while (<>) { |
| 262 |
- my($tai,$log) = split(' ',$_,2); |
| 263 |
- |
| 264 |
- |
| 265 |
- #If date specified, only process lines matching date |
| 266 |
- next LINE if ( $tai lt $starttai ); |
| 267 |
- last if ( $tai gt $endtai ); |
| 268 |
- |
| 269 |
- #only select Logterse output |
| 270 |
- next LINE unless m/terse plugin/; |
| 271 |
- |
| 272 |
- |
| 273 |
- my $abstime = Time::TAI64::tai2unix($tai); |
| 274 |
- my $abshour = floor( $abstime / 3600 ); # Hours since the epoch |
| 275 |
- |
| 276 |
- |
| 277 |
- my ($timestamp_part, $log_part) = split '`'; |
| 278 |
- my (@log_items) = split $FS, $log_part; |
| 279 |
- |
| 280 |
- my (@timestamp_items) = split(' ',$timestamp_part); |
| 281 |
- |
| 282 |
- # we store the more recent recipient domain, for domain statistics |
| 283 |
- # in fact, we only store the first recipient. Could be sort of headhache |
| 284 |
- # to obtain precise stats with many recipients on more than one domain ! |
| 285 |
- my $proc = $timestamp_items[1] ; #numeric Id for the email |
| 286 |
- |
| 287 |
- $totalexamined++; |
| 288 |
- |
| 289 |
- # first spot the fetchmail and local deliveries. |
| 290 |
- |
| 291 |
- # print '<'.$log_items[1].'><'.$log_items[5].'><'.$log_items[8].">\n"; |
| 292 |
- |
| 293 |
- # Spot from local workstation |
| 294 |
- $localflag = 0; |
| 295 |
- $WebMailflag=0; |
| 296 |
- if ($log_items[1] =~ m/.*$DomainName.*/) {$localsendtotal++;$localLANbyhour{$abshour}++;$localflag=1} |
| 297 |
- # see if from localhost |
| 298 |
- elsif ($log_items[1] =~ m/.*$localhost.*/){ |
| 299 |
- # but not if it comes from fetchmail |
| 300 |
-# print $log_items[3]."\n"; |
| 301 |
- if ($log_items[3] =~ m/.*$FETCHMAIL.*/){} |
| 302 |
- else { |
| 303 |
- # might still be from mailman here |
| 304 |
-# print "got webmail\n"; |
| 305 |
- if ($log_items[3] =~ m/.*$MAILMAN.*/){$mailmansendcount++;$localsendtotal++;$localLANbyhour{$abshour}++;$localflag=1} |
| 306 |
- else { |
| 307 |
- # eliminate incoming localhost spoofs |
| 308 |
- if ($log_items[8] =~ m/.*msg denied before queued.*/){} |
| 309 |
- else {$localflag = 1;$WebMailsendtotal++;$WebMailbyhour{$abshour}++;$WebMailflag=1} |
| 310 |
- } |
| 311 |
- } |
| 312 |
- } |
| 313 |
- |
| 314 |
- # try to spot fetchmail emails |
| 315 |
- if ($log_items[0] =~ m/.*$FetchmailIP.*/) {$localAccepttotal++;$localacceptbyhour{$abshour}++} |
| 316 |
- elsif ($log_items[3] =~ m/.*$FETCHMAIL.*/) {$localAccepttotal++;$localacceptbyhour{$abshour}++} |
| 317 |
- |
| 318 |
- |
| 319 |
- # and adjust for recipient field if not set-up by denying plugin - extract from deny msg |
| 320 |
- |
| 321 |
- if (length($log_items[4])==0) { |
| 322 |
- if ($log_items[5] eq 'check_goodrcptto') { |
| 323 |
- if ($log_items[7] gt "invalid recipient") { |
| 324 |
- $log_items[4] = substr($log_items[7],18) #Leave only email address |
| 325 |
- } |
| 326 |
- } |
| 327 |
- } |
| 328 |
- |
| 329 |
- |
| 330 |
- |
| 331 |
- if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) { |
| 332 |
- $currentrcptdomain{ $proc } = lc($log_items[4]) ; |
| 333 |
- $currentrcptdomain{ $proc } =~ s/.*@//; |
| 334 |
- |
| 335 |
- |
| 336 |
- |
| 337 |
-# print $proc,$log_items[4]."\n"; |
| 338 |
- |
| 339 |
- $currentrcptdomain{ $proc } =~ s/[^\w\-\.]//g ; |
| 340 |
- |
| 341 |
-# print $currentrcptdomain{ $proc }."\n"; |
| 342 |
- |
| 343 |
- my $NotableDomain = 0 ; |
| 344 |
- if ( defined ( $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'type' } ) ) { |
| 345 |
- $NotableDomain = 1 ; |
| 346 |
- } else { |
| 347 |
- foreach ( @extdomain ) { |
| 348 |
- if ( $currentrcptdomain{ $proc } =~ m/$_$/ ) { |
| 349 |
- $NotableDomain = 1 ; |
| 350 |
- last ; |
| 351 |
- } |
| 352 |
- } |
| 353 |
- } |
| 354 |
- if ( !$NotableDomain ) { |
| 355 |
- # check for outgoing email |
| 356 |
- if ($localflag==1) {$currentrcptdomain{ $proc } = 'Outgoing'} |
| 357 |
- else {$currentrcptdomain{ $proc } = 'Others'} |
| 358 |
- } else { |
| 359 |
- if ($localflag==1) {$currentrcptdomain{ $proc } = 'Internal'} |
| 360 |
- } |
| 361 |
- $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'total' }++ ; |
| 362 |
- } else { |
| 363 |
- # there more than a recipient for a mail, how many daily ? |
| 364 |
- $morethanonercpt++; |
| 365 |
- } |
| 366 |
- |
| 367 |
- # then categorise the result |
| 368 |
- if (exists $log_items[5]) { |
| 369 |
- #Check for badly formed lines (from earlier testing) |
| 370 |
- |
| 371 |
- if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 372 |
- |
| 373 |
- if ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 374 |
- |
| 375 |
- if ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 376 |
- |
| 377 |
- if ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 378 |
- |
| 379 |
- if ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 380 |
- |
| 381 |
- if ($log_items[5] eq 'rhsbl') { $RBLcount++;$RBLbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 382 |
- |
| 383 |
- if ($log_items[5] eq 'dnsbl') { $RBLcount++;$RBLbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 384 |
- |
| 385 |
- if ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 386 |
- |
| 387 |
- if ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 388 |
- |
| 389 |
- if ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 390 |
- |
| 391 |
- if ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 392 |
- |
| 393 |
- if ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 394 |
- |
| 395 |
- if ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 396 |
- |
| 397 |
- if ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$patternfilterbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 398 |
- |
| 399 |
- if ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$patternfilterbyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 400 |
- |
| 401 |
- if ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 402 |
- |
| 403 |
- if ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 404 |
- |
| 405 |
- if ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$MiscDenybyhour{$abshour}++;mark_domain_rejected($proc);next LINE} |
| 406 |
- |
| 407 |
- if ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one. |
| 408 |
- |
| 409 |
- if ($log_items[5] eq 'spamassassin') { $above15++;$above15byhour{$abshour}++; |
| 410 |
- # and extract the spam score |
| 411 |
- if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1} |
| 412 |
- mark_domain_rejected($proc); |
| 413 |
- next LINE |
| 414 |
- } |
| 415 |
- |
| 416 |
- if ($log_items[5] eq 'virus::clamav') { $infectedcount++;$infectedbyhour{$abshour}++; |
| 417 |
- #extract the virus name |
| 418 |
- if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;} |
| 419 |
- mark_domain_rejected($proc); |
| 420 |
- next LINE |
| 421 |
- } |
| 422 |
- |
| 423 |
- if ($log_items[5] eq 'queued') { $Accepttotal++; |
| 424 |
- #extract the spam score |
| 425 |
- if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") { |
| 426 |
- $score = $1; |
| 427 |
-# print $log_items[8]."<".$score.">\n"; |
| 428 |
- if ($score < $SATagLevel) { $hamcount++;$hambyhour{$abshour}++;$hamavg += $score} |
| 429 |
- else {$spamcount++;$spambyhour{$abshour}++;$spamavg += $score} |
| 430 |
- } |
| 431 |
- if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { |
| 432 |
- $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; |
| 433 |
- $currentrcptdomain{ $proc } = '' ; |
| 434 |
- } |
| 435 |
- next LINE |
| 436 |
- } |
| 437 |
- |
| 438 |
- print $log_items[5]."\n"; #Not detected |
| 439 |
- |
| 440 |
- } |
| 441 |
- |
| 442 |
- |
| 443 |
- |
| 444 |
-} |
| 445 |
- |
| 446 |
-my $QueryNoLogTerse = ($totalexamined==0); #might indicate logterse not installed in qpsmtpd plugins |
| 447 |
- |
| 448 |
-#Calculate some numbers |
| 449 |
- |
| 450 |
-$spamavg = $spamavg / $spamcount if $spamcount; |
| 451 |
-$rejectspamavg = $rejectspamavg / $above15 if $above15; |
| 452 |
-$hamavg = $hamavg / $hamcount if $hamcount; |
| 453 |
- |
| 454 |
-# RBL etc percent of total SMTP sessions |
| 455 |
- |
| 456 |
-my $rblpercent = ( ( $RBLcount / $totalexamined ) * 100 ) if $totalexamined; |
| 457 |
-my $PatternFilterpercent = ( ( $PatternFilterCount / $totalexamined ) * 100 ) if $totalexamined; |
| 458 |
-my $Miscpercent = ( ( $MiscDenyCount / $totalexamined ) * 100 ) if $totalexamined; |
| 459 |
- |
| 460 |
-#Spam and virus percent of total email downloaded |
| 461 |
-#Expressed as a % of total examined |
| 462 |
-my $spampercent = ( ( $spamcount / $totalexamined ) * 100 ) if $totalexamined; |
| 463 |
-my $hampercent = ( ( $hamcount / $totalexamined ) * 100 ) if $totalexamined; |
| 464 |
-my $hrsinperiod = ( ( $end - $start ) / 3600 ); |
| 465 |
-my $emailperhour = ( $totalexamined / $hrsinperiod ) if $totalexamined; |
| 466 |
-my $above15percent = ( $above15 / $totalexamined * 100 ) if $totalexamined; |
| 467 |
-my $infectedpercent = ( ( $infectedcount / ($totalexamined) ) * 100 ) if $totalexamined; |
| 468 |
-my $AcceptPercent = ( ( $Accepttotal / ($totalexamined) ) * 100 ) if $totalexamined; |
| 469 |
- |
| 470 |
-my $oldfh; |
| 471 |
- |
| 472 |
-#Open Sendmail if we are mailing it |
| 473 |
-if ( $opt{'mail'} && !$disabled ) { |
| 474 |
- open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" ) |
| 475 |
- or die "Can't open sendmail: $!\n"; |
| 476 |
- print SENDMAIL "From: $opt{'from'}\n"; |
| 477 |
- print SENDMAIL "To: $opt{'mail'}\n"; |
| 478 |
- print SENDMAIL "Subject: Spam Filter Statistics from $hostname - ", |
| 479 |
- strftime( "%F", localtime($start) ), "\n\n"; |
| 480 |
- $oldfh = select SENDMAIL; |
| 481 |
-} |
| 482 |
- |
| 483 |
-my $telapsed = time - $tstart; |
| 484 |
- |
| 485 |
-if ( !$disabled ) { |
| 486 |
- |
| 487 |
- #Output results |
| 488 |
- print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n"; |
| 489 |
- print "----------------------------------------------------", "\n\n"; |
| 490 |
- |
| 491 |
- print "$0 Version : $opt{'version'}", "\n\n"; |
| 492 |
- print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n"; |
| 493 |
- print "Period Ending : ", strftime( "%c", localtime($end) ), "\n"; |
| 494 |
- print "\n"; |
| 495 |
- |
| 496 |
- print "Clam Version : ", `freshclam -V`; |
| 497 |
- print "SpamAssassin Version : ", `spamassassin -V`; |
| 498 |
- printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel, |
| 499 |
- $SARejectLevel; |
| 500 |
- |
| 501 |
- print "\n"; |
| 502 |
- printf "Reporting Period : %.2f hrs\n", $hrsinperiod; |
| 503 |
- print "----------------------------\n"; |
| 504 |
- print "\n"; |
| 505 |
- |
| 506 |
- printf "All SMTP connections accepted : %8d \n", $totalexamined; |
| 507 |
- |
| 508 |
- if ($localAccepttotal>0) { |
| 509 |
- printf "Connections from Fetchmail : %8d \n", |
| 510 |
- $localAccepttotal; |
| 511 |
- } |
| 512 |
- |
| 513 |
- if ($WebMailsendtotal>0) { |
| 514 |
- printf "Emails sent from WebMail : %8d \n", |
| 515 |
- $WebMailsendtotal; |
| 516 |
- } |
| 517 |
- |
| 518 |
- if ($mailmansendcount > 0) { |
| 519 |
- printf "Emails sent from Mailman : %8d \n", |
| 520 |
- $mailmansendcount; |
| 521 |
- } |
| 522 |
- |
| 523 |
- printf "SMTP from local workstations : %8d \n\n", $localsendtotal; |
| 524 |
- |
| 525 |
- |
| 526 |
- printf "RBL rejected : %8d (%6.2f%%)\n", $RBLcount, |
| 527 |
- $rblpercent || 0; |
| 528 |
- printf "Pattern filter rejected : %8d (%6.2f%%)\n", |
| 529 |
- $PatternFilterCount, $PatternFilterpercent || 0; |
| 530 |
- printf "Rejected due to non conformance : %8d (%6.2f%%)\n", $MiscDenyCount, |
| 531 |
- $Miscpercent || 0; |
| 532 |
- |
| 533 |
- printf "Infected by Virus : %8d (%6.2f%%)\n", $infectedcount, |
| 534 |
- $infectedpercent || 0; |
| 535 |
- |
| 536 |
- printf "Spam rejected (over reject level): %8d (%6.2f%%)\n", $above15, |
| 537 |
- $above15percent || 0; |
| 538 |
- printf "Spam detected (over tag level) : %8d (%6.2f%%)\n", $spamcount, |
| 539 |
- $spampercent || 0; |
| 540 |
- printf "Ham detected (under tag level) : %8d (%6.2f%%)\n", $hamcount, |
| 541 |
- $hampercent || 0; |
| 542 |
- print " --------------------\n"; |
| 543 |
- printf "Total emails accepted : %8d (%6.2f%%)\n", $Accepttotal, |
| 544 |
- $AcceptPercent || 0; |
| 545 |
- |
| 546 |
- printf "Emails per hour : (%8.1f/hr)\n", $emailperhour || 0; |
| 547 |
- print "\n"; |
| 548 |
- printf "Average spam score (accepted): %11.2f\n", $spamavg || 0; |
| 549 |
- printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0; |
| 550 |
- printf "Average ham score : %11.2f\n", $hamavg || 0; |
| 551 |
- print "\n"; |
| 552 |
- print "Statistics by Hour\n"; |
| 553 |
- if ($localAccepttotal>0) { |
| 554 |
- print "------------------------------------------------------------------------------------------------- \n"; |
| 555 |
- print |
| 556 |
- "Hour Fetchml Local WebMail Virus Spam Ham RBL/DNS$rblnotset Execut. Non.Conf.\n"; |
| 557 |
- print "-------------- -------- -------- -------- -------- -------- -------- -------- -------- ---------\n"; |
| 558 |
- |
| 559 |
- my $hour = floor( $start / 3600 ); |
| 560 |
- while ( $hour < $end / 3600 ) { |
| 561 |
- printf( |
| 562 |
- "%s %8d %8d %8d %8d %8d %8d %8d %8d %8d\n", |
| 563 |
- strftime( "%F, %H", localtime( $hour * 3600 ) ), |
| 564 |
- $localacceptbyhour{$hour} || 0, |
| 565 |
- $localLANbyhour{$hour} || 0, |
| 566 |
- $WebMailbyhour{$hour} || 0, |
| 567 |
- $infectedbyhour{$hour} || 0, |
| 568 |
- $spambyhour{$hour} || 0, |
| 569 |
- $hambyhour{$hour} || 0, |
| 570 |
- $RBLbyhour{$hour} || 0, |
| 571 |
- $patternfilterbyhour{$hour} || 0, |
| 572 |
- $MiscDenybyhour{$hour} || 0 |
| 573 |
- ); |
| 574 |
- $hour++; |
| 575 |
- } |
| 576 |
- print "------------------------------------------------------------------------------------------------- \n"; |
| 577 |
- |
| 578 |
- } else { |
| 579 |
- print "---------------------------------------------------------------------------------------- \n"; |
| 580 |
- print |
| 581 |
- "Hour Local WebMail Virus Spam Ham RBL/DNS$rblnotset Execut. Non.Conf.\n"; |
| 582 |
- print "--------------- -------- -------- -------- -------- -------- -------- -------- --------- \n"; |
| 583 |
- |
| 584 |
- my $hour = floor( $start / 3600 ); |
| 585 |
- while ( $hour < $end / 3600 ) { |
| 586 |
- printf( |
| 587 |
- "%s %8d %8d %8d %8d %8d %8d %8d %8d\n", |
| 588 |
- strftime( "%F, %H", localtime( $hour * 3600 ) ), |
| 589 |
- $localLANbyhour{$hour} || 0, |
| 590 |
- $WebMailbyhour{$hour} || 0, |
| 591 |
- $infectedbyhour{$hour} || 0, |
| 592 |
- $spambyhour{$hour} || 0, |
| 593 |
- $hambyhour{$hour} || 0, |
| 594 |
- $RBLbyhour{$hour} || 0, |
| 595 |
- $patternfilterbyhour{$hour} || 0, |
| 596 |
- $MiscDenybyhour{$hour} || 0 |
| 597 |
- ); |
| 598 |
- $hour++; |
| 599 |
- } |
| 600 |
- |
| 601 |
- print "---------------------------------------------------------------------------------------- \n"; |
| 602 |
- |
| 603 |
- } |
| 604 |
- if ($localAccepttotal>0) { |
| 605 |
- print "*Fetchml* means connections from Fetchmail delivering email\n"; |
| 606 |
- } |
| 607 |
- print "*Local* means connections from workstations on local LAN.\n"; |
| 608 |
- print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol.\n"; |
| 609 |
- print " or email was to non existant address.\n"; |
| 610 |
- print "\n"; |
| 611 |
- |
| 612 |
- if ($QueryNoLogTerse) { |
| 613 |
- print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; |
| 614 |
-# print " to enable it follow the instructions at .............................\n"; |
| 615 |
- } |
| 616 |
- |
| 617 |
- |
| 618 |
- if ( !$RHSenabled || !$DNSenabled ) { |
| 619 |
- |
| 620 |
- # comment about RBL not set |
| 621 |
- print |
| 622 |
-"* - This means that one or more of the possible spam black listing services\n that are available have not been enabled.\n"; |
| 623 |
- print " You have not enabled:\n"; |
| 624 |
- |
| 625 |
- if ( !$RHSenabled ) { |
| 626 |
- print " RHSBL\n"; |
| 627 |
- } |
| 628 |
- |
| 629 |
- if ( !$DNSenabled ) { |
| 630 |
- print " DNSBL\n"; |
| 631 |
- } |
| 632 |
- |
| 633 |
- |
| 634 |
- print " To enable these you can use the following commands:\n"; |
| 635 |
- if ( !$RHSenabled ) { |
| 636 |
- print " config setprop qpsmtpd RHSBL enabled\n"; |
| 637 |
- } |
| 638 |
- |
| 639 |
- if ( !$DNSenabled ) { |
| 640 |
- print " config setprop qpsmtpd DNSBL enabled\n"; |
| 641 |
- } |
| 642 |
- |
| 643 |
- # there so much templates to expand... (PS) |
| 644 |
- print " Followed by:\n signal-event email-update and\n svc -t /service/qpsmtpd\n\n"; |
| 645 |
- } |
| 646 |
- |
| 647 |
-# if ($Webmailsendtotal > 0) {print "If you have the mailman contrib installed, then the webmail totals might include some mailman emails\n"} |
| 648 |
- |
| 649 |
- # time to do a 'by recipient domain' report |
| 650 |
- print "\nIncoming mails by recipient domains usage\n"; |
| 651 |
- print "-----------------------------------------\n"; |
| 652 |
- print |
| 653 |
- "Domains Type Total Denied XferErr Accept \%accept\n"; |
| 654 |
- print |
| 655 |
- "---------------------------- ---------- ------ ------ ------- ------ -------\n"; |
| 656 |
- my %total = ( |
| 657 |
- total => 0, |
| 658 |
- deny => 0, |
| 659 |
- xfer => 0, |
| 660 |
- accept => 0, |
| 661 |
- ); |
| 662 |
- foreach my $domain ( |
| 663 |
- sort { |
| 664 |
- join( "\.", reverse( split /\./, $a ) ) cmp |
| 665 |
- join( "\.", reverse( split /\./, $b ) ) |
| 666 |
- } keys %byrcptdomain |
| 667 |
- ) |
| 668 |
- { |
| 669 |
- next if ( ( $byrcptdomain{$domain}{'total'} || 0 ) == 0 ); |
| 670 |
- my $tp = $byrcptdomain{$domain}{'type'} || 'other'; |
| 671 |
- my $to = $byrcptdomain{$domain}{'total'} || 0; |
| 672 |
- my $de = $byrcptdomain{$domain}{'deny'} || 0; |
| 673 |
- my $xr = $byrcptdomain{$domain}{'xfer'} || 0; |
| 674 |
- my $ac = $byrcptdomain{$domain}{'accept'} || 0; |
| 675 |
- printf "%-28s %-10s %6d %6d %7d %6d %6.2f%%\n", $domain, $tp, $to, |
| 676 |
- $de, $xr, $ac, $ac * 100 / $to; |
| 677 |
- $total{'total'} += $to; |
| 678 |
- $total{'deny'} += $de; |
| 679 |
- $total{'xfer'} += $xr; |
| 680 |
- $total{'accept'} += $ac; |
| 681 |
- } |
| 682 |
- print |
| 683 |
- "---------------------------- ---------- ------ ------- ------ ------ -------\n"; |
| 684 |
- |
| 685 |
- # $total{ 'total' } can be equal to 0, bad for divisions... |
| 686 |
- my $perc1 = 0; |
| 687 |
- my $perc2 = 0; |
| 688 |
- if ( $total{'total'} != 0 ) { |
| 689 |
- $perc1 = $total{'accept'} * 100 / $total{'total'}; |
| 690 |
- $perc2 = ( ( $total{'total'} + $morethanonercpt ) / $total{'total'} ); |
| 691 |
- } |
| 692 |
- printf |
| 693 |
- "Total %6d %6d %7d %6d %6.2f%%\n\n", |
| 694 |
- $total{'total'}, $total{'deny'}, $total{'xfer'}, $total{'accept'}, |
| 695 |
- $perc1; |
| 696 |
- printf |
| 697 |
- "%d mails were processed for %d Recipients\nThe average recipients by mail is %4.2f\n\n", |
| 698 |
- $total{'total'}, ( $total{'total'} + $morethanonercpt ), $perc2; |
| 699 |
- |
| 700 |
- if ( $infectedcount > 0 ) { |
| 701 |
- show_virus_variants(); |
| 702 |
- } |
| 703 |
- |
| 704 |
-} # not disabled |
| 705 |
- |
| 706 |
-List_Junkmail(); |
| 707 |
- |
| 708 |
-if ( !$disabled ) { |
| 709 |
- |
| 710 |
- print "\nDone. Report generated in $telapsed sec.\n\n"; |
| 711 |
- |
| 712 |
- #Close Senmdmail if it was opened |
| 713 |
- if ( $opt{'mail'} ) { |
| 714 |
- select $oldfh; |
| 715 |
- close(SENDMAIL); |
| 716 |
- } |
| 717 |
- |
| 718 |
-} |
| 719 |
- |
| 720 |
-#All done |
| 721 |
-exit 0; |
| 722 |
- |
| 723 |
-############################################################################# |
| 724 |
-# Subroutines ############################################################### |
| 725 |
-############################################################################# |
| 726 |
- |
| 727 |
- |
| 728 |
-######################################## |
| 729 |
-# Process parms # |
| 730 |
-######################################## |
| 731 |
-sub parse_arg { |
| 732 |
- my $startdate = shift; |
| 733 |
- my $enddate = shift; |
| 734 |
- |
| 735 |
- my $secsinday = 86400; |
| 736 |
- my $time = 0; |
| 737 |
- |
| 738 |
- my $start = UnixDate( $startdate, "%s" ); |
| 739 |
- my $end = UnixDate( $enddate, "%s" ); |
| 740 |
- |
| 741 |
- if ( !$start && !$end ) { |
| 742 |
- $end = time; |
| 743 |
- $start = $end - $secsinday; |
| 744 |
- return ( $start, $end ); |
| 745 |
- } |
| 746 |
- |
| 747 |
- if ( !$start ) { |
| 748 |
- $start = $end - $secsinday; |
| 749 |
- return ( $start, $end ); |
| 750 |
- } |
| 751 |
- |
| 752 |
- if ( !$end ) { |
| 753 |
- $end = $start + $secsinday; |
| 754 |
- return ( $start, $end ); |
| 755 |
- } |
| 756 |
- |
| 757 |
- if ( $start > $end ) { |
| 758 |
- return ( $end, $start ); |
| 759 |
- } |
| 760 |
- |
| 761 |
- return ( $start, $end ); |
| 762 |
- |
| 763 |
-} |
| 764 |
- |
| 765 |
-sub dbg { |
| 766 |
- my $msg = shift; |
| 767 |
- |
| 768 |
- if ( $opt{debug} ) { |
| 769 |
- print STDERR $msg; |
| 770 |
- } |
| 771 |
-} |
| 772 |
- |
| 773 |
-sub List_Junkmail { |
| 774 |
- |
| 775 |
- # |
| 776 |
- # Show how many junkmails in each user's junkmail folder. |
| 777 |
- # |
| 778 |
- use esmith::AccountsDB; |
| 779 |
- my $adb = esmith::AccountsDB->open_ro; |
| 780 |
- my $entry; |
| 781 |
- foreach my $user ($adb->users) { |
| 782 |
- my $found = 0; |
| 783 |
- my $junkmail_dir = "/home/e-smith/files/users/" . |
| 784 |
- $user->key . "/Maildir/.junkmail"; |
| 785 |
-# print $user->key; |
| 786 |
- foreach my $dir (qw(new cur)) { |
| 787 |
- # Now get the content list for the directory. |
| 788 |
- if (opendir( QDIR, "$junkmail_dir/$dir" )) { |
| 789 |
- while ($entry=readdir(QDIR) ) { |
| 790 |
- next if $entry =~ /^\./; |
| 791 |
- $found++; |
| 792 |
- } |
| 793 |
- |
| 794 |
- closedir(QDIR); |
| 795 |
- } |
| 796 |
- } |
| 797 |
- if ( !$disabled ) { |
| 798 |
- printf "User \"%s\" ", $user->key; |
| 799 |
- printf "- %d email(s) left in junkmail folder\n", $found; |
| 800 |
- } |
| 801 |
- } |
| 802 |
-} |
| 803 |
- |
| 804 |
-sub show_virus_variants |
| 805 |
- |
| 806 |
-# |
| 807 |
-# Show a league table of the different virus types found today |
| 808 |
-# |
| 809 |
- |
| 810 |
-{ |
| 811 |
- |
| 812 |
- print("Virus Statistics by name:\n"); |
| 813 |
- print("---------------------------------------------\n"); |
| 814 |
- foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} } |
| 815 |
- keys %found_viruses) |
| 816 |
- { |
| 817 |
- print "Rejected $found_viruses{$virus}\t$virus\n"; |
| 818 |
- } |
| 819 |
- print("---------------------------------------------\n"); |
| 820 |
-} |
| 821 |
- |
| 822 |
-sub mark_domain_rejected |
| 823 |
- |
| 824 |
-# |
| 825 |
-# Tag domain as having a rejected email |
| 826 |
-# |
| 827 |
-{ |
| 828 |
-my ($proc) = @_; |
| 829 |
-if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { |
| 830 |
- $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'deny' }++ ; |
| 831 |
- $currentrcptdomain{ $proc } = '' ; |
| 832 |
- } |
| 833 |
-} |
| 834 |
+#!/usr/bin/perl -w |
| 835 |
+ |
| 836 |
+############################################################################# |
| 837 |
+# |
| 838 |
+# This script provides daily SpamFilter statistics and deletes all users |
| 839 |
+# junkmails. Configuration of the script is done by the Spam Filter |
| 840 |
+# Server-Manager module |
| 841 |
+# |
| 842 |
+# April 2006 - no longer controlled by server manager, and does not delete files |
| 843 |
+# |
| 844 |
+# This script has been developed |
| 845 |
+# by Jesper Knudsen at http://sme.swerts-knudsen.dk |
| 846 |
+# |
| 847 |
+# Revision History: |
| 848 |
+# |
| 849 |
+# August 13, 2003: Initial version |
| 850 |
+# August 25, 2004: fixed problem when hostname had no-ASCII chars |
| 851 |
+# March 23, 2006 Revised for sme7 RM |
| 852 |
+# March 27, 2006 ditto BJR (http://www.abandonmicrosoft.co.uk) |
| 853 |
+# - Merged Clamav and SA stats |
| 854 |
+# - Moved all analysis to qsmtpd log |
| 855 |
+# - Removed parameterised interval (for simplicity - not sure of format anyway) |
| 856 |
+# - add in archived log files for people who have high turnover |
| 857 |
+# - Alter labels to be more accurate |
| 858 |
+# - Detect deleted spam (over threshold) without using spam score |
| 859 |
+# - Detect RBL rejections |
| 860 |
+# - Detect pattern (executible) rejections |
| 861 |
+# - Look for the DENY labels - add in Miscellaneous category |
| 862 |
+# April 6, 2006 - check qpsmtp log level and also DNS enable properties |
| 863 |
+# - Average spam scores for under and over threshold seperatly |
| 864 |
+# - Log tag and Reject levels |
| 865 |
+# - TBD - check that RBL DENY are being detected (I have no date to check this) |
| 866 |
+# April 7, 2007 - re-written by Charlie Brady totally in Perl |
| 867 |
+# April 16, 2006 - move warnings to report |
| 868 |
+# - Spot fetchmail deliveries |
| 869 |
+# - Spot Internal connections from client PCs |
| 870 |
+# - TBD check that RBL DENY are being detected (I have no data to check this) |
| 871 |
+# April 30, 2006 - Pascal Schirrmann Start Time and End Time to noon - should be a param |
| 872 |
+# so the script can be run at any time in the day. |
| 873 |
+# - adds 'by recipients domains' stats Useful for MX-Backup or multi domains hosts |
| 874 |
+# - Add a 'recipients per mail' stat. Useful : until now the sums are correct :-) |
| 875 |
+# - Correct some messages about rbl who can led to wrong entry in the config database |
| 876 |
+# ( and without expected results, of course !) |
| 877 |
+# - improve a regexp in the SPAM detection |
| 878 |
+# May 1, 2006 - BJR - Fix situation where mxbackup prop is not defined |
| 879 |
+# - fix a spelling and minor format of domain report |
| 880 |
+# May 9, 2006 - bjr - Make RBL percentage a percentage of total connections (else it >100%) |
| 881 |
+# May 9, 2006 - ps - some 'sanity check' in the 'per domains part of the stats (to avoid / 0) |
| 882 |
+# May 12, 2006 - ps - some cleanup in the 'per domains' stats |
| 883 |
+# - Add a version number, logged in the mail |
| 884 |
+# June 20, 2006 - bjr - Minor change to RBL instructions, and adjust domain table format |
| 885 |
+# Feb 19, 2007 - bjr - Adjust table lines oin a couple of places |
| 886 |
+# - bjr - and add documentation details about percentages etc |
| 887 |
+# - bjr - Alter misc to "non conforming" anmd accumulated these hourly |
| 888 |
+# - bjr - Express change over tag count to exclude spam rejected over threshold |
| 889 |
+# - bjr - Change "processsed" to "fully downloaded" |
| 890 |
+# - bjr - Change percentages so that they are all a percetnage of the total emails received |
| 891 |
+# 0.6.1 - bjr - Change to use output from the logterse qpsmtpd plugin |
| 892 |
+# 0.6.2 - bjr - Fix fetchmail tests |
| 893 |
+# 0.6.3 - bjr - adjust for log-items change in order |
| 894 |
+# 0.6.4&5 - bjr - Adjust table formatting |
| 895 |
+# 0.6.6 - bjr - Take outgoing emails out of "others", add "Outgoing" and "Internal" |
| 896 |
+# 0.6.7 - bjr - Fix missing plugins/wrong names. pull invalid recipient out of deny msg for goodrcptto |
| 897 |
+# 0.6.8 - bjr - catch a few more plugin name failures |
| 898 |
+# 0.6.9 - bjr - Catch webmail and mailman |
| 899 |
+# 0.6.10 - bjr - Refine Webmail identification |
| 900 |
+# 0.6.11 - bjr - Fix Webmail identification |
| 901 |
+# 0.6.12 - bjr - split logterse line a bit more carefully (multiple sent to addresss with space and comma confuse it) |
| 902 |
+# 0.6.13 - bjr - add totals and percentages to bottom of the table |
| 903 |
+# - Generalise counts so that columns can be brought in and out |
| 904 |
+# - control columns with Db entries |
| 905 |
+# 0.6.14 - bjr - Add in league tables of qpsmtpd codes and SA rules |
| 906 |
+# - Add in loglevel check |
| 907 |
+# - parameterise email address for report |
| 908 |
+# 0.6.15 - bjr - fix columns included in totals |
| 909 |
+# - sort out domains when more that one email address in recipient field |
| 910 |
+# |
| 911 |
+# TODO |
| 912 |
+# ---- |
| 913 |
+# |
| 914 |
+# Sort junkmail counts largest to smallest |
| 915 |
+# Html output to ibay (ibay name as command line param or db entry) |
| 916 |
+# get "XferErr" column working on domains list (or remove column) |
| 917 |
+# sort out multiple emails recipients, count each one, and log multiple counts |
| 918 |
+# |
| 919 |
+# |
| 920 |
+# |
| 921 |
+############################################################################# |
| 922 |
+# |
| 923 |
+# SMEServer DB usage |
| 924 |
+# ------------------ |
| 925 |
+# |
| 926 |
+# mailstats / Status ("enabled"|"disabled") |
| 927 |
+# / <column header> ("yes"|"no"|"auto") - enable, supress or only show if nonzero |
| 928 |
+# / QpsmtpdCodes ("enabled"|"disabled") |
| 929 |
+# / SARules ("enabled"|"disabled") |
| 930 |
+# / JunkMailList ("enabled"|"disabled") |
| 931 |
+# / SARulePercentThreshold (0.5) - threshold of SArules percentage for report cutoff |
| 932 |
+# / Email (admin) - email to send report |
| 933 |
+# |
| 934 |
+# |
| 935 |
+# |
| 936 |
+# |
| 937 |
+############################################################################# |
| 938 |
+ |
| 939 |
+ |
| 940 |
+# internal modules (part of core perl distribution) |
| 941 |
+use strict; |
| 942 |
+use warnings; |
| 943 |
+use Getopt::Long; |
| 944 |
+use Pod::Usage; |
| 945 |
+use POSIX qw/strftime floor/; |
| 946 |
+use Time::Local; |
| 947 |
+use Date::Manip; |
| 948 |
+use Time::TAI64; |
| 949 |
+use esmith::ConfigDB; |
| 950 |
+use esmith::DomainsDB; |
| 951 |
+use Sys::Hostname; |
| 952 |
+use Switch; |
| 953 |
+ |
| 954 |
+my $hostname = hostname(); |
| 955 |
+ |
| 956 |
+#Configuration section |
| 957 |
+my %opt = (); |
| 958 |
+ |
| 959 |
+$opt{'version'} = '0.6.15'; # please update at each change. |
| 960 |
+$opt{'debug'} = 0; # guess what ? |
| 961 |
+$opt{'sendmail'} = '/usr/sbin/sendmail'; # Path to sendmail stub |
| 962 |
+$opt{'from'} = 'spamfilter-stats'; # Who is the mail from |
| 963 |
+$opt{'end'} = `date --iso-8601`; # midnight today |
| 964 |
+my $yesterday = $opt{ 'end' }; |
| 965 |
+$yesterday =~ s/\-//g ; |
| 966 |
+$yesterday--; |
| 967 |
+$opt{'start'} = `date --iso-8601 -d $yesterday`; # midnight yesterday |
| 968 |
+#$opt{'mail'} = "admin"; - set from db now... |
| 969 |
+$opt{'timezone'} = `date +%z`; |
| 970 |
+Date_Init("TZ=$opt{'timezone'}"); |
| 971 |
+ |
| 972 |
+my $FetchmailIP = '127.0.0.200'; #Apparent Ip address of fetchmail deliveries |
| 973 |
+my $WebmailIP = '127.0.0.1'; #Apparent Ip of Webmail sender |
| 974 |
+my $localhost = 'localhost'; #Apparent sender for webmail |
| 975 |
+my $FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email |
| 976 |
+my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost |
| 977 |
+ |
| 978 |
+my $MinCol = 8; #Minimum column width |
| 979 |
+my $HourColWidth = 16; #Date and time column width |
| 980 |
+ |
| 981 |
+my $SARulethresholdPercent = 10; #If Sa rules less than this of total emails, then cutoff reduced |
| 982 |
+my $maxcutoff = 1; #max percent cutoff applied |
| 983 |
+my $mincutoff = 0.2; #min percent cutoff applied |
| 984 |
+ |
| 985 |
+my $true = 1; |
| 986 |
+my $false = 0; |
| 987 |
+ |
| 988 |
+my $tstart = time; |
| 989 |
+ |
| 990 |
+#Local variables |
| 991 |
+my $YEAR = ( localtime(time) )[5]; # this is years since 1900 |
| 992 |
+ |
| 993 |
+my $total = 0; |
| 994 |
+my $spamcount = 0; |
| 995 |
+my $spamavg = 0; |
| 996 |
+my $hamcount = 0; |
| 997 |
+my $hamavg = 0; |
| 998 |
+my $rejectspamavg = 0; |
| 999 |
+ |
| 1000 |
+my $Accepttotal = 0; |
| 1001 |
+my $localAccepttotal = 0; #Fetchmail connections |
| 1002 |
+my $localsendtotal = 0; #Connections from local PCs |
| 1003 |
+my $totalexamined = 0; #total download + RBL etc |
| 1004 |
+my $WebMailsendtotal = 0; #total from Webmail |
| 1005 |
+my $mailmansendcount = 0; #total from mailman |
| 1006 |
+ |
| 1007 |
+my %found_viruses = (); |
| 1008 |
+my %found_qpcodes = (); |
| 1009 |
+my %found_SARules = (); |
| 1010 |
+ |
| 1011 |
+# replaced by... |
| 1012 |
+my %counts = (); #Hold all counts in 2-D matrix |
| 1013 |
+my @display = (); #used to switch on and off columns - yes, no or auto for each category |
| 1014 |
+my @colwidth = (); #width of each column |
| 1015 |
+ #(auto means only if non zero) - populated from possible db entries |
| 1016 |
+my @finaldisplay = (); #final decision on display or not - true or false |
| 1017 |
+my $disabled; |
| 1018 |
+ |
| 1019 |
+#count column names, used for headings - also used for DB mailstats property names |
| 1020 |
+my $CATHOUR='Hour'; |
| 1021 |
+my $CATFETCHMAIL='Fetchmail'; |
| 1022 |
+my $CATWEBMAIL='WebMail'; |
| 1023 |
+my $CATMAILMAN='Mailman'; |
| 1024 |
+my $CATLOCAL='Local'; |
| 1025 |
+# border between where it came from and where it ended.. |
| 1026 |
+my $countfromhere = 5; |
| 1027 |
+ |
| 1028 |
+my $CATVIRUS='Virus'; |
| 1029 |
+my $CATRBLDNS='RBL/DNS'; |
| 1030 |
+my $CATEXECUT='Execut.'; |
| 1031 |
+my $CATNONCONF='Non.Conf.'; |
| 1032 |
+my $CATSPAMDEL='Del.Spam'; |
| 1033 |
+my $CATSPAM='Qued.Spam?'; |
| 1034 |
+my $CATHAM='Ham'; |
| 1035 |
+my $CATTOTALS='TOTALS'; |
| 1036 |
+my $CATPERCENT='PERCENT'; |
| 1037 |
+my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATNONCONF,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT); |
| 1038 |
+my $GRANDTOTAL = '99'; #subs for count arrays, for grand total |
| 1039 |
+my $PERCENT = '98'; # for column percentages |
| 1040 |
+ |
| 1041 |
+my $categlen = @categs-2; #-2 to avoid the total and percent column |
| 1042 |
+ |
| 1043 |
+my $above15 = 0; |
| 1044 |
+my $RBLcount = 0; |
| 1045 |
+my $MiscDenyCount = 0; |
| 1046 |
+my $PatternFilterCount = 0; |
| 1047 |
+my $noninfectedcount = 0; |
| 1048 |
+my $okemailcount = 0; |
| 1049 |
+my $infectedcount = 0; |
| 1050 |
+my $warnnoreject = " "; |
| 1051 |
+my $rblnotset = ' '; |
| 1052 |
+ |
| 1053 |
+my $FS = "\t"; # field separator used by logterse plugin |
| 1054 |
+my %log_items = (); |
| 1055 |
+my $score; |
| 1056 |
+my %timestamp_items = (); |
| 1057 |
+my $localflag = 0; #indicate if current email is local or not |
| 1058 |
+my $WebMailflag = 0; #indicate if current mail is send from webmail |
| 1059 |
+ |
| 1060 |
+# some storage for by recipient domains stats (PS) |
| 1061 |
+# my bad : I have to deal with multiple simoultaneous connections |
| 1062 |
+# will play with the process number. |
| 1063 |
+# my $currentrcptdomain = '' ; |
| 1064 |
+my %currentrcptdomain ; # temporay store the recipient domain until end of mail processing |
| 1065 |
+my %byrcptdomain ; # Store 'by domains stats' |
| 1066 |
+my @extdomain ; # only useful in some MX-Backup case, when any subdomains are allowed |
| 1067 |
+my $morethanonercpt = 0 ; # count every 'second' recipients for a mail. |
| 1068 |
+ |
| 1069 |
+# store the domain of interest. Every other records are stored in a 'Other' zone |
| 1070 |
+my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n"; |
| 1071 |
+foreach my $domain( $ddb->get_all_by_prop( type => "domain" ) ) { |
| 1072 |
+ $byrcptdomain{ $domain->key }{ 'type' }='local'; |
| 1073 |
+} |
| 1074 |
+$byrcptdomain{ esmith::ConfigDB->open_ro->get('SystemName')->value . "." |
| 1075 |
+ . esmith::ConfigDB->open_ro->get('DomainName')->value }{ 'type' } = 'local'; |
| 1076 |
+ |
| 1077 |
+# is this system a MX-Backup ? |
| 1078 |
+if (esmith::ConfigDB->open_ro->get('mxbackup')){ |
| 1079 |
+ if ( ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('status') || 'disabled' ) eq 'enabled' ) { |
| 1080 |
+ my %MXValues = split( /,/, ( esmith::ConfigDB->open_ro->get('mxbackup')->prop('name') || '' ) ) ; |
| 1081 |
+ foreach my $data ( keys %MXValues ) { |
| 1082 |
+ $byrcptdomain{ $data }{ 'type' } = "mxbackup-$MXValues{ $data }" ; |
| 1083 |
+ if ( $MXValues{ $data } == 1 ) { # subdomains allowed, must take care of this |
| 1084 |
+ push @extdomain, $data ; |
| 1085 |
+ } |
| 1086 |
+ } |
| 1087 |
+ } |
| 1088 |
+} |
| 1089 |
+ |
| 1090 |
+my ( $start, $end ) = parse_arg( $opt{'start'}, $opt{'end'} ); |
| 1091 |
+ |
| 1092 |
+# |
| 1093 |
+# First check current configuration for logging, DNS enable and Max threshold for spamassassin |
| 1094 |
+# |
| 1095 |
+ |
| 1096 |
+my $LogLevel = esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('LogLevel'); |
| 1097 |
+my $HighLogLevel = ( $LogLevel > 6 ); |
| 1098 |
+ |
| 1099 |
+my $RHSenabled = |
| 1100 |
+ ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('RHSBL') eq 'enabled' ); |
| 1101 |
+my $DNSenabled = |
| 1102 |
+ ( esmith::ConfigDB->open_ro->get('qpsmtpd')->prop('DNSBL') eq 'enabled' ); |
| 1103 |
+my $SARejectLevel = |
| 1104 |
+ esmith::ConfigDB->open_ro->get('spamassassin')->prop('RejectLevel'); |
| 1105 |
+my $SATagLevel = |
| 1106 |
+ esmith::ConfigDB->open_ro->get('spamassassin')->prop('TagLevel'); |
| 1107 |
+my $DomainName = |
| 1108 |
+ esmith::ConfigDB->open_ro->get('DomainName')->value; |
| 1109 |
+ |
| 1110 |
+# check that logterse is in use |
| 1111 |
+#my pluginfile = '/var/service/qpsmtpd/config/peers/0'; |
| 1112 |
+ |
| 1113 |
+#and see if mailstats are disabled |
| 1114 |
+if (esmith::ConfigDB->open_ro->get('mailstats')){ |
| 1115 |
+ $disabled = !((esmith::ConfigDB->open_ro->get('mailstats')->prop('Status') || 'enabled') eq 'enabled'); |
| 1116 |
+} else { |
| 1117 |
+ my $db = esmith::ConfigDB->open; my $record = $db->new_record('mailstats', { type => 'report', Status => 'enabled' }); |
| 1118 |
+ $disabled = $false; |
| 1119 |
+} |
| 1120 |
+# check mailstats email recipient |
| 1121 |
+my $email = esmith::ConfigDB->open_ro->get('mailstats')->prop('Email') || 'admin'; |
| 1122 |
+$opt{'mail'} = $email; |
| 1123 |
+ |
| 1124 |
+ |
| 1125 |
+if ( !$RHSenabled || !$DNSenabled ) { |
| 1126 |
+ $rblnotset = '*'; |
| 1127 |
+} |
| 1128 |
+ |
| 1129 |
+if ( $SARejectLevel == 0 ) { |
| 1130 |
+ |
| 1131 |
+ $warnnoreject = "(*Warning* 0 = no reject)"; |
| 1132 |
+ |
| 1133 |
+} |
| 1134 |
+ |
| 1135 |
+# |
| 1136 |
+#--------------------------------------- |
| 1137 |
+# Scan the qpsmtpd log file |
| 1138 |
+#--------------------------------------- |
| 1139 |
+ |
| 1140 |
+ |
| 1141 |
+# Init the hashes |
| 1142 |
+my $nhour = floor( $start / 3600 ); |
| 1143 |
+my $ncateg; |
| 1144 |
+while ( $nhour < $end / 3600 ) { |
| 1145 |
+ $counts{$nhour}=(); |
| 1146 |
+ $ncateg = 0; |
| 1147 |
+ while ( $ncateg < @categs) { |
| 1148 |
+ $counts{$nhour}{$categs[$ncateg-1]} = 0; |
| 1149 |
+ $ncateg++ |
| 1150 |
+ } |
| 1151 |
+ $nhour++; |
| 1152 |
+} |
| 1153 |
+# and grand totals and display status from db entries, and column widths |
| 1154 |
+$ncateg = 0; |
| 1155 |
+while ( $ncateg < @categs) { |
| 1156 |
+ $counts{$GRANDTOTAL}{$categs[$ncateg]} = 0; |
| 1157 |
+ if (esmith::ConfigDB->open_ro->get('mailstats')){ |
| 1158 |
+ $display[$ncateg] = lc(esmith::ConfigDB->open_ro->get('mailstats')->prop($categs[$ncateg])) || "auto"; |
| 1159 |
+ } else { |
| 1160 |
+ $display[$ncateg] = 'auto' |
| 1161 |
+ } |
| 1162 |
+ if ($ncateg == 0) { |
| 1163 |
+ $colwidth[$ncateg] = $HourColWidth |
| 1164 |
+ } else { |
| 1165 |
+ $colwidth[$ncateg] = length($categs[$ncateg])+1 |
| 1166 |
+ } |
| 1167 |
+ if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol} |
| 1168 |
+ $ncateg++ |
| 1169 |
+} |
| 1170 |
+ |
| 1171 |
+my $starttai = Time::TAI64::unixtai64n($start); |
| 1172 |
+my $endtai = Time::TAI64::unixtai64n($end); |
| 1173 |
+my $sum_SARules = 0; |
| 1174 |
+ |
| 1175 |
+LINE: while (<>) { |
| 1176 |
+ my($tai,$log) = split(' ',$_,2); |
| 1177 |
+ |
| 1178 |
+ |
| 1179 |
+ #If date specified, only process lines matching date |
| 1180 |
+ next LINE if ( $tai lt $starttai ); |
| 1181 |
+ last if ( $tai gt $endtai ); |
| 1182 |
+ |
| 1183 |
+ # pull out spamasassin rule lists |
| 1184 |
+ if ($_ =~ m/spamassassin plugin: check_spam:.*tests=(.*)/) { |
| 1185 |
+ my ($SAtests) = split(',',$1); |
| 1186 |
+ foreach my $SAtest ($SAtests) { |
| 1187 |
+ if (!$SAtest eq "") { |
| 1188 |
+ $found_SARules{$SAtest}++; |
| 1189 |
+ $sum_SARules++ |
| 1190 |
+ } |
| 1191 |
+ } |
| 1192 |
+ |
| 1193 |
+ } |
| 1194 |
+ #only select Logterse output |
| 1195 |
+ next LINE unless m/terse plugin/; |
| 1196 |
+ |
| 1197 |
+ |
| 1198 |
+ my $abstime = Time::TAI64::tai2unix($tai); |
| 1199 |
+ my $abshour = floor( $abstime / 3600 ); # Hours since the epoch |
| 1200 |
+ |
| 1201 |
+ |
| 1202 |
+ my ($timestamp_part, $log_part) = split('`',$_,2); #bjr 0.6.12 |
| 1203 |
+ my (@log_items) = split $FS, $log_part; |
| 1204 |
+ |
| 1205 |
+ my (@timestamp_items) = split(' ',$timestamp_part); |
| 1206 |
+ |
| 1207 |
+ # we store the more recent recipient domain, for domain statistics |
| 1208 |
+ # in fact, we only store the first recipient. Could be sort of headhache |
| 1209 |
+ # to obtain precise stats with many recipients on more than one domain ! |
| 1210 |
+ my $proc = $timestamp_items[1] ; #numeric Id for the email |
| 1211 |
+ |
| 1212 |
+ $totalexamined++; |
| 1213 |
+ |
| 1214 |
+ # first spot the fetchmail and local deliveries. |
| 1215 |
+ |
| 1216 |
+ # print '<'.$log_items[1].'><'.$log_items[5].'><'.$log_items[8].">\n"; |
| 1217 |
+ |
| 1218 |
+ # Spot from local workstation |
| 1219 |
+ $localflag = 0; |
| 1220 |
+ $WebMailflag=0; |
| 1221 |
+ if ($log_items[1] =~ m/.*$DomainName.*/) {$localsendtotal++;$counts{$abshour}{$CATLOCAL}++;$localflag=1} |
| 1222 |
+ # see if from localhost |
| 1223 |
+ elsif ($log_items[1] =~ m/.*$localhost.*/){ |
| 1224 |
+ # but not if it comes from fetchmail |
| 1225 |
+# print $log_items[3]."\n"; |
| 1226 |
+ if ($log_items[3] =~ m/.*$FETCHMAIL.*/){} |
| 1227 |
+ else { |
| 1228 |
+ # might still be from mailman here |
| 1229 |
+# print "got webmail\n"; |
| 1230 |
+ if ($log_items[3] =~ m/.*$MAILMAN.*/){$mailmansendcount++;$localsendtotal++;$counts{$abshour}{$CATMAILMAN}++;$localflag=1} |
| 1231 |
+ else { |
| 1232 |
+ # eliminate incoming localhost spoofs |
| 1233 |
+ if ($log_items[8] =~ m/.*msg denied before queued.*/){} |
| 1234 |
+ else {$localflag = 1;$WebMailsendtotal++;$counts{$abshour}{$CATWEBMAIL}++;$WebMailflag=1} |
| 1235 |
+ } |
| 1236 |
+ } |
| 1237 |
+ } |
| 1238 |
+ |
| 1239 |
+ # try to spot fetchmail emails |
| 1240 |
+ if ($log_items[0] =~ m/.*$FetchmailIP.*/) {$localAccepttotal++;$counts{$abshour}{$CATFETCHMAIL}++} |
| 1241 |
+ elsif ($log_items[3] =~ m/.*$FETCHMAIL.*/) {$localAccepttotal++;$counts{$abshour}{$CATFETCHMAIL}++} |
| 1242 |
+ |
| 1243 |
+ |
| 1244 |
+ # and adjust for recipient field if not set-up by denying plugin - extract from deny msg |
| 1245 |
+ |
| 1246 |
+ if (length($log_items[4])==0) { |
| 1247 |
+ if ($log_items[5] eq 'check_goodrcptto') { |
| 1248 |
+ if ($log_items[7] gt "invalid recipient") { |
| 1249 |
+ $log_items[4] = substr($log_items[7],18) #Leave only email address |
| 1250 |
+ } |
| 1251 |
+ } |
| 1252 |
+ } |
| 1253 |
+ |
| 1254 |
+ if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) { |
| 1255 |
+ # reduce to lc and only take first email address in a list |
| 1256 |
+ my $recipientmail = lc($log_items[4]); |
| 1257 |
+ if ($recipientmail =~ m/.*,/) { |
| 1258 |
+ #comma |
| 1259 |
+ $recipientmail =~ m/(.*),/; |
| 1260 |
+ $currentrcptdomain{ $proc } = $1; |
| 1261 |
+ } else { |
| 1262 |
+ $currentrcptdomain{ $proc } = lc($log_items[4]) |
| 1263 |
+ } |
| 1264 |
+ #split to just domain bit. |
| 1265 |
+ $currentrcptdomain{ $proc } =~ s/.*@//; |
| 1266 |
+ $currentrcptdomain{ $proc } =~ s/[^\w\-\.]//g ; |
| 1267 |
+ $currentrcptdomain{ $proc } =~ s/>//g ; |
| 1268 |
+ |
| 1269 |
+# print $currentrcptdomain{ $proc }."\n"; |
| 1270 |
+ |
| 1271 |
+ my $NotableDomain = 0 ; |
| 1272 |
+ if ( defined ( $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'type' } ) ) { |
| 1273 |
+ $NotableDomain = 1 ; |
| 1274 |
+ } else { |
| 1275 |
+ foreach ( @extdomain ) { |
| 1276 |
+ if ( $currentrcptdomain{ $proc } =~ m/$_$/ ) { |
| 1277 |
+ $NotableDomain = 1 ; |
| 1278 |
+ last ; |
| 1279 |
+ } |
| 1280 |
+ } |
| 1281 |
+ } |
| 1282 |
+ if ( !$NotableDomain ) { |
| 1283 |
+ # check for outgoing email |
| 1284 |
+ if ($localflag==1) {$currentrcptdomain{ $proc } = 'Outgoing'} |
| 1285 |
+ else {$currentrcptdomain{ $proc } = 'Others'} |
| 1286 |
+ } else { |
| 1287 |
+ if ($localflag==1) {$currentrcptdomain{ $proc } = 'Internal'} |
| 1288 |
+ } |
| 1289 |
+ $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'total' }++ ; |
| 1290 |
+ } else { |
| 1291 |
+ # there more than a recipient for a mail, how many daily ? |
| 1292 |
+ $morethanonercpt++; |
| 1293 |
+ } |
| 1294 |
+ |
| 1295 |
+ # then categorise the result |
| 1296 |
+ |
| 1297 |
+ |
| 1298 |
+ if (exists $log_items[5]) { |
| 1299 |
+ |
| 1300 |
+ $found_qpcodes{$log_items[5]}++; ##Count different qpsmtpd result codes |
| 1301 |
+ |
| 1302 |
+ #Check for badly formed lines (from earlier testing) |
| 1303 |
+ |
| 1304 |
+ if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1305 |
+ |
| 1306 |
+ if ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1307 |
+ |
| 1308 |
+ if ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1309 |
+ |
| 1310 |
+ if ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1311 |
+ |
| 1312 |
+ if ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1313 |
+ |
| 1314 |
+ if ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} |
| 1315 |
+ |
| 1316 |
+ if ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} |
| 1317 |
+ |
| 1318 |
+ if ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1319 |
+ |
| 1320 |
+ if ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1321 |
+ |
| 1322 |
+ if ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1323 |
+ |
| 1324 |
+ if ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1325 |
+ |
| 1326 |
+ if ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1327 |
+ |
| 1328 |
+ if ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1329 |
+ |
| 1330 |
+ if ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE} |
| 1331 |
+ |
| 1332 |
+ if ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE} |
| 1333 |
+ |
| 1334 |
+ if ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1335 |
+ |
| 1336 |
+ if ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1337 |
+ |
| 1338 |
+ if ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} |
| 1339 |
+ |
| 1340 |
+ if ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one. |
| 1341 |
+ |
| 1342 |
+ if ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++; |
| 1343 |
+ # and extract the spam score |
| 1344 |
+ if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1} |
| 1345 |
+ mark_domain_rejected($proc); |
| 1346 |
+ next LINE |
| 1347 |
+ } |
| 1348 |
+ |
| 1349 |
+ if ($log_items[5] eq 'virus::clamav') { $infectedcount++;$counts{$abshour}{$CATVIRUS}++; |
| 1350 |
+ #extract the virus name |
| 1351 |
+ if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;} |
| 1352 |
+ mark_domain_rejected($proc); |
| 1353 |
+ next LINE |
| 1354 |
+ } |
| 1355 |
+ |
| 1356 |
+ if ($log_items[5] eq 'queued') { $Accepttotal++; |
| 1357 |
+ #extract the spam score |
| 1358 |
+ if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") { |
| 1359 |
+ $score = $1; |
| 1360 |
+# print $log_items[8]."<".$score.">\n"; |
| 1361 |
+ if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score} |
| 1362 |
+ else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score} |
| 1363 |
+ } else { |
| 1364 |
+ # no SA score - so it must be ham |
| 1365 |
+ $hamcount++;$counts{$abshour}{$CATHAM}++; |
| 1366 |
+ } |
| 1367 |
+ if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { |
| 1368 |
+ $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; |
| 1369 |
+ $currentrcptdomain{ $proc } = '' ; |
| 1370 |
+ } |
| 1371 |
+ next LINE |
| 1372 |
+ } |
| 1373 |
+ |
| 1374 |
+ print $log_items[5]."\n"; #Not detected |
| 1375 |
+ |
| 1376 |
+ } |
| 1377 |
+ |
| 1378 |
+} #END OF MAIN LOOP |
| 1379 |
+ |
| 1380 |
+#total up grand total Columns |
| 1381 |
+$nhour = floor( $start / 3600 ); |
| 1382 |
+while ( $nhour < $end / 3600 ) { |
| 1383 |
+ $ncateg = 0; #past the where it came from columns |
| 1384 |
+ while ( $ncateg < @categs) { |
| 1385 |
+ #total columns |
| 1386 |
+ $counts{$GRANDTOTAL}{$categs[$ncateg]} += $counts{$nhour}{$categs[$ncateg]}; |
| 1387 |
+ |
| 1388 |
+ # and total rows |
| 1389 |
+ if ( $ncateg < $categlen && $ncateg>=$countfromhere) {#skip initial columns of non final reasons |
| 1390 |
+ $counts{$nhour}{$categs[@categs-2]} += $counts{$nhour}{$categs[$ncateg]}; |
| 1391 |
+ } |
| 1392 |
+ $ncateg++ |
| 1393 |
+ } |
| 1394 |
+ |
| 1395 |
+ $nhour++; |
| 1396 |
+} |
| 1397 |
+ |
| 1398 |
+ |
| 1399 |
+ |
| 1400 |
+#Compute row totals and row percentages |
| 1401 |
+$nhour = floor( $start / 3600 ); |
| 1402 |
+while ( $nhour < $end / 3600 ) { |
| 1403 |
+ $counts{$nhour}{$categs[@categs-1]} = $counts{$nhour}{$categs[@categs-2]}*100/$totalexamined if $totalexamined; |
| 1404 |
+ $nhour++; |
| 1405 |
+ |
| 1406 |
+} |
| 1407 |
+ |
| 1408 |
+#compute column percentages |
| 1409 |
+ $ncateg = 0; |
| 1410 |
+ while ( $ncateg < @categs) { |
| 1411 |
+ if ($ncateg == @categs-1) { |
| 1412 |
+ $counts{$PERCENT}{$categs[$ncateg]} = $counts{$GRANDTOTAL}{$categs[$ncateg-1]}*100/$totalexamined if $totalexamined; |
| 1413 |
+ } else { |
| 1414 |
+ $counts{$PERCENT}{$categs[$ncateg]} = $counts{$GRANDTOTAL}{$categs[$ncateg]}*100/$totalexamined if $totalexamined; |
| 1415 |
+ } |
| 1416 |
+ $ncateg++ |
| 1417 |
+ } |
| 1418 |
+ |
| 1419 |
+#compute sum of row percentages |
| 1420 |
+$nhour = floor( $start / 3600 ); |
| 1421 |
+while ( $nhour < $end / 3600 ) { |
| 1422 |
+ $counts{$GRANDTOTAL}{$categs[@categs-1]} += $counts{$nhour}{$categs[@categs-1]}; |
| 1423 |
+ $nhour++; |
| 1424 |
+ |
| 1425 |
+} |
| 1426 |
+ |
| 1427 |
+my $QueryNoLogTerse = ($totalexamined==0); #might indicate logterse not installed in qpsmtpd plugins |
| 1428 |
+ |
| 1429 |
+#Calculate some numbers |
| 1430 |
+ |
| 1431 |
+$spamavg = $spamavg / $spamcount if $spamcount; |
| 1432 |
+$rejectspamavg = $rejectspamavg / $above15 if $above15; |
| 1433 |
+$hamavg = $hamavg / $hamcount if $hamcount; |
| 1434 |
+ |
| 1435 |
+# RBL etc percent of total SMTP sessions |
| 1436 |
+ |
| 1437 |
+my $rblpercent = ( ( $RBLcount / $totalexamined ) * 100 ) if $totalexamined; |
| 1438 |
+my $PatternFilterpercent = ( ( $PatternFilterCount / $totalexamined ) * 100 ) if $totalexamined; |
| 1439 |
+my $Miscpercent = ( ( $MiscDenyCount / $totalexamined ) * 100 ) if $totalexamined; |
| 1440 |
+ |
| 1441 |
+#Spam and virus percent of total email downloaded |
| 1442 |
+#Expressed as a % of total examined |
| 1443 |
+my $spampercent = ( ( $spamcount / $totalexamined ) * 100 ) if $totalexamined; |
| 1444 |
+my $hampercent = ( ( $hamcount / $totalexamined ) * 100 ) if $totalexamined; |
| 1445 |
+my $hrsinperiod = ( ( $end - $start ) / 3600 ); |
| 1446 |
+my $emailperhour = ( $totalexamined / $hrsinperiod ) if $totalexamined; |
| 1447 |
+my $above15percent = ( $above15 / $totalexamined * 100 ) if $totalexamined; |
| 1448 |
+my $infectedpercent = ( ( $infectedcount / ($totalexamined) ) * 100 ) if $totalexamined; |
| 1449 |
+my $AcceptPercent = ( ( $Accepttotal / ($totalexamined) ) * 100 ) if $totalexamined; |
| 1450 |
+ |
| 1451 |
+my $oldfh; |
| 1452 |
+ |
| 1453 |
+#Open Sendmail if we are mailing it |
| 1454 |
+if ( $opt{'mail'} && !$disabled ) { |
| 1455 |
+ open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" ) |
| 1456 |
+ or die "Can't open sendmail: $!\n"; |
| 1457 |
+ print SENDMAIL "From: $opt{'from'}\n"; |
| 1458 |
+ print SENDMAIL "To: $opt{'mail'}\n"; |
| 1459 |
+ print SENDMAIL "Subject: Spam Filter Statistics from $hostname - ", |
| 1460 |
+ strftime( "%F", localtime($start) ), "\n\n"; |
| 1461 |
+ $oldfh = select SENDMAIL; |
| 1462 |
+} |
| 1463 |
+ |
| 1464 |
+my $telapsed = time - $tstart; |
| 1465 |
+ |
| 1466 |
+if ( !$disabled ) { |
| 1467 |
+ |
| 1468 |
+ #Output results |
| 1469 |
+ print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n"; |
| 1470 |
+ print "----------------------------------------------------", "\n\n"; |
| 1471 |
+ |
| 1472 |
+ print "$0 Version : $opt{'version'}", "\n\n"; |
| 1473 |
+ print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n"; |
| 1474 |
+ print "Period Ending : ", strftime( "%c", localtime($end) ), "\n"; |
| 1475 |
+ print "\n"; |
| 1476 |
+ |
| 1477 |
+ print "Clam Version : ", `freshclam -V`; |
| 1478 |
+ print "SpamAssassin Version : ", `spamassassin -V`; |
| 1479 |
+ printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel, |
| 1480 |
+ $SARejectLevel; |
| 1481 |
+ if ($HighLogLevel) { |
| 1482 |
+ printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n"; |
| 1483 |
+ printf "\tYou can set it this way:\n"; |
| 1484 |
+ printf "\tconfig setprop qpsmtpd LogLevel 6\n"; |
| 1485 |
+ printf "\tsignal-event email-update\n"; |
| 1486 |
+ printf "\tsv t /var/service/qpsmtpd\n\n"; |
| 1487 |
+ } |
| 1488 |
+ print "\n"; |
| 1489 |
+ printf "Reporting Period : %.2f hrs\n", $hrsinperiod; |
| 1490 |
+ print "----------------------------\n"; |
| 1491 |
+ print "\n"; |
| 1492 |
+ |
| 1493 |
+ printf "All SMTP connections accepted:%8d \n", $totalexamined; |
| 1494 |
+# |
| 1495 |
+# if ($localAccepttotal>0) { |
| 1496 |
+# printf "Connections from Fetchmail : %8d \n", |
| 1497 |
+# $localAccepttotal; |
| 1498 |
+# } |
| 1499 |
+# |
| 1500 |
+# if ($WebMailsendtotal>0) { |
| 1501 |
+# printf "Emails sent from WebMail : %8d \n", |
| 1502 |
+# $WebMailsendtotal; |
| 1503 |
+# } |
| 1504 |
+# |
| 1505 |
+# if ($mailmansendcount > 0) { |
| 1506 |
+# printf "Emails sent from Mailman : %8d \n", |
| 1507 |
+# $mailmansendcount; |
| 1508 |
+# } |
| 1509 |
+# |
| 1510 |
+# printf "SMTP from local workstations : %8d \n\n", $localsendtotal; |
| 1511 |
+# |
| 1512 |
+# |
| 1513 |
+# printf "RBL rejected : %8d (%6.2f%%)\n", $RBLcount, |
| 1514 |
+# $rblpercent || 0; |
| 1515 |
+# printf "Pattern filter rejected : %8d (%6.2f%%)\n", |
| 1516 |
+# $PatternFilterCount, $PatternFilterpercent || 0; |
| 1517 |
+# printf "Rejected due to non conformance : %8d (%6.2f%%)\n", $MiscDenyCount, |
| 1518 |
+# $Miscpercent || 0; |
| 1519 |
+# |
| 1520 |
+# printf "Infected by Virus : %8d (%6.2f%%)\n", $infectedcount, |
| 1521 |
+# $infectedpercent || 0; |
| 1522 |
+# |
| 1523 |
+# printf "Spam rejected (over reject level): %8d (%6.2f%%)\n", $above15, |
| 1524 |
+# $above15percent || 0; |
| 1525 |
+# printf "Spam detected (over tag level) : %8d (%6.2f%%)\n", $spamcount, |
| 1526 |
+# $spampercent || 0; |
| 1527 |
+# printf "Ham detected (under tag level) : %8d (%6.2f%%)\n", $hamcount, |
| 1528 |
+# $hampercent || 0; |
| 1529 |
+# print " --------------------\n"; |
| 1530 |
+# printf "Total emails accepted : %8d (%6.2f%%)\n", $Accepttotal, |
| 1531 |
+# $AcceptPercent || 0; |
| 1532 |
+ |
| 1533 |
+ |
| 1534 |
+ printf "Emails per hour : %8.1f/hr\n", $emailperhour || 0; |
| 1535 |
+ print "\n"; |
| 1536 |
+ printf "Average spam score (accepted): %11.2f\n", $spamavg || 0; |
| 1537 |
+ printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0; |
| 1538 |
+ printf "Average ham score : %11.2f\n", $hamavg || 0; |
| 1539 |
+ print "\n"; |
| 1540 |
+ print "Statistics by Hour\n"; |
| 1541 |
+ |
| 1542 |
+ # |
| 1543 |
+ # start by working out which colunns to show - tag the display array |
| 1544 |
+ # |
| 1545 |
+ $ncateg = 1; ##skip the first column |
| 1546 |
+ $finaldisplay[0] = $true; |
| 1547 |
+ while ( $ncateg < $categlen) { |
| 1548 |
+ if ($display[$ncateg] eq 'yes') { $finaldisplay[$ncateg] = $true } |
| 1549 |
+ elsif ($display[$ncateg] eq 'no') { $finaldisplay[$ncateg] = $false } |
| 1550 |
+ else { |
| 1551 |
+ $finaldisplay[$ncateg] = ($counts{$GRANDTOTAL}{$categs[$ncateg]} != 0); |
| 1552 |
+ if ($finaldisplay[$ncateg]) { |
| 1553 |
+ #if it has been non zero and auto, then make it yes for the future. |
| 1554 |
+ esmith::ConfigDB->open->get('mailstats')->set_prop($categs[$ncateg],'yes') |
| 1555 |
+ } |
| 1556 |
+ |
| 1557 |
+ } |
| 1558 |
+ $ncateg++ |
| 1559 |
+ } |
| 1560 |
+ #make sure total and percentages are shown |
| 1561 |
+ $finaldisplay[@categs-2] = $true; |
| 1562 |
+ $finaldisplay[@categs-1] = $true; |
| 1563 |
+ |
| 1564 |
+ |
| 1565 |
+ # and put together the print lines |
| 1566 |
+ # |
| 1567 |
+ my $Line1; #Full Line across the page |
| 1568 |
+ my $Line2; #Broken Line across the page |
| 1569 |
+ my $Titles; #Column headers |
| 1570 |
+ my $Values; #Values |
| 1571 |
+ my $Totals; #Corresponding totals |
| 1572 |
+ my $Percent; # and column percentages |
| 1573 |
+ |
| 1574 |
+ my $hour = floor( $start / 3600 ); |
| 1575 |
+ $Line1 = ''; |
| 1576 |
+ $Line2 = ''; |
| 1577 |
+ $Titles = ''; |
| 1578 |
+ $Values = ''; |
| 1579 |
+ $Totals = ''; |
| 1580 |
+ $Percent = ''; |
| 1581 |
+ while ( $hour < $end / 3600 ) { |
| 1582 |
+ if ($hour == floor( $start / 3600 )){ |
| 1583 |
+ #Do all the once only things |
| 1584 |
+ $ncateg = 0; |
| 1585 |
+ while ( $ncateg < @categs) { |
| 1586 |
+ if ($finaldisplay[$ncateg]){ |
| 1587 |
+ $Line1 .= substr('---------------------',0,$colwidth[$ncateg]); |
| 1588 |
+ $Line2 .= substr('---------------------',0,$colwidth[$ncateg]-1); |
| 1589 |
+ $Line2 .= " "; |
| 1590 |
+ $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])." "; |
| 1591 |
+ if ($ncateg == 0) { |
| 1592 |
+ $Totals .= substr('TOTALS ',0,$colwidth[$ncateg]-2); |
| 1593 |
+ $Percent .= substr('PERCENTAGES ',0,$colwidth[$ncateg]-1); |
| 1594 |
+ } else { |
| 1595 |
+ # identify bottom right group and supress unless db->ShowGranPerc set |
| 1596 |
+ if ($ncateg==@categs-1){ |
| 1597 |
+ $Totals .= sprintf('%'.$colwidth[$ncateg].'.1f',$counts{$GRANDTOTAL}{$categs[$ncateg]}).'%'; |
| 1598 |
+ } else { |
| 1599 |
+ $Totals .= sprintf('%'.$colwidth[$ncateg].'d',$counts{$GRANDTOTAL}{$categs[$ncateg]}); |
| 1600 |
+ } |
| 1601 |
+ $Percent .= sprintf('%'.($colwidth[$ncateg]-1).'.1f',$counts{$PERCENT}{$categs[$ncateg]}).'%'; |
| 1602 |
+ } |
| 1603 |
+ } |
| 1604 |
+ $ncateg++ |
| 1605 |
+ } |
| 1606 |
+ } |
| 1607 |
+ |
| 1608 |
+ $ncateg = 0; |
| 1609 |
+ while ( $ncateg < @categs) { |
| 1610 |
+ if ($finaldisplay[$ncateg]){ |
| 1611 |
+ if ($ncateg == 0) { |
| 1612 |
+ $Values .= strftime( "%F, %H", localtime( $hour * 3600 ) )." " |
| 1613 |
+ } elsif ($ncateg == @categs-1) { |
| 1614 |
+ #percentages in last column |
| 1615 |
+ $Values .= sprintf('%'.($colwidth[$ncateg]-2).'.1f',$counts{$hour}{$categs[$ncateg]})."%"; |
| 1616 |
+ } else { |
| 1617 |
+ #body numbers |
| 1618 |
+ $Values .= sprintf('%'.($colwidth[$ncateg]-1).'d',$counts{$hour}{$categs[$ncateg]})." "; |
| 1619 |
+ } |
| 1620 |
+ if (($ncateg == @categs-1)){$Values=$Values."\n"} #&& ($hour == floor($end / 3600)-1) |
| 1621 |
+ } |
| 1622 |
+ $ncateg++ |
| 1623 |
+ } |
| 1624 |
+ |
| 1625 |
+ $hour++; |
| 1626 |
+ } |
| 1627 |
+ |
| 1628 |
+ # print it. |
| 1629 |
+ print $Line1."\n"; |
| 1630 |
+ print $Titles."\n"; |
| 1631 |
+ print $Line2."\n"; |
| 1632 |
+ print $Values."\n"; |
| 1633 |
+ print $Line2."\n"; |
| 1634 |
+ print $Totals."\n"; |
| 1635 |
+ print $Percent."\n"; |
| 1636 |
+ print $Line1."\n"; |
| 1637 |
+ |
| 1638 |
+ |
| 1639 |
+ if ($localAccepttotal>0) { |
| 1640 |
+ print "*Fetchml* means connections from Fetchmail delivering email\n"; |
| 1641 |
+ } |
| 1642 |
+ print "*Local* means connections from workstations on local LAN.\n"; |
| 1643 |
+ print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol.\n"; |
| 1644 |
+ print " or email was to non existant address.\n"; |
| 1645 |
+ print "\n"; |
| 1646 |
+ |
| 1647 |
+ if ($QueryNoLogTerse) { |
| 1648 |
+ print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; |
| 1649 |
+# print " to enable it follow the instructions at .............................\n"; |
| 1650 |
+ } |
| 1651 |
+ |
| 1652 |
+ |
| 1653 |
+ if ( !$RHSenabled || !$DNSenabled ) { |
| 1654 |
+ |
| 1655 |
+ # comment about RBL not set |
| 1656 |
+ print |
| 1657 |
+"* - This means that one or more of the possible spam black listing services\n that are available have not been enabled.\n"; |
| 1658 |
+ print " You have not enabled:\n"; |
| 1659 |
+ |
| 1660 |
+ if ( !$RHSenabled ) { |
| 1661 |
+ print " RHSBL\n"; |
| 1662 |
+ } |
| 1663 |
+ |
| 1664 |
+ if ( !$DNSenabled ) { |
| 1665 |
+ print " DNSBL\n"; |
| 1666 |
+ } |
| 1667 |
+ |
| 1668 |
+ |
| 1669 |
+ print " To enable these you can use the following commands:\n"; |
| 1670 |
+ if ( !$RHSenabled ) { |
| 1671 |
+ print " config setprop qpsmtpd RHSBL enabled\n"; |
| 1672 |
+ } |
| 1673 |
+ |
| 1674 |
+ if ( !$DNSenabled ) { |
| 1675 |
+ print " config setprop qpsmtpd DNSBL enabled\n"; |
| 1676 |
+ } |
| 1677 |
+ |
| 1678 |
+ # there so much templates to expand... (PS) |
| 1679 |
+ print " Followed by:\n signal-event email-update and\n sv t /var/service/qpsmtpd\n\n"; |
| 1680 |
+ } |
| 1681 |
+ |
| 1682 |
+# if ($Webmailsendtotal > 0) {print "If you have the mailman contrib installed, then the webmail totals might include some mailman emails\n"} |
| 1683 |
+ |
| 1684 |
+ # time to do a 'by recipient domain' report |
| 1685 |
+ print "\nIncoming mails by recipient domains usage\n"; |
| 1686 |
+ print "-----------------------------------------\n"; |
| 1687 |
+ print |
| 1688 |
+ "Domains Type Total Denied XferErr Accept \%accept\n"; |
| 1689 |
+ print |
| 1690 |
+ "---------------------------- ---------- ------ ------ ------- ------ -------\n"; |
| 1691 |
+ my %total = ( |
| 1692 |
+ total => 0, |
| 1693 |
+ deny => 0, |
| 1694 |
+ xfer => 0, |
| 1695 |
+ accept => 0, |
| 1696 |
+ ); |
| 1697 |
+ foreach my $domain ( |
| 1698 |
+ sort { |
| 1699 |
+ join( "\.", reverse( split /\./, $a ) ) cmp |
| 1700 |
+ join( "\.", reverse( split /\./, $b ) ) |
| 1701 |
+ } keys %byrcptdomain |
| 1702 |
+ ) |
| 1703 |
+ { |
| 1704 |
+ next if ( ( $byrcptdomain{$domain}{'total'} || 0 ) == 0 ); |
| 1705 |
+ my $tp = $byrcptdomain{$domain}{'type'} || 'other'; |
| 1706 |
+ my $to = $byrcptdomain{$domain}{'total'} || 0; |
| 1707 |
+ my $de = $byrcptdomain{$domain}{'deny'} || 0; |
| 1708 |
+ my $xr = $byrcptdomain{$domain}{'xfer'} || 0; |
| 1709 |
+ my $ac = $byrcptdomain{$domain}{'accept'} || 0; |
| 1710 |
+ printf "%-28s %-10s %6d %6d %7d %6d %6.2f%%\n", $domain, $tp, $to, |
| 1711 |
+ $de, $xr, $ac, $ac * 100 / $to; |
| 1712 |
+ $total{'total'} += $to; |
| 1713 |
+ $total{'deny'} += $de; |
| 1714 |
+ $total{'xfer'} += $xr; |
| 1715 |
+ $total{'accept'} += $ac; |
| 1716 |
+ } |
| 1717 |
+ print |
| 1718 |
+ "---------------------------- ---------- ------ ------- ------ ------ -------\n"; |
| 1719 |
+ |
| 1720 |
+ # $total{ 'total' } can be equal to 0, bad for divisions... |
| 1721 |
+ my $perc1 = 0; |
| 1722 |
+ my $perc2 = 0; |
| 1723 |
+ |
| 1724 |
+ |
| 1725 |
+ if ( $total{'total'} != 0 ) { |
| 1726 |
+ $perc1 = $total{'accept'} * 100 / $total{'total'}; |
| 1727 |
+ $perc2 = ( ( $total{'total'} + $morethanonercpt ) / $total{'total'} ); |
| 1728 |
+ } |
| 1729 |
+ printf |
| 1730 |
+ "Total %6d %6d %7d %6d %6.2f%%\n\n", |
| 1731 |
+ $total{'total'}, $total{'deny'}, $total{'xfer'}, $total{'accept'}, |
| 1732 |
+ $perc1; |
| 1733 |
+ printf |
| 1734 |
+ "%d mails were processed for %d Recipients\nThe average recipients by mail is %4.2f\n\n", |
| 1735 |
+ $total{'total'}, ( $total{'total'} + $morethanonercpt ), $perc2; |
| 1736 |
+ |
| 1737 |
+ if ( $infectedcount > 0 ) { |
| 1738 |
+ show_virus_variants(); |
| 1739 |
+ } |
| 1740 |
+ |
| 1741 |
+ # get enable/disable subsections |
| 1742 |
+ my $enableqpsmtpdcodes; |
| 1743 |
+ my $enableSARules; |
| 1744 |
+ my $enablejunkMailList; |
| 1745 |
+ if (esmith::ConfigDB->open_ro->get('mailstats')){ |
| 1746 |
+ $enableqpsmtpdcodes = esmith::ConfigDB->open_ro->get('mailstats')->prop("QpsmtpdCodes") || "enabled" eq "enabled" || $true; |
| 1747 |
+ $enableSARules = esmith::ConfigDB->open_ro->get('mailstats')->prop("SARules") || "enabled" eq "enabled" || $true; |
| 1748 |
+ $enablejunkMailList = esmith::ConfigDB->open_ro->get('mailstats')->prop("JunkMailList") || "enabled" eq "enabled" || $true; |
| 1749 |
+ } else { |
| 1750 |
+ $enableqpsmtpdcodes = $true; |
| 1751 |
+ $enableSARules = $true; |
| 1752 |
+ $enablejunkMailList = $true; |
| 1753 |
+ } |
| 1754 |
+ |
| 1755 |
+ if ($enableqpsmtpdcodes) {show_qpsmtpd_codes();} |
| 1756 |
+ |
| 1757 |
+ if ($enableSARules) {show_SARules_codes();} |
| 1758 |
+ |
| 1759 |
+ if ($enablejunkMailList) {List_Junkmail();} |
| 1760 |
+ |
| 1761 |
+ print "\nDone. Report generated in $telapsed sec.\n\n"; |
| 1762 |
+ |
| 1763 |
+ #Close Senmdmail if it was opened |
| 1764 |
+ if ( $opt{'mail'} ) { |
| 1765 |
+ select $oldfh; |
| 1766 |
+ close(SENDMAIL); |
| 1767 |
+ } |
| 1768 |
+ |
| 1769 |
+} ##report disabled |
| 1770 |
+ |
| 1771 |
+#All done |
| 1772 |
+exit 0; |
| 1773 |
+ |
| 1774 |
+############################################################################# |
| 1775 |
+# Subroutines ############################################################### |
| 1776 |
+############################################################################# |
| 1777 |
+ |
| 1778 |
+ |
| 1779 |
+######################################## |
| 1780 |
+# Process parms # |
| 1781 |
+######################################## |
| 1782 |
+sub parse_arg { |
| 1783 |
+ my $startdate = shift; |
| 1784 |
+ my $enddate = shift; |
| 1785 |
+ |
| 1786 |
+ my $secsinday = 86400; |
| 1787 |
+ my $time = 0; |
| 1788 |
+ |
| 1789 |
+ my $start = UnixDate( $startdate, "%s" ); |
| 1790 |
+ my $end = UnixDate( $enddate, "%s" ); |
| 1791 |
+ |
| 1792 |
+ if ( !$start && !$end ) { |
| 1793 |
+ $end = time; |
| 1794 |
+ $start = $end - $secsinday; |
| 1795 |
+ return ( $start, $end ); |
| 1796 |
+ } |
| 1797 |
+ |
| 1798 |
+ if ( !$start ) { |
| 1799 |
+ $start = $end - $secsinday; |
| 1800 |
+ return ( $start, $end ); |
| 1801 |
+ } |
| 1802 |
+ |
| 1803 |
+ if ( !$end ) { |
| 1804 |
+ $end = $start + $secsinday; |
| 1805 |
+ return ( $start, $end ); |
| 1806 |
+ } |
| 1807 |
+ |
| 1808 |
+ if ( $start > $end ) { |
| 1809 |
+ return ( $end, $start ); |
| 1810 |
+ } |
| 1811 |
+ |
| 1812 |
+ return ( $start, $end ); |
| 1813 |
+ |
| 1814 |
+} |
| 1815 |
+ |
| 1816 |
+sub dbg { |
| 1817 |
+ my $msg = shift; |
| 1818 |
+ |
| 1819 |
+ if ( $opt{debug} ) { |
| 1820 |
+ print STDERR $msg; |
| 1821 |
+ } |
| 1822 |
+} |
| 1823 |
+ |
| 1824 |
+sub List_Junkmail { |
| 1825 |
+ |
| 1826 |
+ # |
| 1827 |
+ # Show how many junkmails in each user's junkmail folder. |
| 1828 |
+ # |
| 1829 |
+ use esmith::AccountsDB; |
| 1830 |
+ my $adb = esmith::AccountsDB->open_ro; |
| 1831 |
+ my $entry; |
| 1832 |
+ foreach my $user ($adb->users) { |
| 1833 |
+ my $found = 0; |
| 1834 |
+ my $junkmail_dir = "/home/e-smith/files/users/" . |
| 1835 |
+ $user->key . "/Maildir/.junkmail"; |
| 1836 |
+# print $user->key; |
| 1837 |
+ foreach my $dir (qw(new cur)) { |
| 1838 |
+ # Now get the content list for the directory. |
| 1839 |
+ if (opendir( QDIR, "$junkmail_dir/$dir" )) { |
| 1840 |
+ while ($entry=readdir(QDIR) ) { |
| 1841 |
+ next if $entry =~ /^\./; |
| 1842 |
+ $found++; |
| 1843 |
+ } |
| 1844 |
+ |
| 1845 |
+ closedir(QDIR); |
| 1846 |
+ } |
| 1847 |
+ } |
| 1848 |
+ if ( !$disabled ) { |
| 1849 |
+ printf "User \"%s\" ", $user->key; |
| 1850 |
+ printf "- %d email(s) left in junkmail folder\n", $found; |
| 1851 |
+ } |
| 1852 |
+ } |
| 1853 |
+} |
| 1854 |
+ |
| 1855 |
+sub show_virus_variants |
| 1856 |
+ |
| 1857 |
+# |
| 1858 |
+# Show a league table of the different virus types found today |
| 1859 |
+# |
| 1860 |
+ |
| 1861 |
+{ |
| 1862 |
+ |
| 1863 |
+ print("Virus Statistics by name:\n"); |
| 1864 |
+ print("---------------------------------------------\n"); |
| 1865 |
+ foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} } |
| 1866 |
+ keys %found_viruses) |
| 1867 |
+ { |
| 1868 |
+ print "Rejected $found_viruses{$virus}\t$virus\n"; |
| 1869 |
+ } |
| 1870 |
+ print("---------------------------------------------\n\n"); |
| 1871 |
+} |
| 1872 |
+ |
| 1873 |
+sub show_qpsmtpd_codes |
| 1874 |
+ |
| 1875 |
+# |
| 1876 |
+# Show a league table of the qpsmtpd result codes found today |
| 1877 |
+# |
| 1878 |
+ |
| 1879 |
+{ |
| 1880 |
+ |
| 1881 |
+ print("Qpsmtpd codes league table:\n"); |
| 1882 |
+ print("---------------------------------------------\n"); |
| 1883 |
+ print("Count\tPercent\tReason\t\n"); |
| 1884 |
+ print("---------------------------------------------\n"); |
| 1885 |
+ foreach my $qpcode (sort { $found_qpcodes{$b} <=> $found_qpcodes{$a} } |
| 1886 |
+ keys %found_qpcodes) |
| 1887 |
+ { |
| 1888 |
+ print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined; |
| 1889 |
+ } |
| 1890 |
+ print("---------------------------------------------\n\n"); |
| 1891 |
+} |
| 1892 |
+ |
| 1893 |
+sub show_SARules_codes |
| 1894 |
+ |
| 1895 |
+# |
| 1896 |
+# Show a league table of the SARules result codes found today |
| 1897 |
+# suppress any lower than DB mailstats/SARulePercentThreshold |
| 1898 |
+# |
| 1899 |
+ |
| 1900 |
+{ |
| 1901 |
+ |
| 1902 |
+ my ($percentthreshold); |
| 1903 |
+ my ($defaultpercentthreshold); |
| 1904 |
+ |
| 1905 |
+ if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { |
| 1906 |
+ $defaultpercentthreshold = $maxcutoff |
| 1907 |
+ } else { |
| 1908 |
+ $defaultpercentthreshold = $mincutoff |
| 1909 |
+ } |
| 1910 |
+ if (esmith::ConfigDB->open_ro->get('mailstats')){ |
| 1911 |
+ $percentthreshold = esmith::ConfigDB->open_ro->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold; |
| 1912 |
+ } else { |
| 1913 |
+ $percentthreshold = $defaultpercentthreshold |
| 1914 |
+ } |
| 1915 |
+ print("Spamassassin Rules:\n"); |
| 1916 |
+ print("---------------------------------------------\n"); |
| 1917 |
+ print("Count\tPercent\tRule\t\n"); |
| 1918 |
+ print("---------------------------------------------\n"); |
| 1919 |
+ foreach my $SARule (sort { $found_SARules{$b} <=> $found_SARules{$a} } |
| 1920 |
+ keys %found_SARules) |
| 1921 |
+ { |
| 1922 |
+ my $percent = $found_SARules{$SARule}*100/$totalexamined if $totalexamined; |
| 1923 |
+ if ($percent > $percentthreshold) { |
| 1924 |
+ print "$found_SARules{$SARule}\t".sprintf('%4.1f',$percent)."%\t$SARule\n" if $totalexamined; |
| 1925 |
+ } |
| 1926 |
+ } |
| 1927 |
+ print("---------------------------------------------\n\n"); |
| 1928 |
+} |
| 1929 |
+ |
| 1930 |
+ |
| 1931 |
+sub mark_domain_rejected |
| 1932 |
+ |
| 1933 |
+# |
| 1934 |
+# Tag domain as having a rejected email |
| 1935 |
+# |
| 1936 |
+{ |
| 1937 |
+my ($proc) = @_; |
| 1938 |
+if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { |
| 1939 |
+ $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'deny' }++ ; |
| 1940 |
+ $currentrcptdomain{ $proc } = '' ; |
| 1941 |
+ } |
| 1942 |
+} |
| 1943 |
\ No newline at end of file |
Parent Directory
| 
Revision Log
| 
Revision Graph
