diff -urN smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl --- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl 2016-10-16 14:40:37.689999942 +0100 +++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl 2016-10-12 14:26:59.000000000 +0100 @@ -20,6 +20,10 @@ # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection # bjr - 14Jun16 - make compatible with qpsmtpd 0.96 # bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7) +# bjr - 04Aug16 - Add code to log and count the blacklist RBL urls that have triggered, this (NFR) is Bugzilla 9717 +# bjr - 04Aug16 - Add code to expand the junkmail table to include daily ham and spam and deleted spam for each user - (NFR bugzilla 9716) +# bjr - 05Aug16 - Add code to log remote relay incoming emails +# bjr - 10Oct16 - Add code to show stats for the smeoptimizer package # ############################################################################# # @@ -45,6 +49,15 @@ # / HTMLPage - "yes" / "no" - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no" # ############################################################################# +# +# +# TODO +# +# 1. Delete loglines records from any previous run of same table +# 2. Add tracking LogId for each cont in the table +# 3. Use link directory file to generate h1 / h2 tags for title and section headings +# 4. Ditto for links to underlying data +# # internal modules (part of core perl distribution) use strict; @@ -60,6 +73,7 @@ use Sys::Hostname; use Switch; use DBIx::Simple; +use URI::URL; #use CGI; #use HTML::TextToHTML; @@ -81,7 +95,7 @@ #Configuration section my %opt = ( - version => '0.7.5a', # please update at each change. + version => '0.7.10', # please update at each change. debug => 0, # guess what ? sendmail => '/usr/sbin/sendmail', # Path to sendmail stub from => 'spamfilter-stats', # Who is the mail from @@ -96,7 +110,7 @@ my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost my $DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything) my $DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval - +my $localIPregexp = ".*((127\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(192\.168\.)).*"; my $MinCol = 6; #Minimum column width my $HourColWidth = 16; #Date and time column width @@ -135,10 +149,12 @@ my %found_SARules = (); my %junkcount = (); my %unrecog_plugin = (); +my %blacklistURL = (); #Count of use of each balcklist rhsbl +my %usercounts = (); #Count per received email of sucessful delivery, queued spam and deleted Spam, and rejected # replaced by... -my %counts = (); #Hold all counts in 2-D matrix -my @display = (); #used to switch on and off columns - yes, no or auto for each category +my %counts = (); #Hold all counts in 2-D matrix +my @display = (); #used to switch on and off columns - yes, no or auto for each category my @colwidth = (); #width of each column #(auto means only if non zero) - populated from possible db entries my @finaldisplay = (); #final decision on display or not - true or false @@ -149,8 +165,9 @@ my $CATWEBMAIL='WebMail'; my $CATMAILMAN='Mailman'; my $CATLOCAL='Local'; +my $CATRELAY="Relay"; # border between where it came from and where it ended.. -my $countfromhere = 5; #Temp - Check this not moved!! +my $countfromhere = 6; #Temp - Check this not moved!! my $CATVIRUS='Virus'; my $CATRBLDNS='RBL/DNS'; @@ -166,7 +183,7 @@ my $CATPERCENT='PERCENT'; my $CATDMARC="DMARC Rej."; my $CATLOAD="Rej.Load"; -my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT); +my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATRELAY,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT); my $GRANDTOTAL = '99'; #subs for count arrays, for grand total my $PERCENT = '98'; # for column percentages @@ -238,10 +255,21 @@ #dbg("DMARC-EMAILS:".$DMARC_Report_emails); +# and setup list of local domains for spotting the local one in a list of email addresses (Remote station processing) +use esmith::DomainsDB; +my $d = esmith::DomainsDB->open_ro(); +my @domains = $d->keys(); +my $alldomains = "("; +foreach my $dom (@domains){$alldomains .= $dom."|"} +$alldomains .= ")"; +#print $alldomains; + # Saving the Log lines processed my %LogLines = (); #Save all the log lines processed for writing to the DB my %LogId = (); #Save the Log Ids. my $CurrentLogId = ""; +my $Sequence = 0; + # store the domain of interest. Every other records are stored in a 'Other' zone my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n"; @@ -306,6 +334,7 @@ my $enableGeoiptable; my $enablejunkMailList; my $savedata; +my $enableblacklist; #Enabled according to setting in qpsmtpd if ($cdb->get('mailstats')){ $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false; $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false; @@ -319,12 +348,15 @@ $enableGeoiptable = $true; $savedata = $false; } - $savedata = $false; #TEMP!! - -# -#--------------------------------------- -# Scan the qpsmtpd log file(s) -#--------------------------------------- + $enableblacklist = ($cdb->get('qpsmtpd')->prop("RHSBL") || "disabled") eq "enabled" || ($cdb->get('qpsmtpd')->prop("URIBL") || "disabled") eq "enabled"; + #$savedata = $false; #TEMP!! +#if ($savedata){print STDERR "yes"} else {print STDERR "no"} + +my $makeHTMLemail = "no"; +#if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!! +my $makeHTMLpage = "no"; +#if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"} +#if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"} # Init the hashes @@ -373,6 +405,12 @@ my $count = -1; #for loop reduction in debugging mode +# +#--------------------------------------- +# Scan the qpsmtpd log file(s) +#--------------------------------------- + + my $CurrentMailId = ""; LINE: while (<>) { @@ -391,36 +429,38 @@ #Count lines and skip out if debugging $count++; - last LINE if ($opt{debug} && $count >= 100000); + #last LINE if ($opt{debug} && $count >= 100); #dbg("REST:".$log); + #Loglines to Saved String for later DB write if ($savedata) { my $CurrentLine = $_; $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/; - if ($2 ne $CurrentMailId) { - $CurrentLogId = $1."-".$2; - $CurrentMailId = $2; + my $l = length($CurrentLine); + if ($l != 0){ + if (defined($2)){ #print STDERR "Undefined \$2:".$_.":".$count.":".$l;exit} + if ($2 ne $CurrentMailId) { + print "CL:$CurrentLine*\n" if !defined($1); + $CurrentLogId = $1."-".$2; + $CurrentMailId = $2; + $Sequence = 0; + } else {$Sequence++} + #$CurrentLogId .=":".$Sequence; + $LogLines{$CurrentLogId.":".$Sequence} = $_; + } } - $LogLines{$CurrentLogId} = $_; - #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n"; + #print STDERR $CurrentLogId.":".$LogLines{$CurrentLogId}."\n"; + #exit } - #Count lines and skip out if debugging - $count++; - last LINE if ($opt{debug} && $count >= 100); - #dbg("REST:".$log); - # pull out spamasassin rule lists if ( $_ =~m/spamassassin: pass, Ham,(.*) ) #if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ ) { - dbg("SPAM:".$log); - - + #dbg("SPAM:".$log); #New version does not seem to have spammassasin tests in logs - #if (exists($2){ #my (@SAtests) = split(',',$2); #foreach my $SAtest (@SAtests) { @@ -451,11 +491,6 @@ #only select Logterse output next LINE unless m/logging::logterse:/; - - #Count lines and skip out if debugging - $count++; - last LINE if ($opt{debug} && $count >= 100000); - #dbg("REST:".$log); my $abstime = Time::TAI64::tai2unix($tai); my $abshour = floor( $abstime / 3600 ); # Hours since the epoch @@ -465,35 +500,89 @@ my (@log_items) = split $FS, $log_part; my (@timestamp_items) = split(' ',$timestamp_part); - + + my $result= "rejected"; #Tag as rejected unti we know otherwise # we store the more recent recipient domain, for domain statistics # in fact, we only store the first recipient. Could be sort of headhache # to obtain precise stats with many recipients on more than one domain ! my $proc = $timestamp_items[1] ; #numeric Id for the email + my $emailnum = $proc; #proc gets modified later... + if ($emailnum == 23244) { + dbg("TM0:".$timestamp_items[0]); + dbg("TM1:".$timestamp_items[1]); + dbg("TM2:".$timestamp_items[2]); + dbg("TM3:".$timestamp_items[3]); + dbg("LOG0:".$log_items[0]); + dbg("LOG1:".$log_items[1]); + dbg("LOG2:".$log_items[2]); + dbg("LOG3:".$log_items[3]); + dbg("LOG4:".$log_items[4]); + dbg("LOG5:".$log_items[5]); + dbg("LOG6:".$log_items[6]); + dbg("LOG7:".$log_items[7]); + dbg("IPregexp:".$localIPregexp); + if (!test_for_private_ip($log_items[0])) {dbg("Log0 not found");} + if (test_for_private_ip($log_items[2])){ dbg("Log2 match")} + if ($log_items[5] eq "queued") {dbg("LOG5 match")} + } + $totalexamined++; - #dbg("LOG1:".$log_items[1]); - #dbg("LOG3:".$log_items[3]); +# dbg("LOG8:".$log_items[8]); # first spot the fetchmail and local deliveries. # Spot from local workstation $localflag = 0; $WebMailflag = 0; - if ( $log_items[1] =~ m/.*$DomainName.*/ ) { + if ( $log_items[1] =~ m/$DomainName/ ) { #bjr + #dbg("LOG1-Found:".$log_items[1]); $localsendtotal++; $counts{$abshour}{$CATLOCAL}++; $localflag = 1; } + + #Or a remote station + elsif ((!test_for_private_ip($log_items[0])) and (test_for_private_ip($log_items[2])) and ($log_items[5] eq "queued")) + { + #Remote user + $localflag = 1; + $counts{$abshour}{$CATRELAY}++; + } + + elsif (($log_items[2] =~ m/$WebmailIP/) and (!test_for_private_ip($log_items[0]))) { + #Webmail +# if ($emailnum == 19608){ + dbg("WEBMAIL:"); + dbg("TM0:".$timestamp_items[0]); + dbg("TM1:".$timestamp_items[1]); + dbg("TM2:".$timestamp_items[2]); + dbg("TM3:".$timestamp_items[3]); + dbg("LOG0:".$log_items[0]); + dbg("LOG1:".$log_items[1]); + dbg("LOG2:".$log_items[2]); + dbg("LOG3:".$log_items[3]); + dbg("LOG4:".$log_items[4]); + dbg("LOG5:".$log_items[5]); + dbg("LOG6:".$log_items[6]); + dbg("LOG7:".$log_items[7]); + #exit; +# } + $localflag = 1; + $WebMailsendtotal++; + $counts{$abshour}{$CATWEBMAIL}++; + $WebMailflag = 1; + } # see if from localhost - elsif ( $log_items[1] =~ m/.*$localhost.*/ ) { + elsif ( $log_items[1] =~ m/$localhost/ ) { # but not if it comes from fetchmail - if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { } + if ( $log_items[3] =~ m/$FETCHMAIL/ ) { } else { + $localflag = 1; # might still be from mailman here - if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) { + if ( $log_items[3] =~ m/$MAILMAN/ ) { $mailmansendcount++; $localsendtotal++; $counts{$abshour}{$CATMAILMAN}++; @@ -501,12 +590,10 @@ } else { #Or sent to the DMARC server - dbg("LOG4:".$log_items[4]); + #dbg("LOG4:".$log_items[4]); #check for email address in $DMARC_Report_emails string - #if ($log_items[4] =~ m/.*$DMARCDomain.*/) { my $logemail = $log_items[4]; - #print STDERR "/",$log_items[4]."/\n"; - if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){ + if ((index($DMARC_Report_emails,$logemail)>=0) or ($logemail =~ m/$DMARCDomain/)){ $localsendtotal++; $DMARCSendCount++; $localflag = 1; @@ -514,10 +601,11 @@ else { #print STDERR "no match:.".$logemail; if (exists $log_items[8]){ - dbg("LOG8:".$log_items[8]); + #dbg("LOG8:".$log_items[8]); # ignore incoming localhost spoofs - if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { } + if ( $log_items[8] =~ m/msg denied before queued/ ) { } else { + #Webmail $localflag = 1; $WebMailsendtotal++; $counts{$abshour}{$CATWEBMAIL}++; @@ -536,12 +624,12 @@ } # try to spot fetchmail emails - if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) { - dbg("LOG0:".$log_items[0]); + if ( $log_items[0] =~ m/$FetchmailIP/ ) { + #dbg("LOG0:".$log_items[0]); $localAccepttotal++; $counts{$abshour}{$CATFETCHMAIL}++; } - elsif ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { + elsif ( $log_items[3] =~ m/$FETCHMAIL/ ) { $localAccepttotal++; $counts{$abshour}{$CATFETCHMAIL}++; } @@ -549,12 +637,12 @@ # and adjust for recipient field if not set-up by denying plugin - extract from deny msg if ( length( $log_items[4] ) == 0 ) { - dbg("LOG7:".$log_items[0]); + #dbg("LOG7:".$log_items[0]); if ( $log_items[5] eq 'check_goodrcptto' ) { if ( $log_items[7] gt "invalid recipient" ) { $log_items[4] = substr( $log_items[7], 18 ); #Leave only email address - dbg("LOG4:".$log_items[0]); + #dbg("LOG4:".$log_items[0]); } } @@ -563,7 +651,7 @@ # if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) { # reduce to lc and process each e,mail if a list, pseperatedy commas my $recipientmail = lc( $log_items[4] ); - dbg("LOG4:".$log_items[0]); + #dbg("LOG4:".$log_items[0]); if ( $recipientmail =~ m/.*,/ ) { #comma - split the line and deal with each domain @@ -599,146 +687,179 @@ # then categorise the result - if (exists $log_items[5]) { + if (exists $log_items[5]) { - if ($log_items[5] eq 'naughty') { + if ($log_items[5] eq 'naughty') { my $rejreason = $log_items[7]; $rejreason = /.*(\(.*\)).*/; - $rejreason = $1; + if (!defined($1)){$rejreason = "unknown"} + else {$rejreason = $1} $found_qpcodes{$log_items[5]."-".$rejreason}++} else {$found_qpcodes{$log_items[5]}++} ##Count different qpsmtpd result codes - #Check for badly formed lines (from earlier testing) - - if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} - elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} - elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one. + elsif ($log_items[5] eq 'tnef2mime') { } #Not expecting this one. - elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++; - # and extract the spam score -# if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") - if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") + elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++; + # and extract the spam score + # if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") + if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") {$rejectspamavg += $1} - mark_domain_rejected($proc); - next LINE - } + mark_domain_rejected($proc); + } - elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++; - #extract the virus name - if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;} - else {$found_viruses{$log_items[7]}++} #Some other message!! - dbg("LOG7:".$log_items[7]); - mark_domain_rejected($proc); - next LINE - } + elsif (($log_items[5] eq 'virus::clamav') or ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++; + #extract the virus name + if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;} + else {$found_viruses{$log_items[7]}++} #Some other message!! + #dbg("LOG7:".$log_items[7]); + mark_domain_rejected($proc); + } - elsif ($log_items[5] eq 'queued') { $Accepttotal++; - #extract the spam score - if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") { - $score = $1; -# print $log_items[8]."<".$score.">\n"; - if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score} - else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score} - } else { - # no SA score - treat it as ham - $hamcount++;$counts{$abshour}{$CATHAM}++; - } - if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; - $currentrcptdomain{ $proc } = '' ; - } - next LINE - } - + elsif ($log_items[5] eq 'queued') { $Accepttotal++; + #extract the spam score + # Remove count for rejectred as it looks as if it might get through!! + $result= "queued"; + if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") { + $score = trim($1); + if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric + { + if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;} + else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";} + } else { + print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n"; + } + } else { + # no SA score - treat it as ham + $hamcount++;$counts{$abshour}{$CATHAM}++; + } + if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; + $currentrcptdomain{ $proc } = '' ; + } + } - elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'naughty') { + elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} + + elsif ($log_items[5] eq 'naughty') { #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify if ($log_items[7] =~ m/(karma)/) { - $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE} + $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)} elsif ($log_items[7] =~ m/(dnsbl)/){ - $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} elsif ($log_items[7] =~ m/(helo)/){ - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} else { #Unidentified Naughty rejection - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE} + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++} } - elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} #Treat it as Unconf if not recognised - else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE} + else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++} } #Log[5] exists - -# print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected -# next LINE - - - + #Entry if not local send + if ($localflag == 0) { + if (length($log_items[4]) > 0){ + # Need to check here for multiple email addresses + my @emails = split(",",lc($log_items[4])); + if (scalar(@emails) > 1) { + #Just pick the first local address to hang it on. + # TEMP - just go for the first address until I can work out how to spot the 1st "local" one + $usercounts{$emails[0]}{$result}++; + $usercounts{$emails[0]}{"proc"} = $proc; + #Compare with @domains array until we get a local one + my $gotone = $false; + foreach my $email (@emails){ + #Extract the domain from the email address + my $fullemail = $email; + $email = s/.*\@(.*)$/$1/; + #and see if it is local + if ($email =~ m/$alldomains/){ + $usercounts{lc($fullemail)}{$result}++; + $usercounts{lc($fullemail)}{"proc"} = $proc; + $gotone = $true; + last; + } + } + if (!$gotone) { + $usercounts{'No internal email $proc'}{$result}++; + $usercounts{'No internal email $proc'}{"proc"} = $proc; + } + + } else { + $usercounts{lc($log_items[4])}{$result}++; + $usercounts{lc($log_items[4])}{"proc"} = $proc; + } + } + } + #exit if $emailnum == 15858; + } #END OF MAIN LOOP #total up grand total Columns @@ -750,7 +871,7 @@ $counts{$GRANDTOTAL}{$categs[$ncateg]} += $counts{$nhour}{$categs[$ncateg]}; # and total rows - if ( $ncateg < $categlen && $ncateg>=$countfromhere) {#skip initial columns of non final reasons + if ( $ncateg < $categlen and $ncateg>=$countfromhere) {#skip initial columns of non final reasons $counts{$nhour}{$categs[@categs-2]} += $counts{$nhour}{$categs[$ncateg]}; } $ncateg++ @@ -815,7 +936,7 @@ my $oldfh; #Open Sendmail if we are mailing it -if ( $opt{'mail'} && !$disabled ) { +if ( $opt{'mail'} and !$disabled ) { open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" ) or die "Can't open sendmail: $!\n"; print SENDMAIL "From: $opt{'from'}\n"; @@ -841,44 +962,44 @@ my $oldFH = select $outputFH; - print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n"; - print "----------------------------------------------------", "\n\n"; - - print "$0 Version : $opt{'version'}", "\n\n"; - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n"; + print "SMEServer daily Anti-Virus and Spamfilter statistics from $hostname - ".strftime( "%F", localtime($start))."\n"; + print "----------------------------------------------------------------------------------", "\n\n"; + print "$0 Version : $opt{'version'}", "\n"; + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n"; print "Period Ending : ", strftime( "%c", localtime($end) ), "\n"; - print "\n"; - - print "Clam Version/DB Count/Last DB update: ", `freshclam -V`."\n"; - print "SpamAssassin Version : ", `spamassassin -V`."\n"; - printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel; + print "Clam Version/DB Count/Last DB update: ",`freshclam -V`; + print "SpamAssassin Version : ",`spamassassin -V`; + printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,$SARejectLevel; if ($HighLogLevel) { printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n"; printf "\tYou can set it this way:\n"; printf "\tconfig setprop qpsmtpd LogLevel 6\n"; printf "\tsignal-event email-update\n"; - printf "\tsv t /var/service/qpsmtpd\n\n"; + printf "\tsv t /var/service/qpsmtpd\n"; } - print "\n\n"; printf "Reporting Period : %.2f hrs\n", $hrsinperiod; - #print "----------------------------\n"; - #print "\n"; - printf "All SMTP connections accepted:%8d \n", $totalexamined; - printf "Emails per hour : %8.1f/hr\n", $emailperhour || 0; - print "\n"; printf "Average spam score (accepted): %11.2f\n", $spamavg || 0; printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0; printf "Average ham score : %11.2f\n", $hamavg || 0; - printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount || 0; - if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;} + printf "Number of DMARC reporting emails sent:\t%11d (not shown on table)\n", $DMARCSendCount || 0; + if ($hamcount != 0){ printf "Number of emails approved through DMARC:\t%11d (%3d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;} + + my $smeoptimizerprog = "/usr/local/smeoptimizer/SMEOptimizer.pl"; + if (-e $smeoptimizerprog) { + #smeoptimizer installed - get result of status + my @smeoptimizerlines = split(/\n/,`/usr/local/smeoptimizer/SMEOptimizer.pl -status`); + print("SMEOptimizer status:\n"); + print("\t".$smeoptimizerlines[6]."\n"); + print("\t".$smeoptimizerlines[7]."\n"); + print("\t".$smeoptimizerlines[8]."\n"); + print("\t".$smeoptimizerlines[9]."\n"); + print("\t".$smeoptimizerlines[10]."\n"); + } + - print "\n\n"; - print "\nStatistics by Hour\n"; - print "-------------------\n"; - #print "\n"; - + print "\nStatistics by Hour:\n"; # # start by working out which colunns to show - tag the display array # @@ -968,24 +1089,17 @@ # # print it. # - my $makeHTMLemail = "no"; - #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!! - my $makeHTMLpage = "no"; - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"} - #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"} - - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion .... - print $Titles."\n"; - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto - #$Line2 =~ s/-/a/g; - #print $Line2."\n"; - #print "\n"; - print $Values."\n"; + + print $Line1."\n"; + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion .... + print $Titles."\n"; + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto + print $Line2."\n"; + print $Values; print $Line2."\n"; print $Totals."\n"; print $Percent."\n"; print $Line1."\n"; - print "\n"; if ($localAccepttotal>0) { print "*Fetchml* means connections from Fetchmail delivering email\n"; @@ -1021,7 +1135,7 @@ } - if ( !$RHSenabled || !$DNSenabled ) { + if ( !$RHSenabled or !$DNSenabled ) { # comment about RBL not set print @@ -1114,11 +1228,15 @@ if ($enableSARules) {show_SARules_codes();} - if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();} + if ($enableGeoiptable and $finaldisplay[$BadCountryCateg]){show_Geoip_results();} if ($enablejunkMailList) {List_Junkmail();} + + if ($enableblacklist) {show_blacklist_counts();} + + show_user_stats(); - print "\nDone. Report generated in $telapsed sec.\n\n"; + print "\nReport generated in $telapsed sec.\n"; if ($savedata) { save_data(); } else @@ -1128,25 +1246,26 @@ select $oldFH; close $outputFH; - if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr} - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){ + if ($makeHTMLemail eq "no" or $makeHTMLemail eq "both") {print $tablestr} + if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" or $makeHTMLpage eq "yes"){ #Convert text to html and send it require CGI; require TextToHTML; my $cgi = new CGI; my $text = $tablestr; - print $cgi->header(); my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20); my $conv = new HTML::TextToHTML(); $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2, underline_length_tolerance=>1); - my $html=" \n"; + + my $html = $cgi->header(); + $html .=" \n"; $html .= "