diff -Nur smeserver-mailstats-1.1.bz8656.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1.bz8656/root/usr/bin/spamfilter-stats-7.pl --- smeserver-mailstats-1.1.bz8656.old/root/usr/bin/spamfilter-stats-7.pl 2015-06-25 21:00:48.064715149 +0200 +++ smeserver-mailstats-1.1.bz8656/root/usr/bin/spamfilter-stats-7.pl 2015-06-25 21:13:05.272509457 +0200 @@ -1,5 +1,3 @@ - - #!/usr/bin/perl -w ############################################################################# @@ -13,8 +11,11 @@ # bjr - 02sept12 - Add in qpsmtpd failure code auth::auth_cvm_unix_local as per Bug 7089 # bjr - 10Jun15 - Sort out multiple files as input parameters as per bug 5613 # - Sort out geoip failure status as per Bug 4262 -# - change final message about the DB (it is created automatically these days by the rpm) -# +# - change final message about the DB (it is created automatically these days by the rpm) +# bjr - 17Jun15 - Add annotation showing Badcountries being eliminated +# - correct Spamfilter details extract, as per Bug 8656 +# - Add analysis table of Geoip results +# bjr - 19Jun15 - Add totals for the League tables # ############################################################################# # @@ -25,10 +26,13 @@ # / ("yes"|"no"|"auto") - enable, supress or only show if nonzero # / QpsmtpdCodes ("enabled"|"disabled") # / SARules ("enabled"|"disabled") +# / GeoipTable ("enabled"|"disabled") +# / GeoipCutoffPercent (0.5%) - threshold to show Geoip country in league table # / JunkMailList ("enabled"|"disabled") # / SARulePercentThreshold (0.5) - threshold of SArules percentage for report cutoff # / Email (admin) - email to send report # / SaveDataToMySQL - save data to MySQL database (default is "no") +# / ShowLeagueTotals - Show totals row after league tables - (default is "yes") # / DBHost - MySQL server hostname (default is "localhost"). # / DBPort - MySQL server post (default is "3306") # / Interval - "day", "week", "fortnight", "month", "99999" - last is number of seconds (default is day) @@ -67,7 +71,7 @@ #Configuration section my %opt = ( - version => '0.6.27', # please update at each change. + version => '0.6.28', # please update at each change. debug => 0, # guess what ? sendmail => '/usr/sbin/sendmail', # Path to sendmail stub from => 'spamfilter-stats', # Who is the mail from @@ -137,6 +141,7 @@ my $CATEXECUT='Execut.'; my $CATNONCONF='Non.Conf.'; my $CATBADCOUNTRIES='Geoip.'; +my $BadCountryCateg=8; #Careful here this number could change if more added before. my $CATSPAMDEL='Del.Spam'; my $CATSPAM='Qued.Spam?'; my $CATHAM='Ham'; @@ -158,6 +163,10 @@ my $warnnoreject = " "; my $rblnotset = ' '; +my %found_countries = (); +my $total_countries = 0; +my $BadCountries = ""; #From the DB + my $FS = "\t"; # field separator used by logterse plugin my %log_items = ( "", "", "", "", "", "", "", "" ); my $score; @@ -233,7 +242,7 @@ # #--------------------------------------- -# Scan the qpsmtpd log file +# Scan the qpsmtpd log file(s) #--------------------------------------- @@ -277,12 +286,10 @@ my @ARGV2; foreach ( map { glob } @ARGV){ push(@ARGV2,($_)); -#print "\nFound $_"; } @ARGV=@ARGV2; LINE: while (<>) { -#print "\nLine:$_"; my($tai,$log) = split(' ',$_,2); @@ -291,10 +298,10 @@ next LINE if ( $tai gt $endtai ); # pull out spamasassin rule lists - if ( $_ =~m/spamassassin plugin: check_spam:.*hits=(.*), required.*tests=(.*)/ ) + if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ ) { - my ($SAtests) = split(',',$2); - foreach my $SAtest ($SAtests) { + my (@SAtests) = split(',',$2); + foreach my $SAtest (@SAtests) { if (!$SAtest eq "") { $found_SARules{$SAtest}{'count'}++; $found_SARules{$SAtest}{'totalhits'} += $1; @@ -303,10 +310,17 @@ } } + + #Pull out Geoip countries for analysis table + if ( $_ =~m/check_badcountries plugin \(connect\): GeoIP Country: (.*)/ ) + { + $found_countries{$1}++; + $total_countries++; + } + #only select Logterse output next LINE unless m/terse plugin/; -#print "\nFound $_"; my $abstime = Time::TAI64::tai2unix($tai); @@ -738,9 +752,13 @@ print "*Fetchml* means connections from Fetchmail delivering email\n"; } print "*Local* means connections from workstations on local LAN.\n"; - print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol.\n"; - print " or email was to non existant address.\n"; - print "\n"; + print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol"; + print " or email was to non existant address.\n"; + + if ($finaldisplay[$BadCountryCateg]){ + $BadCountries = $cdb->get('qpsmtpd')->prop('BadCountries') || "*none*"; + print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n"; + } if ($QueryNoLogTerse) { print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; @@ -839,17 +857,20 @@ # get enable/disable subsections my $enableqpsmtpdcodes; my $enableSARules; + my $enableGeoiptable; my $enablejunkMailList; my $savedata; if ($cdb->get('mailstats')){ $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false; $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false; $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false; + $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false; $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false; } else { $enableqpsmtpdcodes = $true; $enableSARules = $true; $enablejunkMailList = $true; + $enableGeoiptable = $true; $savedata = $false; } @@ -857,6 +878,8 @@ if ($enableSARules) {show_SARules_codes();} + if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();} + if ($enablejunkMailList) {List_Junkmail();} print "\nDone. Report generated in $telapsed sec.\n\n"; @@ -1020,6 +1043,55 @@ print("---------------------------------------------\n\n"); } +sub show_Geoip_results +# +# Show league table of GEoip results +# +{ + + my ($percentthreshold); + my ($reject); + my ($percent); + my ($totalpercent)=0; + if ($cdb->get('mailstats')){ + $percentthreshold = $cdb->get('mailstats')->prop("GeoipCutoffPercent") || 0.5; + } else { + $percentthreshold = 0.5; + } + print("Geoip results: (cutoff at $percentthreshold%) \n"); + print("---------------------------------\n"); + print("Country\tPercent\tCount\tRejected?\n"); + print("---------------------------------\n"); + foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} } + keys %found_countries) + { + $percent = $found_countries{$country} * 100 / $total_countries + if $total_countries; + $totalpercent = $totalpercent + $percent; + if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";} + if ( $percent >= $percentthreshold ) { + print "$country\t" + . sprintf( '%4.1f', $percent ) + . "%\t$found_countries{$country}","\t$reject\n" + if $total_countries; + } + + } + print("---------------------------------\n"); + my ($showtotals); + if ($cdb->get('mailstats')){ + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); + } else { + $showtotals = $true; + } + + if ($showtotals){ + print "TOTALS\t$totalpercent%\t$total_countries\n"; + print("---------------------------------\n\n"); + } + print "\n"; +} + sub show_SARules_codes # @@ -1028,9 +1100,9 @@ # { - my ($percentthreshold); my ($defaultpercentthreshold); + my ($totalpercent) = 0; if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { $defaultpercentthreshold = $maxcutoff @@ -1042,27 +1114,41 @@ } else { $percentthreshold = $defaultpercentthreshold } - print("Spamassassin Rules:\n"); + + print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n"); print("---------------------------------------------\n"); - print("Count\tPercent\tRule\t\n"); + print("Count\tPercent\tScore\t\t\n"); print("---------------------------------------------\n"); foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} } keys %found_SARules) { - my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined - if $totalexamined; - my $avehits = $found_SARules{$SARule}{'totalhits'} / - $found_SARules{$SARule}{'count'} - if $found_SARules{$SARule}{'count'}; - if ( $percent > $percentthreshold ) { - print "$found_SARules{$SARule}{'count'}\t" - . sprintf( '%4.1f', $percent ) . "%\t" - . sprintf( '%4.1f', $avehits ) - . "\t$SARule\n" - if $totalexamined; - } + my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined + if $totalexamined; + #$totalpercent = $totalpercent + $percent; + my $avehits = $found_SARules{$SARule}{'totalhits'} / + $found_SARules{$SARule}{'count'} + if $found_SARules{$SARule}{'count'}; + if ( $percent >= $percentthreshold ) { + print "$found_SARules{$SARule}{'count'}\t" + . sprintf( '%4.1f', $percent ) . "%\t" + . sprintf( '%4.1f', $avehits ) + . "\t$SARule\n" + if $totalexamined; +} } - print("---------------------------------------------\n\n"); + print("---------------------------------------------\n"); + my ($showtotals); + if ($cdb->get('mailstats')){ + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); + } else { + $showtotals = $true; + } + + if ($showtotals){ + print "$totalexamined\t(TOTALS)\n"; + print("---------------------------------------------\n"); + } + print "\n"; }