diff -urN smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl --- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl 2016-10-16 14:40:37.689999942 +0100 +++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl 2016-10-12 14:26:59.000000000 +0100 @@ -20,6 +20,10 @@ # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection # bjr - 14Jun16 - make compatible with qpsmtpd 0.96 # bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7) +# bjr - 04Aug16 - Add code to log and count the blacklist RBL urls that have triggered, this (NFR) is Bugzilla 9717 +# bjr - 04Aug16 - Add code to expand the junkmail table to include daily ham and spam and deleted spam for each user - (NFR bugzilla 9716) +# bjr - 05Aug16 - Add code to log remote relay incoming emails +# bjr - 10Oct16 - Add code to show stats for the smeoptimizer package # ############################################################################# # @@ -45,6 +49,15 @@ # / HTMLPage - "yes" / "no" - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no" # ############################################################################# +# +# +# TODO +# +# 1. Delete loglines records from any previous run of same table +# 2. Add tracking LogId for each cont in the table +# 3. Use link directory file to generate h1 / h2 tags for title and section headings +# 4. Ditto for links to underlying data +# # internal modules (part of core perl distribution) use strict; @@ -60,6 +73,7 @@ use Sys::Hostname; use Switch; use DBIx::Simple; +use URI::URL; #use CGI; #use HTML::TextToHTML; @@ -81,7 +95,7 @@ #Configuration section my %opt = ( - version => '0.7.5a', # please update at each change. + version => '0.7.10', # please update at each change. debug => 0, # guess what ? sendmail => '/usr/sbin/sendmail', # Path to sendmail stub from => 'spamfilter-stats', # Who is the mail from @@ -96,7 +110,7 @@ my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost my $DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything) my $DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval - +my $localIPregexp = ".*((127\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(192\.168\.)).*"; my $MinCol = 6; #Minimum column width my $HourColWidth = 16; #Date and time column width @@ -135,10 +149,12 @@ my %found_SARules = (); my %junkcount = (); my %unrecog_plugin = (); +my %blacklistURL = (); #Count of use of each balcklist rhsbl +my %usercounts = (); #Count per received email of sucessful delivery, queued spam and deleted Spam, and rejected # replaced by... -my %counts = (); #Hold all counts in 2-D matrix -my @display = (); #used to switch on and off columns - yes, no or auto for each category +my %counts = (); #Hold all counts in 2-D matrix +my @display = (); #used to switch on and off columns - yes, no or auto for each category my @colwidth = (); #width of each column #(auto means only if non zero) - populated from possible db entries my @finaldisplay = (); #final decision on display or not - true or false @@ -149,8 +165,9 @@ my $CATWEBMAIL='WebMail'; my $CATMAILMAN='Mailman'; my $CATLOCAL='Local'; +my $CATRELAY="Relay"; # border between where it came from and where it ended.. -my $countfromhere = 5; #Temp - Check this not moved!! +my $countfromhere = 6; #Temp - Check this not moved!! my $CATVIRUS='Virus'; my $CATRBLDNS='RBL/DNS'; @@ -166,7 +183,7 @@ my $CATPERCENT='PERCENT'; my $CATDMARC="DMARC Rej."; my $CATLOAD="Rej.Load"; -my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT); +my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATRELAY,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT); my $GRANDTOTAL = '99'; #subs for count arrays, for grand total my $PERCENT = '98'; # for column percentages @@ -238,10 +255,21 @@ #dbg("DMARC-EMAILS:".$DMARC_Report_emails); +# and setup list of local domains for spotting the local one in a list of email addresses (Remote station processing) +use esmith::DomainsDB; +my $d = esmith::DomainsDB->open_ro(); +my @domains = $d->keys(); +my $alldomains = "("; +foreach my $dom (@domains){$alldomains .= $dom."|"} +$alldomains .= ")"; +#print $alldomains; + # Saving the Log lines processed my %LogLines = (); #Save all the log lines processed for writing to the DB my %LogId = (); #Save the Log Ids. my $CurrentLogId = ""; +my $Sequence = 0; + # store the domain of interest. Every other records are stored in a 'Other' zone my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n"; @@ -306,6 +334,7 @@ my $enableGeoiptable; my $enablejunkMailList; my $savedata; +my $enableblacklist; #Enabled according to setting in qpsmtpd if ($cdb->get('mailstats')){ $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false; $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false; @@ -319,12 +348,15 @@ $enableGeoiptable = $true; $savedata = $false; } - $savedata = $false; #TEMP!! - -# -#--------------------------------------- -# Scan the qpsmtpd log file(s) -#--------------------------------------- + $enableblacklist = ($cdb->get('qpsmtpd')->prop("RHSBL") || "disabled") eq "enabled" || ($cdb->get('qpsmtpd')->prop("URIBL") || "disabled") eq "enabled"; + #$savedata = $false; #TEMP!! +#if ($savedata){print STDERR "yes"} else {print STDERR "no"} + +my $makeHTMLemail = "no"; +#if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!! +my $makeHTMLpage = "no"; +#if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"} +#if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"} # Init the hashes @@ -373,6 +405,12 @@ my $count = -1; #for loop reduction in debugging mode +# +#--------------------------------------- +# Scan the qpsmtpd log file(s) +#--------------------------------------- + + my $CurrentMailId = ""; LINE: while (<>) { @@ -391,36 +429,38 @@ #Count lines and skip out if debugging $count++; - last LINE if ($opt{debug} && $count >= 100000); + #last LINE if ($opt{debug} && $count >= 100); #dbg("REST:".$log); + #Loglines to Saved String for later DB write if ($savedata) { my $CurrentLine = $_; $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/; - if ($2 ne $CurrentMailId) { - $CurrentLogId = $1."-".$2; - $CurrentMailId = $2; + my $l = length($CurrentLine); + if ($l != 0){ + if (defined($2)){ #print STDERR "Undefined \$2:".$_.":".$count.":".$l;exit} + if ($2 ne $CurrentMailId) { + print "CL:$CurrentLine*\n" if !defined($1); + $CurrentLogId = $1."-".$2; + $CurrentMailId = $2; + $Sequence = 0; + } else {$Sequence++} + #$CurrentLogId .=":".$Sequence; + $LogLines{$CurrentLogId.":".$Sequence} = $_; + } } - $LogLines{$CurrentLogId} = $_; - #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n"; + #print STDERR $CurrentLogId.":".$LogLines{$CurrentLogId}."\n"; + #exit } - #Count lines and skip out if debugging - $count++; - last LINE if ($opt{debug} && $count >= 100); - #dbg("REST:".$log); - # pull out spamasassin rule lists if ( $_ =~m/spamassassin: pass, Ham,(.*)= 100000); - #dbg("REST:".$log); my $abstime = Time::TAI64::tai2unix($tai); my $abshour = floor( $abstime / 3600 ); # Hours since the epoch @@ -465,35 +500,89 @@ my (@log_items) = split $FS, $log_part; my (@timestamp_items) = split(' ',$timestamp_part); - + + my $result= "rejected"; #Tag as rejected unti we know otherwise # we store the more recent recipient domain, for domain statistics # in fact, we only store the first recipient. Could be sort of headhache # to obtain precise stats with many recipients on more than one domain ! my $proc = $timestamp_items[1] ; #numeric Id for the email + my $emailnum = $proc; #proc gets modified later... + if ($emailnum == 23244) { + dbg("TM0:".$timestamp_items[0]); + dbg("TM1:".$timestamp_items[1]); + dbg("TM2:".$timestamp_items[2]); + dbg("TM3:".$timestamp_items[3]); + dbg("LOG0:".$log_items[0]); + dbg("LOG1:".$log_items[1]); + dbg("LOG2:".$log_items[2]); + dbg("LOG3:".$log_items[3]); + dbg("LOG4:".$log_items[4]); + dbg("LOG5:".$log_items[5]); + dbg("LOG6:".$log_items[6]); + dbg("LOG7:".$log_items[7]); + dbg("IPregexp:".$localIPregexp); + if (!test_for_private_ip($log_items[0])) {dbg("Log0 not found");} + if (test_for_private_ip($log_items[2])){ dbg("Log2 match")} + if ($log_items[5] eq "queued") {dbg("LOG5 match")} + } + $totalexamined++; - #dbg("LOG1:".$log_items[1]); - #dbg("LOG3:".$log_items[3]); +# dbg("LOG8:".$log_items[8]); # first spot the fetchmail and local deliveries. # Spot from local workstation $localflag = 0; $WebMailflag = 0; - if ( $log_items[1] =~ m/.*$DomainName.*/ ) { + if ( $log_items[1] =~ m/$DomainName/ ) { #bjr + #dbg("LOG1-Found:".$log_items[1]); $localsendtotal++; $counts{$abshour}{$CATLOCAL}++; $localflag = 1; } + + #Or a remote station + elsif ((!test_for_private_ip($log_items[0])) and (test_for_private_ip($log_items[2])) and ($log_items[5] eq "queued")) + { + #Remote user + $localflag = 1; + $counts{$abshour}{$CATRELAY}++; + } + + elsif (($log_items[2] =~ m/$WebmailIP/) and (!test_for_private_ip($log_items[0]))) { + #Webmail +# if ($emailnum == 19608){ + dbg("WEBMAIL:"); + dbg("TM0:".$timestamp_items[0]); + dbg("TM1:".$timestamp_items[1]); + dbg("TM2:".$timestamp_items[2]); + dbg("TM3:".$timestamp_items[3]); + dbg("LOG0:".$log_items[0]); + dbg("LOG1:".$log_items[1]); + dbg("LOG2:".$log_items[2]); + dbg("LOG3:".$log_items[3]); + dbg("LOG4:".$log_items[4]); + dbg("LOG5:".$log_items[5]); + dbg("LOG6:".$log_items[6]); + dbg("LOG7:".$log_items[7]); + #exit; +# } + $localflag = 1; + $WebMailsendtotal++; + $counts{$abshour}{$CATWEBMAIL}++; + $WebMailflag = 1; + } # see if from localhost - elsif ( $log_items[1] =~ m/.*$localhost.*/ ) { + elsif ( $log_items[1] =~ m/$localhost/ ) { # but not if it comes from fetchmail - if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { } + if ( $log_items[3] =~ m/$FETCHMAIL/ ) { } else { + $localflag = 1; # might still be from mailman here - if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) { + if ( $log_items[3] =~ m/$MAILMAN/ ) { $mailmansendcount++; $localsendtotal++; $counts{$abshour}{$CATMAILMAN}++; @@ -501,12 +590,10 @@ } else { #Or sent to the DMARC server - dbg("LOG4:".$log_items[4]); + #dbg("LOG4:".$log_items[4]); #check for email address in $DMARC_Report_emails string - #if ($log_items[4] =~ m/.*$DMARCDomain.*/) { my $logemail = $log_items[4]; - #print STDERR "/",$log_items[4]."/\n"; - if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){ + if ((index($DMARC_Report_emails,$logemail)>=0) or ($logemail =~ m/$DMARCDomain/)){ $localsendtotal++; $DMARCSendCount++; $localflag = 1; @@ -514,10 +601,11 @@ else { #print STDERR "no match:.".$logemail; if (exists $log_items[8]){ - dbg("LOG8:".$log_items[8]); + #dbg("LOG8:".$log_items[8]); # ignore incoming localhost spoofs - if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { } + if ( $log_items[8] =~ m/msg denied before queued/ ) { } else { + #Webmail $localflag = 1; $WebMailsendtotal++; $counts{$abshour}{$CATWEBMAIL}++; @@ -536,12 +624,12 @@ } # try to spot fetchmail emails - if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) { - dbg("LOG0:".$log_items[0]); + if ( $log_items[0] =~ m/$FetchmailIP/ ) { + #dbg("LOG0:".$log_items[0]); $localAccepttotal++; $counts{$abshour}{$CATFETCHMAIL}++; } - elsif ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { + elsif ( $log_items[3] =~ m/$FETCHMAIL/ ) { $localAccepttotal++; $counts{$abshour}{$CATFETCHMAIL}++; } @@ -549,12 +637,12 @@ # and adjust for recipient field if not set-up by denying plugin - extract from deny msg if ( length( $log_items[4] ) == 0 ) { - dbg("LOG7:".$log_items[0]); + #dbg("LOG7:".$log_items[0]); if ( $log_items[5] eq 'check_goodrcptto' ) { if ( $log_items[7] gt "invalid recipient" ) { $log_items[4] = substr( $log_items[7], 18 ); #Leave only email address - dbg("LOG4:".$log_items[0]); + #dbg("LOG4:".$log_items[0]); } } @@ -563,7 +651,7 @@ # if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) { # reduce to lc and process each e,mail if a list, pseperatedy commas my $recipientmail = lc( $log_items[4] ); - dbg("LOG4:".$log_items[0]); + #dbg("LOG4:".$log_items[0]); if ( $recipientmail =~ m/.*,/ ) { #comma - split the line and deal with each domain @@ -599,146 +687,179 @@ # then categorise the result - if (exists $log_items[5]) { + if (exists $log_items[5]) { - if ($log_items[5] eq 'naughty') { + if ($log_items[5] eq 'naughty') { my $rejreason = $log_items[7]; $rejreason = /.*(\(.*\)).*/; - $rejreason = $1; + if (!defined($1)){$rejreason = "unknown"} + else {$rejreason = $1} $found_qpcodes{$log_items[5]."-".$rejreason}++} else {$found_qpcodes{$log_items[5]}++} ##Count different qpsmtpd result codes - #Check for badly formed lines (from earlier testing) - - if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} - elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} - elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one. + elsif ($log_items[5] eq 'tnef2mime') { } #Not expecting this one. - elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++; - # and extract the spam score -# if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") - if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") + elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++; + # and extract the spam score + # if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") + if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") {$rejectspamavg += $1} - mark_domain_rejected($proc); - next LINE - } + mark_domain_rejected($proc); + } - elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++; - #extract the virus name - if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;} - else {$found_viruses{$log_items[7]}++} #Some other message!! - dbg("LOG7:".$log_items[7]); - mark_domain_rejected($proc); - next LINE - } + elsif (($log_items[5] eq 'virus::clamav') or ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++; + #extract the virus name + if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;} + else {$found_viruses{$log_items[7]}++} #Some other message!! + #dbg("LOG7:".$log_items[7]); + mark_domain_rejected($proc); + } - elsif ($log_items[5] eq 'queued') { $Accepttotal++; - #extract the spam score - if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") { - $score = $1; -# print $log_items[8]."<".$score.">\n"; - if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score} - else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score} - } else { - # no SA score - treat it as ham - $hamcount++;$counts{$abshour}{$CATHAM}++; - } - if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; - $currentrcptdomain{ $proc } = '' ; - } - next LINE - } - + elsif ($log_items[5] eq 'queued') { $Accepttotal++; + #extract the spam score + # Remove count for rejectred as it looks as if it might get through!! + $result= "queued"; + if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") { + $score = trim($1); + if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric + { + if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;} + else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";} + } else { + print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n"; + } + } else { + # no SA score - treat it as ham + $hamcount++;$counts{$abshour}{$CATHAM}++; + } + if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ; + $currentrcptdomain{ $proc } = '' ; + } + } - elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'naughty') { + elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} + + elsif ($log_items[5] eq 'naughty') { #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify if ($log_items[7] =~ m/(karma)/) { - $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE} + $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)} elsif ($log_items[7] =~ m/(dnsbl)/){ - $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++} elsif ($log_items[7] =~ m/(helo)/){ - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} else { #Unidentified Naughty rejection - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE} + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++} } - elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} - - elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} - elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} + + elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)} #Treat it as Unconf if not recognised - else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE} + else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++} } #Log[5] exists - -# print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected -# next LINE - - - + #Entry if not local send + if ($localflag == 0) { + if (length($log_items[4]) > 0){ + # Need to check here for multiple email addresses + my @emails = split(",",lc($log_items[4])); + if (scalar(@emails) > 1) { + #Just pick the first local address to hang it on. + # TEMP - just go for the first address until I can work out how to spot the 1st "local" one + $usercounts{$emails[0]}{$result}++; + $usercounts{$emails[0]}{"proc"} = $proc; + #Compare with @domains array until we get a local one + my $gotone = $false; + foreach my $email (@emails){ + #Extract the domain from the email address + my $fullemail = $email; + $email = s/.*\@(.*)$/$1/; + #and see if it is local + if ($email =~ m/$alldomains/){ + $usercounts{lc($fullemail)}{$result}++; + $usercounts{lc($fullemail)}{"proc"} = $proc; + $gotone = $true; + last; + } + } + if (!$gotone) { + $usercounts{'No internal email $proc'}{$result}++; + $usercounts{'No internal email $proc'}{"proc"} = $proc; + } + + } else { + $usercounts{lc($log_items[4])}{$result}++; + $usercounts{lc($log_items[4])}{"proc"} = $proc; + } + } + } + #exit if $emailnum == 15858; + } #END OF MAIN LOOP #total up grand total Columns @@ -750,7 +871,7 @@ $counts{$GRANDTOTAL}{$categs[$ncateg]} += $counts{$nhour}{$categs[$ncateg]}; # and total rows - if ( $ncateg < $categlen && $ncateg>=$countfromhere) {#skip initial columns of non final reasons + if ( $ncateg < $categlen and $ncateg>=$countfromhere) {#skip initial columns of non final reasons $counts{$nhour}{$categs[@categs-2]} += $counts{$nhour}{$categs[$ncateg]}; } $ncateg++ @@ -815,7 +936,7 @@ my $oldfh; #Open Sendmail if we are mailing it -if ( $opt{'mail'} && !$disabled ) { +if ( $opt{'mail'} and !$disabled ) { open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" ) or die "Can't open sendmail: $!\n"; print SENDMAIL "From: $opt{'from'}\n"; @@ -841,44 +962,44 @@ my $oldFH = select $outputFH; - print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n"; - print "----------------------------------------------------", "\n\n"; - - print "$0 Version : $opt{'version'}", "\n\n"; - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n"; + print "SMEServer daily Anti-Virus and Spamfilter statistics from $hostname - ".strftime( "%F", localtime($start))."\n"; + print "----------------------------------------------------------------------------------", "\n\n"; + print "$0 Version : $opt{'version'}", "\n"; + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n"; print "Period Ending : ", strftime( "%c", localtime($end) ), "\n"; - print "\n"; - - print "Clam Version/DB Count/Last DB update: ", `freshclam -V`."\n"; - print "SpamAssassin Version : ", `spamassassin -V`."\n"; - printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel; + print "Clam Version/DB Count/Last DB update: ",`freshclam -V`; + print "SpamAssassin Version : ",`spamassassin -V`; + printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,$SARejectLevel; if ($HighLogLevel) { printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n"; printf "\tYou can set it this way:\n"; printf "\tconfig setprop qpsmtpd LogLevel 6\n"; printf "\tsignal-event email-update\n"; - printf "\tsv t /var/service/qpsmtpd\n\n"; + printf "\tsv t /var/service/qpsmtpd\n"; } - print "\n\n"; printf "Reporting Period : %.2f hrs\n", $hrsinperiod; - #print "----------------------------\n"; - #print "\n"; - printf "All SMTP connections accepted:%8d \n", $totalexamined; - printf "Emails per hour : %8.1f/hr\n", $emailperhour || 0; - print "\n"; printf "Average spam score (accepted): %11.2f\n", $spamavg || 0; printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0; printf "Average ham score : %11.2f\n", $hamavg || 0; - printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount || 0; - if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;} + printf "Number of DMARC reporting emails sent:\t%11d (not shown on table)\n", $DMARCSendCount || 0; + if ($hamcount != 0){ printf "Number of emails approved through DMARC:\t%11d (%3d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;} + + my $smeoptimizerprog = "/usr/local/smeoptimizer/SMEOptimizer.pl"; + if (-e $smeoptimizerprog) { + #smeoptimizer installed - get result of status + my @smeoptimizerlines = split(/\n/,`/usr/local/smeoptimizer/SMEOptimizer.pl -status`); + print("SMEOptimizer status:\n"); + print("\t".$smeoptimizerlines[6]."\n"); + print("\t".$smeoptimizerlines[7]."\n"); + print("\t".$smeoptimizerlines[8]."\n"); + print("\t".$smeoptimizerlines[9]."\n"); + print("\t".$smeoptimizerlines[10]."\n"); + } + - print "\n\n"; - print "\nStatistics by Hour\n"; - print "-------------------\n"; - #print "\n"; - + print "\nStatistics by Hour:\n"; # # start by working out which colunns to show - tag the display array # @@ -968,24 +1089,17 @@ # # print it. # - my $makeHTMLemail = "no"; - #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!! - my $makeHTMLpage = "no"; - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"} - #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"} - - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion .... - print $Titles."\n"; - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto - #$Line2 =~ s/-/a/g; - #print $Line2."\n"; - #print "\n"; - print $Values."\n"; + + print $Line1."\n"; + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion .... + print $Titles."\n"; + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto + print $Line2."\n"; + print $Values; print $Line2."\n"; print $Totals."\n"; print $Percent."\n"; print $Line1."\n"; - print "\n"; if ($localAccepttotal>0) { print "*Fetchml* means connections from Fetchmail delivering email\n"; @@ -1021,7 +1135,7 @@ } - if ( !$RHSenabled || !$DNSenabled ) { + if ( !$RHSenabled or !$DNSenabled ) { # comment about RBL not set print @@ -1114,11 +1228,15 @@ if ($enableSARules) {show_SARules_codes();} - if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();} + if ($enableGeoiptable and $finaldisplay[$BadCountryCateg]){show_Geoip_results();} if ($enablejunkMailList) {List_Junkmail();} + + if ($enableblacklist) {show_blacklist_counts();} + + show_user_stats(); - print "\nDone. Report generated in $telapsed sec.\n\n"; + print "\nReport generated in $telapsed sec.\n"; if ($savedata) { save_data(); } else @@ -1128,25 +1246,26 @@ select $oldFH; close $outputFH; - if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr} - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){ + if ($makeHTMLemail eq "no" or $makeHTMLemail eq "both") {print $tablestr} + if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" or $makeHTMLpage eq "yes"){ #Convert text to html and send it require CGI; require TextToHTML; my $cgi = new CGI; my $text = $tablestr; - print $cgi->header(); my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20); my $conv = new HTML::TextToHTML(); $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2, underline_length_tolerance=>1); - my $html=" \n"; + + my $html = $cgi->header(); + $html .=" \n"; $html .= "Mailstats -".strftime( "%F", localtime($start) ).""; $html .= "\n"; $html .= "\n"; $html .= $conv->process_chunk($text); $html .= "\n"; - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html} + if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" ) {print $html} #And drop it into a file if ($makeHTMLpage eq "yes") { my $filename = "mailstats.html"; @@ -1206,7 +1325,7 @@ } elsif ($base =~m/\d+/){ $sec=0;$min=0;$hour=$base; }; - #$mday="17"; #$mday="03"; #$mday="16"; #Temp!! + #$mday="05"; #$mday="03"; #$mday="16"; #Temp!! $time = timelocal($sec,$min,$hour,$mday,$mon,$year); } @@ -1255,10 +1374,9 @@ } my $i = keys %junkcount; if ( $i > 0 ) { - print "\n\n"; print("\nJunk Mails left in folder:\n"); - print("---------------------------\n\n"); - print("\nCount\tUser\n"); + print("---------------------------\n"); + print("Count\tUser\n"); print("-------------------------\n"); foreach my $thisuser ( sort { $junkcount{$b} <=> $junkcount{$a} } @@ -1282,20 +1400,20 @@ # { - - print("Virus Statistics by name:\n"); - print("------------------------------------------------------------------------\n"); + my $line = "------------------------------------------------------------------------\n"; + print("\nVirus Statistics by name:\n"); + print($line); foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} } keys %found_viruses) { - if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){ + if (index($virus,"Sanesecurity") !=-1 or index($virus,"UNOFFICIAL") !=-1){ print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n"; } else { print "Rejected $found_viruses{$virus}\t$virus\n"; } } - print("------------------------------------------------------------------------\n\n"); + print($line); } sub show_qpsmtpd_codes @@ -1305,18 +1423,94 @@ # { - - print("Qpsmtpd codes league table:\n"); - print("---------------------------------------------\n"); - print("Count\tPercent\tReason\t\n"); - print("---------------------------------------------\n"); + my $line = "---------------------------------------------\n"; + print("\nQpsmtpd codes league table:\n"); + print($line); + print("Count\tPercent\tReason\n"); + print($line); foreach my $qpcode (sort { $found_qpcodes{$b} <=> $found_qpcodes{$a} } keys %found_qpcodes) { - print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined; + print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t\t$qpcode\n" if $totalexamined; + } + print($line); +} + +sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s }; + +sub get_domain +{ #my $url= URI->new(shift); + #my $domain = $url->host; + my $url = shift; + $url =~ s!^.*https?://(?:www\.)?!!i; + $url =~ s!/.*!!; + $url =~ s/[\?\#\:].*//; + my $domain = trim($url); + return $domain; +} + +sub show_blacklist_counts + +# +# Show a sorted league table of the blacklist URL counts +# + +{ + my $line = "------------------\n"; + print("\nBlacklist details:\n"); + print($line); + if ($cdb->get('qpsmtpd')->prop("RHSBL") eq "enabled") {print "RBLLIST:".$cdb->get('qpsmtpd')->prop("RBLList")."\n";} + if ($cdb->get('qpsmtpd')->prop("URIBL") eq "enabled") {print "UBLLIST:".$cdb->get('qpsmtpd')->prop("UBLList")."\n";} + if (!$cdb->get('qpsmtpd')->prop("SBLList") eq "") {print "SBLLIST:".$cdb->get('qpsmtpd')->prop("SBLList")."\n";} + print($line); + print("Count\tURL\n"); + print($line); + foreach my $blcode (sort { $blacklistURL{$b} <=> $blacklistURL{$a} } + keys %blacklistURL) + { + print sprintf('%3u',$blacklistURL{$blcode})."\t$blcode\n"; } - print("---------------------------------------------\n\n"); - print "\n\n"; + print($line); +} + + +sub show_user_stats + +# +# Show a sorted league table of the user counts +# + +{ + #Compute totals for each entry + my $grandtotals=0; + my $totalqueued=0; + my $totalspam=0; + my $totalrejected=0; + foreach my $user (keys %usercounts){ + $usercounts{$user}{"queued"} = 0 if !(exists $usercounts{$user}{"queued"}); + $usercounts{$user}{"rejected"} = 0 if !(exists $usercounts{$user}{"rejected"}); + $usercounts{$user}{"spam"} = 0 if !(exists $usercounts{$user}{"spam"}); + $usercounts{$user}{"totals"} = $usercounts{$user}{"queued"}+$usercounts{$user}{"rejected"}+$usercounts{$user}{"spam"}; + $grandtotals += $usercounts{$user}{"totals"}; + $totalspam += $usercounts{$user}{"spam"}; + $totalqueued += $usercounts{$user}{"queued"}; + $totalrejected += $usercounts{$user}{"rejected"}; + } + my $line = "--------------------------------------------------\n"; + print("\nStatistics by email address received:\n"); + print($line); + print("Queued\tRejected\tSpam tagged\tEmail Address\n"); + print($line); + foreach my $user (sort { $usercounts{$b}{"totals"} <=> $usercounts{$a}{"totals"} } + keys %usercounts) + { + print sprintf('%3u',$usercounts{$user}{"queued"})."\t".sprintf('%3u',$usercounts{$user}{"rejected"})."\t\t".sprintf('%3u',$usercounts{$user}{"spam"})."\t\t$user\n"; + } + print($line); + print sprintf('%3u',$totalqueued)."\t".sprintf('%3u',$totalrejected)."\t\t".sprintf('%3u',$totalspam)."\n"; + print($line); + + } sub show_Geoip_results @@ -1335,10 +1529,11 @@ $percentthreshold = 0.5; } if ($total_countries > 0) { - print("Geoip results: (cutoff at $percentthreshold%) \n"); - print("---------------------------------\n"); + my $line = "---------------------------------------------\n"; + print("\nGeoip results: (cutoff at $percentthreshold%) \n"); + print($line); print("Country\tPercent\tCount\tRejected?\n"); - print("---------------------------------\n"); + print($line); foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} } keys %found_countries) { @@ -1347,14 +1542,14 @@ $totalpercent = $totalpercent + $percent; if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";} if ( $percent >= $percentthreshold ) { - print "$country\t" + print "$country\t\t" . sprintf( '%4.1f', $percent ) - . "%\t$found_countries{$country}","\t$reject\n" + . "%\t\t$found_countries{$country}","\t$reject\n" if $total_countries; } } - print("---------------------------------\n"); + print($line); my ($showtotals); if ($cdb->get('mailstats')){ $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); @@ -1363,10 +1558,9 @@ } if ($showtotals){ - print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n"; - print("---------------------------------\n\n"); + print "TOTALS\t\t".sprintf("%4.1f",$totalpercent)."%\t\t$total_countries\n"; + print($line); } - print "\n"; } } @@ -1384,7 +1578,7 @@ if ($sum_SARules > 0){ - if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { + if ($totalexamined >0 and $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { $defaultpercentthreshold = $maxcutoff } else { $defaultpercentthreshold = $mincutoff @@ -1394,17 +1588,15 @@ } else { $percentthreshold = $defaultpercentthreshold } - - print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n"); - print("---------------------------------------------\n"); + my $line = "---------------------------------------------\n"; + print("\nSpamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n"); + print($line); print("Count\tPercent\tScore\t\t\n"); - print("---------------------------------------------\n"); + print($line); foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} } keys %found_SARules) { - my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined - if $totalexamined; - #$totalpercent = $totalpercent + $percent; + my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined if $totalexamined; my $avehits = $found_SARules{$SARule}{'totalhits'} / $found_SARules{$SARule}{'count'} if $found_SARules{$SARule}{'count'}; @@ -1416,7 +1608,7 @@ if $totalexamined; } } - print("---------------------------------------------\n"); + print($line); my ($showtotals); if ($cdb->get('mailstats')){ $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); @@ -1426,7 +1618,7 @@ if ($showtotals){ print "$totalexamined\t(TOTALS)\n"; - print("---------------------------------------------\n"); + print($line); } print "\n"; } @@ -1631,15 +1823,25 @@ } $nhour++; } - # and write out the log lines saved - - foreach my $logid (keys %LogLines){ - - $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')"); + # and write out the log lines saved - only if html wanted + if ($makeHTMLemail eq 'yes' or $makeHTMLemail eq 'both' or $makeHTMLpage eq 'yes'){ + foreach my $logid (keys %LogLines){ + $reccount++; + #Extract from keys + my $extract = $logid; + $extract =~/^(.*)-(.*):(.*)$/; + my $Log64n = $1; + my $LogMailId = $2; + my $LogSeq = $3; + my $LogLine = $dbh->quote($LogLines{$logid}); + my $sql = "INSERT INTO LogData (Log64n,MailID,Sequence,LogStr) VALUES ('"; + $sql .= $Log64n."','".$LogMailId."','".$LogSeq."',".$LogLine.")"; + $dbh->do($sql) or die($sql); + } + $dbh->disconnect(); + $telapsed = time - $tstart; + print "Saved $reccount records in $telapsed sec."; } - $dbh->disconnect(); - my $telapsed = time - $tstart; - print "Saved $reccount records in $telapsed sec."; } sub check_date_rec @@ -1706,3 +1908,47 @@ my $daterec = $sth->fetchrow_hashref(); $daterec->{"dateid"}; } + + sub dump_entries + { + my $msg = shift; + #dbg($msg); + #dbg("TM0:".$timestamp_items[0]); + #dbg("TM1:".$timestamp_items[1]); + #dbg("TM2:".$timestamp_items[2]); + #dbg("TM3:".$timestamp_items[3]); + #dbg("LOG0:".$log_items[0]); + #dbg("LOG1:".$log_items[1]); + #dbg("LOG2:".$log_items[2]); + #dbg("LOG3:".$log_items[3]); + #dbg("LOG4:".$log_items[4]); + #dbg("LOG5:".$log_items[5]); + #dbg("LOG6:".$log_items[6]); + #dbg("LOG7:".$log_items[7]); + #if ($opt{debug} == 1){exit;} +} + +#sub test_for_private_ip { + #use NetAddr::IP; + #my $ip = shift; + #$ip =~ s/^\D*(([0-9]{1,3}\.){3}[0-9]{1,3}).*/$1/e; + #print "\nIP:$ip"; + #my $nip = NetAddr::IP->new($ip); + #if ($nip){ + #if ( $nip->is_rfc1918() ){ + #return 1; + #} else { return 0} + #} else { return 0} +#} + + +sub test_for_private_ip { + use NetAddr::IP; + $_ = shift; + return unless /(\d+\.\d+\.\d+\.\d+)/; + my $ip = NetAddr::IP->new($1); + return unless $ip; + return $ip->is_rfc1918(); +} + +