diff -Nur smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl --- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl 2016-07-02 04:29:21.671000000 -0400 +++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl 2016-07-02 04:39:34.037000000 -0400 @@ -18,6 +18,8 @@ # bjr - 19Jun15 - Add totals for the League tables # bjr and Unnilennium - 08Apr16 - Add in else for unrecognised plugin detection # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection +# bjr - 14Jun16 - make compatible with qpsmtpd 0.96 +# bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7) # ############################################################################# # @@ -37,8 +39,10 @@ # / ShowLeagueTotals - Show totals row after league tables - (default is "yes") # / DBHost - MySQL server hostname (default is "localhost"). # / DBPort - MySQL server post (default is "3306") -# / Interval - "day", "week", "fortnight", "month", "99999" - last is number of seconds (default is day) +# / Interval - "daily", "weekly", "fortnightly", "monthly", "99999" - last is number of hours (default is daily) # / Base - "Midnight", "Midday", "Now", "99" hour (0-23) (default is midnight) +# / HTMLEmail - "yes", "no", "both" - default is "No" - Send email in HTML +# / HTMLPage - "yes" / "no" - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no" # ############################################################################# @@ -55,6 +59,10 @@ use esmith::DomainsDB; use Sys::Hostname; use Switch; +use DBIx::Simple; + +#use CGI; +#use HTML::TextToHTML; my $hostname = hostname(); my $cdb = esmith::ConfigDB->open_ro or die "Couldn't open ConfigDB : $!\n"; @@ -73,12 +81,11 @@ #Configuration section my %opt = ( - version => '0.6.29', # please update at each change. + version => '0.7.5a', # please update at each change. debug => 0, # guess what ? sendmail => '/usr/sbin/sendmail', # Path to sendmail stub from => 'spamfilter-stats', # Who is the mail from - mail => # mailstats email recipient - $cdb->get('mailstats')->prop('Email') || 'admin', + mail => $cdb->get('mailstats')->prop('Email') || 'admin', # mailstats email recipient timezone => `date +%z`, ); @@ -87,8 +94,10 @@ my $localhost = 'localhost'; #Apparent sender for webmail my $FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost +my $DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything) +my $DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval -my $MinCol = 8; #Minimum column width +my $MinCol = 6; #Minimum column width my $HourColWidth = 16; #Date and time column width my $SARulethresholdPercent = 10; #If Sa rules less than this of total emails, then cutoff reduced @@ -116,6 +125,10 @@ my $totalexamined = 0; #total download + RBL etc my $WebMailsendtotal = 0; #total from Webmail my $mailmansendcount = 0; #total from mailman +my $DMARCSendCount = 0; #total DMARC reporting emails sent (approx) +my $DMARCOkCount = 0; #Total emails approved through DMARC + + my %found_viruses = (); my %found_qpcodes = (); @@ -137,25 +150,35 @@ my $CATMAILMAN='Mailman'; my $CATLOCAL='Local'; # border between where it came from and where it ended.. -my $countfromhere = 5; - +my $countfromhere = 5; #Temp - Check this not moved!! + my $CATVIRUS='Virus'; my $CATRBLDNS='RBL/DNS'; my $CATEXECUT='Execut.'; my $CATNONCONF='Non.Conf.'; my $CATBADCOUNTRIES='Geoip.'; -my $BadCountryCateg=8; #Careful here this number could change if more added before. +my $CATKARMA="Karma"; + my $CATSPAMDEL='Del.Spam'; my $CATSPAM='Qued.Spam?'; my $CATHAM='Ham'; my $CATTOTALS='TOTALS'; my $CATPERCENT='PERCENT'; -my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT); +my $CATDMARC="DMARC Rej."; +my $CATLOAD="Rej.Load"; +my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT); my $GRANDTOTAL = '99'; #subs for count arrays, for grand total my $PERCENT = '98'; # for column percentages my $categlen = @categs-2; #-2 to avoid the total and percent column +# +# Index for certain columns - check these do not move if we add columns +# +my $BadCountryCateg=9; +my $DMARCcateg = 5; #Not used. +my $KarmaCateg=$BadCountryCateg+3; + my $above15 = 0; my $RBLcount = 0; my $MiscDenyCount = 0; @@ -187,6 +210,38 @@ my $morethanonercpt = 0 ; # count every 'second' recipients for a mail. my $recipcount = 0; # count every recipient email address received. +# +#Load up the emails curreently stored for DMARC reporting - so that we cna spot the reports being sent. +#Held in an slqite db, created by the DMARC perl lib. +# +my $dsn = "dbi:SQLite:dbname=/var/lib/qpsmtpd/dmarc/reports.sqlite"; #Taken from /etc/mail-dmarc.ini +# doesn't seem to need +my $user = ""; +my $pass = ""; +my $DMARC_Report_emails = ""; #Flat string of all email addresses + + if (my $dbix = DBIx::Simple->connect( $dsn, $user, $pass )){ + my $result = $dbix->query("select rua from report_policy_published;"); + $result->bind(my ($emailaddress)); + while ($result->fetch){ + #print STDERR "$emailaddress"; + #remember email from logterse entry has chevrons round it - so we add them here to guarantee the alighment of the match + #Remove the mailto: + $emailaddress =~ s/mailto://g; + # and map any commas to >< + $emailaddress =~ s/,/>\n" + } + $dbix->disconnect(); + } else { $DMARC_Report_emails = "None found - DB not opened"} + + +#dbg("DMARC-EMAILS:".$DMARC_Report_emails); + +# Saving the Log lines processed +my %LogLines = (); #Save all the log lines processed for writing to the DB +my %LogId = (); #Save the Log Ids. +my $CurrentLogId = ""; # store the domain of interest. Every other records are stored in a 'Other' zone my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n"; @@ -212,6 +267,8 @@ my ( $start, $end ) = analysis_period(); +dbg("Time interval:".strftime("%a %b %e %H:%M:%S %Y", localtime($start))."->".strftime("%a %b %e %H:%M:%S %Y", localtime($end))."\n"); + # # First check current configuration for logging, DNS enable and Max threshold for spamassassin # @@ -243,6 +300,27 @@ } +# get enable/disable subsections +my $enableqpsmtpdcodes; +my $enableSARules; +my $enableGeoiptable; +my $enablejunkMailList; +my $savedata; +if ($cdb->get('mailstats')){ + $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false; + $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false; + $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false; + $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false; + $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false; + } else { + $enableqpsmtpdcodes = $true; + $enableSARules = $true; + $enablejunkMailList = $true; + $enableGeoiptable = $true; + $savedata = $false; + } + $savedata = $false; #TEMP!! + # #--------------------------------------- # Scan the qpsmtpd log file(s) @@ -263,6 +341,7 @@ } # and grand totals, percent and display status from db entries, and column widths $ncateg = 0; +my $colpadding = 0; while ( $ncateg < @categs) { $counts{$GRANDTOTAL}{$categs[$ncateg]} = 0; $counts{$PERCENT}{$categs[$ncateg]} = 0; @@ -273,11 +352,11 @@ $display[$ncateg] = 'auto' } if ($ncateg == 0) { - $colwidth[$ncateg] = $HourColWidth + $colwidth[$ncateg] = $HourColWidth + $colpadding; } else { - $colwidth[$ncateg] = length($categs[$ncateg])+1 + $colwidth[$ncateg] = length($categs[$ncateg])+1+$colpadding; } - if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol} + if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol + $colpadding} $ncateg++ } @@ -292,39 +371,91 @@ } @ARGV=@ARGV2; +my $count = -1; #for loop reduction in debugging mode + +my $CurrentMailId = ""; + LINE: while (<>) { - my($tai,$log) = split(' ',$_,2); + #print STDERR $starttai,$endtai,$_,"\n"; + + next LINE if !(my($tai,$log) = split(' ',$_,2)); + #dbg("TAI:".$tai); + + #dbg("REST1:".$log); + #If date specified, only process lines matching date next LINE if ( $tai lt $starttai ); next LINE if ( $tai gt $endtai ); + #Count lines and skip out if debugging + $count++; + last LINE if ($opt{debug} && $count >= 100000); + #dbg("REST:".$log); + + #Loglines to Saved String for later DB write + if ($savedata) { + my $CurrentLine = $_; + $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/; + if ($2 ne $CurrentMailId) { + $CurrentLogId = $1."-".$2; + $CurrentMailId = $2; + } + $LogLines{$CurrentLogId} = $_; + #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n"; + } + + #Count lines and skip out if debugging + $count++; + last LINE if ($opt{debug} && $count >= 100); + #dbg("REST:".$log); + + # pull out spamasassin rule lists - if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ ) + if ( $_ =~m/spamassassin: pass, Ham,(.*)= 100000); + #dbg("REST:".$log); my $abstime = Time::TAI64::tai2unix($tai); my $abshour = floor( $abstime / 3600 ); # Hours since the epoch @@ -342,6 +473,9 @@ $totalexamined++; + #dbg("LOG1:".$log_items[1]); + #dbg("LOG3:".$log_items[3]); + # first spot the fetchmail and local deliveries. # Spot from local workstation @@ -355,11 +489,9 @@ # see if from localhost elsif ( $log_items[1] =~ m/.*$localhost.*/ ) { - # but not if it comes from fetchmail if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { } else { - # might still be from mailman here if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) { $mailmansendcount++; @@ -368,21 +500,44 @@ $localflag = 1; } else { - - # eliminate incoming localhost spoofs - if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { } - else { - $localflag = 1; - $WebMailsendtotal++; - $counts{$abshour}{$CATWEBMAIL}++; - $WebMailflag = 1; - } + #Or sent to the DMARC server + dbg("LOG4:".$log_items[4]); + #check for email address in $DMARC_Report_emails string + #if ($log_items[4] =~ m/.*$DMARCDomain.*/) { + my $logemail = $log_items[4]; + #print STDERR "/",$log_items[4]."/\n"; + if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){ + $localsendtotal++; + $DMARCSendCount++; + $localflag = 1; + } + else { + #print STDERR "no match:.".$logemail; + if (exists $log_items[8]){ + dbg("LOG8:".$log_items[8]); + # ignore incoming localhost spoofs + if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { } + else { + $localflag = 1; + $WebMailsendtotal++; + $counts{$abshour}{$CATWEBMAIL}++; + $WebMailflag = 1; + } + } + else { + $localflag = 1; + $WebMailsendtotal++; + $counts{$abshour}{$CATWEBMAIL}++; + $WebMailflag = 1; + } + } } } } # try to spot fetchmail emails if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) { + dbg("LOG0:".$log_items[0]); $localAccepttotal++; $counts{$abshour}{$CATFETCHMAIL}++; } @@ -394,10 +549,13 @@ # and adjust for recipient field if not set-up by denying plugin - extract from deny msg if ( length( $log_items[4] ) == 0 ) { + dbg("LOG7:".$log_items[0]); if ( $log_items[5] eq 'check_goodrcptto' ) { if ( $log_items[7] gt "invalid recipient" ) { $log_items[4] = - substr( $log_items[7], 18 ) #Leave only email address + substr( $log_items[7], 18 ); #Leave only email address + dbg("LOG4:".$log_items[0]); + } } } @@ -405,6 +563,7 @@ # if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) { # reduce to lc and process each e,mail if a list, pseperatedy commas my $recipientmail = lc( $log_items[4] ); + dbg("LOG4:".$log_items[0]); if ( $recipientmail =~ m/.*,/ ) { #comma - split the line and deal with each domain @@ -442,7 +601,12 @@ if (exists $log_items[5]) { - $found_qpcodes{$log_items[5]}++; ##Count different qpsmtpd result codes + if ($log_items[5] eq 'naughty') { + my $rejreason = $log_items[7]; + $rejreason = /.*(\(.*\)).*/; + $rejreason = $1; + $found_qpcodes{$log_items[5]."-".$rejreason}++} + else {$found_qpcodes{$log_items[5]}++} ##Count different qpsmtpd result codes #Check for badly formed lines (from earlier testing) @@ -488,27 +652,31 @@ elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++; # and extract the spam score - if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1} +# if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") + if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") + {$rejectspamavg += $1} mark_domain_rejected($proc); next LINE } - elsif ($log_items[5] eq 'virus::clamav') { $infectedcount++;$counts{$abshour}{$CATVIRUS}++; + elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++; #extract the virus name - if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;} + if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;} + else {$found_viruses{$log_items[7]}++} #Some other message!! + dbg("LOG7:".$log_items[7]); mark_domain_rejected($proc); next LINE } elsif ($log_items[5] eq 'queued') { $Accepttotal++; #extract the spam score - if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") { + if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") { $score = $1; # print $log_items[8]."<".$score.">\n"; if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score} else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score} } else { - # no SA score - so it must be ham + # no SA score - treat it as ham $hamcount++;$counts{$abshour}{$CATHAM}++; } if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) { @@ -523,15 +691,53 @@ elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'naughty') { + #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify + if ($log_items[7] =~ m/(karma)/) { + $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[7] =~ m/(dnsbl)/){ + $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE} + elsif ($log_items[7] =~ m/(helo)/){ + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + else { + #Unidentified Naughty rejection + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE} + } + elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + + elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE} + #Treat it as Unconf if not recognised else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE} + } #Log[5] exists + + +# print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected +# next LINE -/* - print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected - next LINE -*/ - } } #END OF MAIN LOOP @@ -624,18 +830,28 @@ if ( !$disabled ) { #Output results + + # NEW - save the print to a variable so that it can be processed into html. + # + #Save current output selection and divert into variable + # + my $output; + my $tablestr=""; + open(my $outputFH, '>', \$tablestr) or die; # This shouldn't fail + my $oldFH = select $outputFH; + + print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n"; print "----------------------------------------------------", "\n\n"; print "$0 Version : $opt{'version'}", "\n\n"; - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n"; + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n"; print "Period Ending : ", strftime( "%c", localtime($end) ), "\n"; print "\n"; - print "Clam Version : ", `freshclam -V`; - print "SpamAssassin Version : ", `spamassassin -V`; - printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel, - $SARejectLevel; + print "Clam Version/DB Count/Last DB update: ", `freshclam -V`."\n"; + print "SpamAssassin Version : ", `spamassassin -V`."\n"; + printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel; if ($HighLogLevel) { printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n"; printf "\tYou can set it this way:\n"; @@ -643,10 +859,10 @@ printf "\tsignal-event email-update\n"; printf "\tsv t /var/service/qpsmtpd\n\n"; } - print "\n"; + print "\n\n"; printf "Reporting Period : %.2f hrs\n", $hrsinperiod; - print "----------------------------\n"; - print "\n"; + #print "----------------------------\n"; + #print "\n"; printf "All SMTP connections accepted:%8d \n", $totalexamined; @@ -655,8 +871,13 @@ printf "Average spam score (accepted): %11.2f\n", $spamavg || 0; printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0; printf "Average ham score : %11.2f\n", $hamavg || 0; - print "\n"; - print "Statistics by Hour\n"; + printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount || 0; + if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;} + + print "\n\n"; + print "\nStatistics by Hour\n"; + print "-------------------\n"; + #print "\n"; # # start by working out which colunns to show - tag the display array @@ -682,13 +903,13 @@ # and put together the print lines - # + my $Line1; #Full Line across the page my $Line2; #Broken Line across the page my $Titles; #Column headers my $Values; #Values my $Totals; #Corresponding totals - my $Percent; # and column percentages + my $Percent; # and column percentages my $hour = floor( $start / 3600 ); $Line1 = ''; @@ -706,7 +927,7 @@ $Line1 .= substr('---------------------',0,$colwidth[$ncateg]); $Line2 .= substr('---------------------',0,$colwidth[$ncateg]-1); $Line2 .= " "; - $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])." "; + $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])."|"; if ($ncateg == 0) { $Totals .= substr('TOTALS ',0,$colwidth[$ncateg]-2); $Percent .= substr('PERCENTAGES ',0,$colwidth[$ncateg]-1); @@ -744,39 +965,58 @@ $hour++; } - # print it. - print $Line1."\n"; + # + # print it. + # + my $makeHTMLemail = "no"; + #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!! + my $makeHTMLpage = "no"; + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"} + #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"} + + if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion .... print $Titles."\n"; - print $Line2."\n"; + if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto + #$Line2 =~ s/-/a/g; + #print $Line2."\n"; + #print "\n"; print $Values."\n"; print $Line2."\n"; print $Totals."\n"; print $Percent."\n"; print $Line1."\n"; - + print "\n"; if ($localAccepttotal>0) { print "*Fetchml* means connections from Fetchmail delivering email\n"; } - print "*Local* means connections from workstations on local LAN.\n"; + print "*Local* means connections from workstations on local LAN.\n\n"; print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol"; - print " or email was to non existant address.\n"; + print " or email was to non existant address.\n\n"; + + if ($finaldisplay[$KarmaCateg]){ + print "*Karma* means email was rejected based on the mailserver's previous activities.\n\n"; + } + if ($finaldisplay[$BadCountryCateg]){ $BadCountries = $cdb->get('qpsmtpd')->prop('BadCountries') || "*none*"; - print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n"; + print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n\n"; } + + if (scalar keys %unrecog_plugin > 0){ #Show unrecog plugins found print "*Unrecognised plugins found - categorised as Non-Conf\n"; foreach my $unrec (keys %unrecog_plugin){ print "\t$unrec\t($unrecog_plugin{$unrec})\n"; - } + } + print "\n"; } if ($QueryNoLogTerse) { - print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; + print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n\n"; # print " to enable it follow the instructions at .............................\n"; } @@ -813,7 +1053,7 @@ # if ($Webmailsendtotal > 0) {print "If you have the mailman contrib installed, then the webmail totals might include some mailman emails\n"} # time to do a 'by recipient domain' report - print "\nIncoming mails by recipient domains usage\n"; + print "Incoming mails by recipient domains usage\n"; print "-----------------------------------------\n"; print "Domains Type Total Denied XferErr Accept \%accept\n"; @@ -869,25 +1109,6 @@ show_virus_variants(); } - # get enable/disable subsections - my $enableqpsmtpdcodes; - my $enableSARules; - my $enableGeoiptable; - my $enablejunkMailList; - my $savedata; - if ($cdb->get('mailstats')){ - $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false; - $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false; - $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false; - $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false; - $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false; - } else { - $enableqpsmtpdcodes = $true; - $enableSARules = $true; - $enablejunkMailList = $true; - $enableGeoiptable = $true; - $savedata = $false; - } if ($enableqpsmtpdcodes) {show_qpsmtpd_codes();} @@ -905,8 +1126,39 @@ "config setprop mailstats SaveDataToMySQL yes\n"; } + select $oldFH; + close $outputFH; + if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr} + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){ + #Convert text to html and send it + require CGI; + require TextToHTML; + my $cgi = new CGI; + my $text = $tablestr; + print $cgi->header(); + my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20); + my $conv = new HTML::TextToHTML(); + $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2, + underline_length_tolerance=>1); + my $html=" \n"; + $html .= "Mailstats -".strftime( "%F", localtime($start) ).""; + $html .= "\n"; + $html .= "\n"; + $html .= $conv->process_chunk($text); + $html .= "\n"; + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html} + #And drop it into a file + if ($makeHTMLpage eq "yes") { + my $filename = "mailstats.html"; + open(my $fh, '>', $filename) or die "Could not open file '$filename' $!"; + print $fh $html; + close $fh; + } + + } - #Close Senmdmail if it was opened + + #Close Sendmail if it was opened if ( $opt{'mail'} ) { select $oldfh; close(SENDMAIL); @@ -934,17 +1186,17 @@ if ($cdb->get('mailstats')) { - my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily'; + my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily'; #"fortnightly"; #"daily";# #; TEMP!! if ($interval eq "weekly") { $secsininterval = 86400*7; } elsif ($interval eq "fortnightly") { $secsininterval = 86400*14; } elsif ($interval eq "monthly") { - $secsininterval = 86400; + $secsininterval = 86400*30; } elsif ($interval =~m/\d+/) { $secsininterval = $interval*3600; }; - my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight'; + my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight'; my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); if ($base eq "Midnight"){ @@ -954,7 +1206,8 @@ } elsif ($base =~m/\d+/){ $sec=0;$min=0;$hour=$base; }; - $time = timelocal($sec,$min,$hour,$mday,$mon,$year) + #$mday="17"; #$mday="03"; #$mday="16"; #Temp!! + $time = timelocal($sec,$min,$hour,$mday,$mon,$year); } my $start = str2time( $startdate ); @@ -966,7 +1219,8 @@ sub dbg { my $msg = shift; - + my $time = scalar localtime; + $msg = $time.":".$msg."\n"; if ( $opt{debug} ) { print STDERR $msg; } @@ -1001,9 +1255,10 @@ } my $i = keys %junkcount; if ( $i > 0 ) { - print("Junk Mails left in folder:\n"); - print("-------------------------\n"); - print("Count\tUser\n"); + print "\n\n"; + print("\nJunk Mails left in folder:\n"); + print("---------------------------\n\n"); + print("\nCount\tUser\n"); print("-------------------------\n"); foreach my $thisuser ( sort { $junkcount{$b} <=> $junkcount{$a} } @@ -1033,7 +1288,7 @@ foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} } keys %found_viruses) { - if (index($virus,"Sanesecurity")!=-1){ + if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){ print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n"; } else { print "Rejected $found_viruses{$virus}\t$virus\n"; @@ -1061,6 +1316,7 @@ print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined; } print("---------------------------------------------\n\n"); + print "\n\n"; } sub show_Geoip_results @@ -1078,38 +1334,40 @@ } else { $percentthreshold = 0.5; } - print("Geoip results: (cutoff at $percentthreshold%) \n"); - print("---------------------------------\n"); - print("Country\tPercent\tCount\tRejected?\n"); - print("---------------------------------\n"); - foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} } - keys %found_countries) - { - $percent = $found_countries{$country} * 100 / $total_countries - if $total_countries; - $totalpercent = $totalpercent + $percent; - if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";} - if ( $percent >= $percentthreshold ) { - print "$country\t" - . sprintf( '%4.1f', $percent ) - . "%\t$found_countries{$country}","\t$reject\n" - if $total_countries; - } - - } - print("---------------------------------\n"); - my ($showtotals); - if ($cdb->get('mailstats')){ - $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); - } else { - $showtotals = $true; - } - - if ($showtotals){ - print "TOTALS\t$totalpercent%\t$total_countries\n"; - print("---------------------------------\n\n"); + if ($total_countries > 0) { + print("Geoip results: (cutoff at $percentthreshold%) \n"); + print("---------------------------------\n"); + print("Country\tPercent\tCount\tRejected?\n"); + print("---------------------------------\n"); + foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} } + keys %found_countries) + { + $percent = $found_countries{$country} * 100 / $total_countries + if $total_countries; + $totalpercent = $totalpercent + $percent; + if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";} + if ( $percent >= $percentthreshold ) { + print "$country\t" + . sprintf( '%4.1f', $percent ) + . "%\t$found_countries{$country}","\t$reject\n" + if $total_countries; + } + + } + print("---------------------------------\n"); + my ($showtotals); + if ($cdb->get('mailstats')){ + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); + } else { + $showtotals = $true; + } + + if ($showtotals){ + print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n"; + print("---------------------------------\n\n"); + } + print "\n"; } - print "\n"; } sub show_SARules_codes @@ -1123,52 +1381,55 @@ my ($percentthreshold); my ($defaultpercentthreshold); my ($totalpercent) = 0; - - if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { - $defaultpercentthreshold = $maxcutoff - } else { - $defaultpercentthreshold = $mincutoff - } - if ($cdb->get('mailstats')){ - $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold; - } else { - $percentthreshold = $defaultpercentthreshold - } - - print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n"); - print("---------------------------------------------\n"); - print("Count\tPercent\tScore\t\t\n"); - print("---------------------------------------------\n"); - foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} } - keys %found_SARules) - { - my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined - if $totalexamined; - #$totalpercent = $totalpercent + $percent; - my $avehits = $found_SARules{$SARule}{'totalhits'} / - $found_SARules{$SARule}{'count'} - if $found_SARules{$SARule}{'count'}; - if ( $percent >= $percentthreshold ) { - print "$found_SARules{$SARule}{'count'}\t" - . sprintf( '%4.1f', $percent ) . "%\t" - . sprintf( '%4.1f', $avehits ) - . "\t$SARule\n" + + if ($sum_SARules > 0){ + + if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { + $defaultpercentthreshold = $maxcutoff + } else { + $defaultpercentthreshold = $mincutoff + } + if ($cdb->get('mailstats')){ + $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold; + } else { + $percentthreshold = $defaultpercentthreshold + } + + print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n"); + print("---------------------------------------------\n"); + print("Count\tPercent\tScore\t\t\n"); + print("---------------------------------------------\n"); + foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} } + keys %found_SARules) + { + my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined if $totalexamined; -} - } - print("---------------------------------------------\n"); - my ($showtotals); - if ($cdb->get('mailstats')){ - $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); - } else { - $showtotals = $true; - } - - if ($showtotals){ - print "$totalexamined\t(TOTALS)\n"; + #$totalpercent = $totalpercent + $percent; + my $avehits = $found_SARules{$SARule}{'totalhits'} / + $found_SARules{$SARule}{'count'} + if $found_SARules{$SARule}{'count'}; + if ( $percent >= $percentthreshold ) { + print "$found_SARules{$SARule}{'count'}\t" + . sprintf( '%4.1f', $percent ) . "%\t" + . sprintf( '%4.1f', $avehits ) + . "\t$SARule\n" + if $totalexamined; + } + } print("---------------------------------------------\n"); + my ($showtotals); + if ($cdb->get('mailstats')){ + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); + } else { + $showtotals = $true; + } + + if ($showtotals){ + print "$totalexamined\t(TOTALS)\n"; + print("---------------------------------------------\n"); + } + print "\n"; } - print "\n"; } @@ -1370,9 +1631,15 @@ } $nhour++; } - $dbh->disconnect(); - my $telapsed = time - $tstart; - print "Saved $reccount records in $telapsed sec."; + # and write out the log lines saved + + foreach my $logid (keys %LogLines){ + + $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')"); + } + $dbh->disconnect(); + my $telapsed = time - $tstart; + print "Saved $reccount records in $telapsed sec."; } sub check_date_rec @@ -1439,5 +1706,3 @@ my $daterec = $sth->fetchrow_hashref(); $daterec->{"dateid"}; } - -