smeserver-mailstats/contribs10/smeserver-mailstats-1.1.bz9588.qpsmtpd0_96compatible.patch

diff -Nur smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl
--- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl      2016-07-02 04:29:21.671000000 -0400
+++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl  2016-07-02 04:39:34.037000000 -0400
@@ -18,6 +18,8 @@
 # bjr - 19Jun15         - Add totals for the League tables
 # bjr and  Unnilennium  - 08Apr16 - Add in else for unrecognised plugin detection
 # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection
+# bjr - 14Jun16 - make compatible with qpsmtpd 0.96
+# bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7)
 #
 #############################################################################
 #
@@ -37,8 +39,10 @@
 #                                               / ShowLeagueTotals  - Show totals row after league tables - (default is "yes")
 #            / DBHost - MySQL server hostname (default is "localhost").
 #            / DBPort - MySQL server post (default is "3306")
-#            / Interval - "day", "week", "fortnight", "month", "99999" - last is number of seconds (default is day)
+#            / Interval - "daily", "weekly", "fortnightly", "monthly", "99999" - last is number of hours (default is daily)
 #            / Base - "Midnight", "Midday", "Now", "99" hour (0-23) (default is midnight)
+#                                               / HTMLEmail - "yes", "no", "both" - default is "No" - Send email in HTML
+#                                               / HTMLPage - "yes"  / "no"  - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no"
 #
 #############################################################################
 
@@ -55,6 +59,10 @@
 use esmith::DomainsDB;
 use Sys::Hostname;
 use Switch;
+use DBIx::Simple;
+
+#use CGI;
+#use HTML::TextToHTML;
 
 my $hostname = hostname();
 my $cdb = esmith::ConfigDB->open_ro or die "Couldn't open ConfigDB : $!\n";
@@ -73,12 +81,11 @@
 
 #Configuration section
 my %opt = (
-    version => '0.6.29',                        # please update at each change.
+    version => '0.7.5a',                         # please update at each change.
     debug => 0,                                 # guess what ?
     sendmail => '/usr/sbin/sendmail',           # Path to sendmail stub
     from => 'spamfilter-stats',                 # Who is the mail from
-    mail =>                                     # mailstats email recipient
-       $cdb->get('mailstats')->prop('Email') || 'admin',
+    mail => $cdb->get('mailstats')->prop('Email') || 'admin', # mailstats email recipient
     timezone => `date +%z`,
 );
 
@@ -87,8 +94,10 @@
 my $localhost = 'localhost';           #Apparent sender for webmail
 my $FETCHMAIL = 'FETCHMAIL';    #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email
 my $MAILMAN = "bounces";        #sender when mailman sending when orig is localhost
+my $DMARCDomain="dmarc";                               #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
+my $DMARCOkPattern="dmarc: pass";  #Pattern to use to detect DMARC approval
 
-my $MinCol = 8;                 #Minimum column width
+my $MinCol = 6;                 #Minimum column width
 my $HourColWidth = 16;            #Date and time column width
 
 my $SARulethresholdPercent = 10;  #If Sa rules less than this of total emails, then cutoff reduced
@@ -116,6 +125,10 @@
 my $totalexamined = 0;                #total download + RBL etc
 my $WebMailsendtotal = 0;             #total from Webmail
 my $mailmansendcount = 0;             #total from mailman
+my $DMARCSendCount = 0;                            #total DMARC reporting emails sent (approx)
+my $DMARCOkCount = 0;                 #Total emails approved through DMARC
+
+
 
 my %found_viruses = ();
 my %found_qpcodes  = ();
@@ -137,25 +150,35 @@
 my $CATMAILMAN='Mailman';
 my $CATLOCAL='Local';
 # border between where it came from and where it ended..
-my $countfromhere = 5;
-
+my $countfromhere = 5;  #Temp - Check this not moved!!
+ 
 my $CATVIRUS='Virus';
 my $CATRBLDNS='RBL/DNS';
 my $CATEXECUT='Execut.';
 my $CATNONCONF='Non.Conf.';
 my $CATBADCOUNTRIES='Geoip.';
-my $BadCountryCateg=8;     #Careful here this number could change if more added before.
+my $CATKARMA="Karma";
+
 my $CATSPAMDEL='Del.Spam';
 my $CATSPAM='Qued.Spam?';
 my $CATHAM='Ham';
 my $CATTOTALS='TOTALS';
 my $CATPERCENT='PERCENT';
-my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
+my $CATDMARC="DMARC Rej.";
+my $CATLOAD="Rej.Load";
+my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
 my $GRANDTOTAL = '99';                #subs for count arrays, for grand total
 my $PERCENT = '98';                 # for column percentages
 
 my $categlen = @categs-2;  #-2 to avoid the total and percent column
 
+#
+# Index for certain columns - check these do not move if we add columns
+#
+my $BadCountryCateg=9;     
+my $DMARCcateg = 5;  #Not used.
+my $KarmaCateg=$BadCountryCateg+3;
+
 my $above15 = 0;
 my $RBLcount  = 0;
 my $MiscDenyCount = 0;
@@ -187,6 +210,38 @@
 my $morethanonercpt = 0 ;    # count every 'second' recipients for a mail.
 my $recipcount = 0;                 # count every recipient email address received.
 
+#
+#Load up the emails curreently stored for DMARC reporting - so that we cna spot the reports being sent.
+#Held in an slqite db, created by the DMARC perl lib.
+#
+my $dsn = "dbi:SQLite:dbname=/var/lib/qpsmtpd/dmarc/reports.sqlite"; #Taken from /etc/mail-dmarc.ini
+# doesn't seem to need 
+my $user = "";
+my $pass = "";
+my $DMARC_Report_emails = ""; #Flat string of all email addresses
+
+ if (my $dbix = DBIx::Simple->connect( $dsn, $user, $pass )){
+        my $result = $dbix->query("select rua from report_policy_published;");
+        $result->bind(my ($emailaddress));
+        while ($result->fetch){
+                       #print STDERR "$emailaddress";
+                       #remember email from logterse entry has chevrons round it - so we add them here to guarantee the alighment of the match
+                       #Remove the mailto:
+                       $emailaddress =~ s/mailto://g;
+                       # and map any commas to ><
+                       $emailaddress =~ s/,/></g;                      
+                       $DMARC_Report_emails .= "<".$emailaddress.">\n"
+        }
+        $dbix->disconnect();
+       } else { $DMARC_Report_emails = "None found - DB not opened"}
+
+
+#dbg("DMARC-EMAILS:".$DMARC_Report_emails); 
+
+# Saving the Log lines processed
+my %LogLines = ();  #Save all the log lines processed for writing to the DB
+my %LogId = ();     #Save the Log Ids.
+my $CurrentLogId = ""; 
 
 # store the domain of interest. Every other records are stored in a 'Other' zone
 my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n";
@@ -212,6 +267,8 @@
 
 my ( $start, $end ) = analysis_period();
 
+dbg("Time interval:".strftime("%a %b %e %H:%M:%S %Y", localtime($start))."->".strftime("%a %b %e %H:%M:%S %Y", localtime($end))."\n");
+
 #
 # First check current configuration for logging, DNS enable and Max threshold for spamassassin
 #
@@ -243,6 +300,27 @@
 
 }
 
+# get enable/disable subsections
+my $enableqpsmtpdcodes;
+my $enableSARules;
+my $enableGeoiptable;
+my $enablejunkMailList;
+my $savedata;
+if ($cdb->get('mailstats')){
+               $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
+               $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
+               $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false;
+               $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false;
+               $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false;
+       } else {
+               $enableqpsmtpdcodes = $true;
+               $enableSARules = $true;
+               $enablejunkMailList = $true;
+               $enableGeoiptable = $true;
+               $savedata = $false;
+       }
+       $savedata = $false;  #TEMP!!
+
 #
 #---------------------------------------
 # Scan the qpsmtpd log file(s)
@@ -263,6 +341,7 @@
 }
 # and grand totals, percent and display status from db entries, and column widths
 $ncateg = 0;
+my $colpadding = 0;
 while ( $ncateg < @categs) {
   $counts{$GRANDTOTAL}{$categs[$ncateg]} = 0;
   $counts{$PERCENT}{$categs[$ncateg]} = 0;
@@ -273,11 +352,11 @@
     $display[$ncateg] = 'auto'
   }
   if ($ncateg == 0) {
-    $colwidth[$ncateg] = $HourColWidth
+    $colwidth[$ncateg] = $HourColWidth + $colpadding;
   } else {
-    $colwidth[$ncateg] = length($categs[$ncateg])+1
+    $colwidth[$ncateg] = length($categs[$ncateg])+1+$colpadding;
   }
-  if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol}
+  if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol + $colpadding}
   $ncateg++
 }
 
@@ -292,39 +371,91 @@
 }
 @ARGV=@ARGV2;
 
+my $count = -1; #for loop reduction in debugging mode
+
+my $CurrentMailId = "";
+
 LINE: while (<>) {
-    my($tai,$log) = split(' ',$_,2);
 
+               #print STDERR $starttai,$endtai,$_,"\n";
+               
 
+    next LINE if !(my($tai,$log) = split(' ',$_,2));
+    #dbg("TAI:".$tai);
+
+    #dbg("REST1:".$log);
+    
     #If date specified, only process lines matching date
     next LINE if ( $tai lt $starttai );
     next LINE if ( $tai gt $endtai );
 
+               #Count lines and skip out if debugging
+               $count++;
+               last LINE if ($opt{debug} && $count >= 100000);
+    #dbg("REST:".$log);
+
+               #Loglines to Saved String for later DB write
+               if ($savedata) {
+                       my $CurrentLine = $_;
+                       $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/;
+                       if ($2 ne $CurrentMailId) {
+                               $CurrentLogId = $1."-".$2;
+                               $CurrentMailId = $2;
+                       }
+                       $LogLines{$CurrentLogId} = $_;
+                       #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
+               }
+
+               #Count lines and skip out if debugging
+               $count++;
+               last LINE if ($opt{debug} && $count >= 100);
+    #dbg("REST:".$log);
+                       
+
     # pull out spamasassin rule lists
-    if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
+    if ( $_ =~m/spamassassin: pass, Ham,(.*)</ )
+    #if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
     {
-     my (@SAtests) = split(',',$2);
-     foreach my $SAtest (@SAtests) {
-       if (!$SAtest eq "") {
-         $found_SARules{$SAtest}{'count'}++;
-         $found_SARules{$SAtest}{'totalhits'} += $1;
-         $sum_SARules++
-       }
-     }
-
+                dbg("SPAM:".$log);
+                
+                
+                #New version does not seem to have spammassasin tests in logs 
+
+                #if (exists($2){  
+                        #my (@SAtests) = split(',',$2);
+                        #foreach my $SAtest (@SAtests) {
+                                #if (!$SAtest eq "") {
+                                        #$found_SARules{$SAtest}{'count'}++;
+                                        #$found_SARules{$SAtest}{'totalhits'} += $1;
+                                        #$sum_SARules++
+                                #}
+                        #}
+                       #}
+                       
     }
-
+    
+    
     #Pull out Geoip countries for analysis table
-    if ( $_ =~m/check_badcountries plugin \(connect\): GeoIP Country: (.*)/ )
+    if ( $_ =~m/check_badcountries: GeoIP Country: (.*)/ )
     {
      $found_countries{$1}++;
      $total_countries++;
     }
+    
+    #Pull out DMARC approvals
+    if ( $_ =~m/.*$DMARCOkPattern.*/ )
+    {
+                       $DMARCOkCount++;
+    }
+    
 
     #only select Logterse output
-    next LINE unless m/terse plugin/;
-
-
+    next LINE unless m/logging::logterse:/;
+               
+               #Count lines and skip out if debugging
+               $count++;
+               last LINE if ($opt{debug} && $count >= 100000);
+    #dbg("REST:".$log);
 
     my $abstime = Time::TAI64::tai2unix($tai);
     my $abshour = floor( $abstime / 3600 );    # Hours since the epoch
@@ -342,6 +473,9 @@
 
     $totalexamined++;
 
+               #dbg("LOG1:".$log_items[1]);
+               #dbg("LOG3:".$log_items[3]);
+               
     # first spot the fetchmail and local deliveries.
 
     # Spot from local workstation
@@ -355,11 +489,9 @@
 
     # see if from localhost
     elsif ( $log_items[1] =~ m/.*$localhost.*/ ) {
-
         # but not if it comes from fetchmail
         if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { }
         else {
-
             # might still be from mailman here
             if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) {
                 $mailmansendcount++;
@@ -368,21 +500,44 @@
                 $localflag = 1;
             }
             else {
-
-                # eliminate incoming localhost spoofs
-                if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
-                else {
-                    $localflag = 1;
-                    $WebMailsendtotal++;
-                    $counts{$abshour}{$CATWEBMAIL}++;
-                    $WebMailflag = 1;
-                }
+                                                               #Or sent to the DMARC server
+                                                               dbg("LOG4:".$log_items[4]);
+                                                               #check for email address in $DMARC_Report_emails string
+                                                               #if ($log_items[4] =~ m/.*$DMARCDomain.*/) {
+                                                               my $logemail = $log_items[4];
+                                                               #print STDERR "/",$log_items[4]."/\n";
+                                                               if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){
+                                                                       $localsendtotal++;
+                                                                       $DMARCSendCount++;
+                                                                       $localflag = 1;
+                                                               }
+                                                               else {
+                                                                       #print STDERR "no match:.".$logemail;
+                                                                       if (exists $log_items[8]){
+                                                                               dbg("LOG8:".$log_items[8]);
+                                                                               # ignore incoming localhost spoofs
+                                                                               if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
+                                                                               else {
+                                                                                               $localflag = 1;
+                                                                                               $WebMailsendtotal++;
+                                                                                               $counts{$abshour}{$CATWEBMAIL}++;
+                                                                                               $WebMailflag = 1;
+                                                                               }
+                                                                       }       
+                                                                       else {
+                                                                               $localflag = 1;
+                                                                               $WebMailsendtotal++;
+                                                                               $counts{$abshour}{$CATWEBMAIL}++;
+                                                                               $WebMailflag = 1;
+                                                                       }
+                                                               }
             }
         }
     }
 
     # try to spot fetchmail emails
     if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) {
+                               dbg("LOG0:".$log_items[0]);
         $localAccepttotal++;
         $counts{$abshour}{$CATFETCHMAIL}++;
     }
@@ -394,10 +549,13 @@
 # and adjust for recipient field if not set-up by denying plugin - extract from deny msg
 
     if ( length( $log_items[4] ) == 0 ) {
+        dbg("LOG7:".$log_items[0]);
         if ( $log_items[5] eq 'check_goodrcptto' ) {
             if ( $log_items[7] gt "invalid recipient" ) {
                 $log_items[4] =
-                  substr( $log_items[7], 18 )    #Leave only email address
+                  substr( $log_items[7], 18 );    #Leave only email address
+                dbg("LOG4:".$log_items[0]);
+
             }
         }
     }
@@ -405,6 +563,7 @@
     #        if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) {
     # reduce to lc and process each e,mail if a list, pseperatedy commas
     my $recipientmail = lc( $log_items[4] );
+    dbg("LOG4:".$log_items[0]);    
     if ( $recipientmail =~ m/.*,/ ) {
 
         #comma - split the line and deal with each domain
@@ -442,7 +601,12 @@
 
    if (exists $log_items[5]) {
 
-    $found_qpcodes{$log_items[5]}++;  ##Count different qpsmtpd result codes
+    if ($log_items[5] eq 'naughty') {
+                       my $rejreason = $log_items[7];
+                       $rejreason = /.*(\(.*\)).*/;
+                       $rejreason = $1;
+                       $found_qpcodes{$log_items[5]."-".$rejreason}++}
+               else {$found_qpcodes{$log_items[5]}++}  ##Count different qpsmtpd result codes
 
     #Check for badly formed lines (from earlier testing)
 
@@ -488,27 +652,31 @@
 
    elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
             # and extract the spam score
-            if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1}
+#            if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") 
+            if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") 
+                                                       {$rejectspamavg += $1}
             mark_domain_rejected($proc);
             next LINE
    }
 
-   elsif ($log_items[5] eq 'virus::clamav') { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
+   elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
             #extract the virus name
-            if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;}
+            if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
+            else {$found_viruses{$log_items[7]}++} #Some other message!!
+            dbg("LOG7:".$log_items[7]);
             mark_domain_rejected($proc);
             next LINE
    }
 
    elsif ($log_items[5] eq 'queued') { $Accepttotal++;
             #extract the spam score
-            if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") {
+            if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") {
               $score = $1;
 #              print $log_items[8]."<".$score.">\n";
               if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score}
               else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score}
             } else {
-              # no SA score - so it must be ham
+              # no SA score - treat it as ham
               $hamcount++;$counts{$abshour}{$CATHAM}++;
             }
             if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
@@ -523,15 +691,53 @@
 
    elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
  
+   elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+ 
+   elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
+ 
+   elsif ($log_items[5] eq 'naughty') {
+                #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify
+                if ($log_items[7] =~ m/(karma)/) {
+                        $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
+                elsif ($log_items[7] =~ m/(dnsbl)/){
+                        $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
+                elsif ($log_items[7] =~ m/(helo)/){
+                               $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+                else {
+                        #Unidentified Naughty rejection
+                        $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE}
+               }
+   elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+
+   elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE}
+
+   elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
+
+   elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE}
+
+   elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+
+    elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+
+   elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+
+   elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+ 
+   elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+
+   elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+
+  elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+
         #Treat it as Unconf if not recognised
         else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE}
+ } #Log[5] exists
+
+
+#   print "Unexpected failure string in log file: ".$log_items[5]."\n";  #Not detected
+#   next LINE
 
-/*
-   print "Unexpected failure string in log file: ".$log_items[5]."\n";  #Not detected
-   next LINE
-*/
 
-   }
 
 }  #END OF MAIN LOOP
 
@@ -624,18 +830,28 @@
 if ( !$disabled ) {
 
     #Output results
+
+    # NEW - save the print to a variable so that it can be processed into html.
+    #
+    #Save current output selection and divert into variable
+    #
+               my $output;
+               my $tablestr="";
+               open(my $outputFH, '>', \$tablestr) or die; # This shouldn't fail
+               my $oldFH = select $outputFH;
+
+
     print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n";
     print "----------------------------------------------------", "\n\n";
 
     print "$0 Version : $opt{'version'}", "\n\n";
-    print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n";
+    print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n";
     print "Period Ending    : ", strftime( "%c", localtime($end) ),   "\n";
     print "\n";
 
-    print "Clam Version : ",         `freshclam -V`;
-    print "SpamAssassin Version : ", `spamassassin -V`;
-    printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,
-      $SARejectLevel;
+    print "Clam Version/DB Count/Last DB update: ",         `freshclam -V`."\n";
+    print "SpamAssassin Version : ", `spamassassin -V`."\n";
+    printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel;
     if ($HighLogLevel) {
       printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n";
       printf "\tYou can set it this way:\n";
@@ -643,10 +859,10 @@
       printf "\tsignal-event email-update\n";
       printf "\tsv t /var/service/qpsmtpd\n\n";
     }
-    print "\n";
+    print "\n\n";
     printf "Reporting Period : %.2f hrs\n", $hrsinperiod;
-    print "----------------------------\n";
-    print "\n";
+    #print "----------------------------\n";
+    #print "\n";
 
     printf "All SMTP connections accepted:%8d          \n", $totalexamined;
 
@@ -655,8 +871,13 @@
     printf "Average spam score (accepted): %11.2f\n", $spamavg       || 0;
     printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0;
     printf "Average ham score            : %11.2f\n", $hamavg        || 0;
-    print "\n";
-    print "Statistics by Hour\n";
+    printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount      || 0;
+    if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
+        
+    print "\n\n";
+    print "\nStatistics by Hour\n";
+    print "-------------------\n";
+    #print "\n";
 
     #
     # start by working out which colunns to show - tag the display array
@@ -682,13 +903,13 @@
 
 
     # and put together the print lines
-    #
+        
     my $Line1;             #Full Line across the page
     my $Line2;             #Broken Line across the page
     my $Titles;            #Column headers
     my $Values;            #Values
     my $Totals;            #Corresponding totals
-    my $Percent;        # and column percentages
+    my $Percent;           # and column percentages
 
     my $hour = floor( $start / 3600 );
     $Line1 = '';
@@ -706,7 +927,7 @@
                   $Line1 .= substr('---------------------',0,$colwidth[$ncateg]);
                   $Line2 .= substr('---------------------',0,$colwidth[$ncateg]-1);
                   $Line2 .= " ";
-                  $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])." ";
+                  $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])."|";
                   if ($ncateg == 0) {
                     $Totals .= substr('TOTALS                                   ',0,$colwidth[$ncateg]-2);
                     $Percent .= substr('PERCENTAGES                              ',0,$colwidth[$ncateg]-1);
@@ -744,39 +965,58 @@
         $hour++;
     }
 
-      # print it.
-    print $Line1."\n";
+    #
+    # print it.
+    #
+               my $makeHTMLemail = "no";
+    #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
+               my $makeHTMLpage = "no";
+               if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
+    #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
+
+    if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";}   #These lines mess up the HTML conversion ....
     print $Titles."\n";
-    print $Line2."\n";
+    if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";}   #ditto
+    #$Line2 =~ s/-/a/g;
+    #print $Line2."\n";
+    #print "\n";
     print $Values."\n";
     print $Line2."\n";
     print $Totals."\n";
     print $Percent."\n";
     print $Line1."\n";
-
+    print "\n";
 
     if ($localAccepttotal>0) {
       print "*Fetchml* means connections from Fetchmail delivering email\n";
     }
-    print "*Local* means connections from workstations on local LAN.\n";
+    print "*Local* means connections from workstations on local LAN.\n\n";
     print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol";
-    print "  or email was to non existant address.\n";
+    print "  or email was to non existant address.\n\n";
+
+   if ($finaldisplay[$KarmaCateg]){
+                       print "*Karma* means email was rejected based on the mailserver's previous activities.\n\n";
+   }
+
 
    if ($finaldisplay[$BadCountryCateg]){
         $BadCountries  = $cdb->get('qpsmtpd')->prop('BadCountries') || "*none*";
-                               print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n";
+                               print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n\n";
    }
    
+   
+   
    if (scalar keys %unrecog_plugin > 0){
                 #Show unrecog plugins found
                 print "*Unrecognised plugins found - categorised as Non-Conf\n";
                 foreach my $unrec (keys %unrecog_plugin){
                         print "\t$unrec\t($unrecog_plugin{$unrec})\n";
-               }
+                       }
+                       print "\n";
                }
 
     if ($QueryNoLogTerse) {
-      print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n";
+      print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n\n";
 #      print " to enable it follow the instructions at .............................\n";
     }
 
@@ -813,7 +1053,7 @@
 #    if ($Webmailsendtotal > 0) {print "If you have the mailman contrib installed, then the webmail totals might include some mailman emails\n"}
 
     # time to do a 'by recipient domain' report
-    print "\nIncoming mails by recipient domains usage\n";
+    print "Incoming mails by recipient domains usage\n";
     print "-----------------------------------------\n";
     print
         "Domains                      Type       Total  Denied XferErr Accept \%accept\n";
@@ -869,25 +1109,6 @@
         show_virus_variants();
     }
 
-    # get enable/disable subsections
-    my $enableqpsmtpdcodes;
-    my $enableSARules;
-    my $enableGeoiptable;
-    my $enablejunkMailList;
-    my $savedata;
-    if ($cdb->get('mailstats')){
-        $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
-        $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
-        $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false;
-        $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false;
-        $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false;
-      } else {
-        $enableqpsmtpdcodes = $true;
-        $enableSARules = $true;
-        $enablejunkMailList = $true;
-        $enableGeoiptable = $true;
-        $savedata = $false;
-      }
 
     if ($enableqpsmtpdcodes) {show_qpsmtpd_codes();}
 
@@ -905,8 +1126,39 @@
        "config setprop mailstats SaveDataToMySQL yes\n";
       }
 
+               select $oldFH;
+               close $outputFH;
+               if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr}
+               if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){
+                       #Convert text to html and send it
+                       require CGI;
+                       require TextToHTML;
+                       my $cgi = new CGI;
+                       my $text = $tablestr;
+                       print $cgi->header();
+                       my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20);
+                       my $conv = new HTML::TextToHTML();
+                       $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2,
+                                                                               underline_length_tolerance=>1);
+                       my $html="<!DOCTYPE html> <html>\n";
+                       $html .=  "<head><title>Mailstats -".strftime( "%F", localtime($start) )."</title>";
+                       $html .=  "<link rel='stylesheet' type='text/css' href='mailstats.css' /></head>\n";
+                       $html .= "<body>\n";
+                       $html .= $conv->process_chunk($text);
+                       $html .= "</body></html>\n";
+                       if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html}
+                       #And drop it into a file
+                       if ($makeHTMLpage eq "yes") {
+                               my $filename = "mailstats.html";
+                               open(my $fh, '>', $filename) or die "Could not open file '$filename' $!";
+                               print $fh $html;
+                               close $fh;
+                       }
+                       
+               }
 
-    #Close Senmdmail if it was opened
+
+    #Close Sendmail if it was opened
     if ( $opt{'mail'} ) {
         select $oldfh;
         close(SENDMAIL);
@@ -934,17 +1186,17 @@
 
     if ($cdb->get('mailstats'))
     {
-      my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily';
+      my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily'; #"fortnightly"; #"daily";# #; TEMP!!
       if ($interval eq "weekly") {
         $secsininterval = 86400*7;
       } elsif ($interval eq "fortnightly") {
         $secsininterval = 86400*14;
       } elsif ($interval eq "monthly") {
-        $secsininterval = 86400;
+        $secsininterval = 86400*30;
       } elsif ($interval =~m/\d+/) {
         $secsininterval = $interval*3600;
       };
-      my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight';
+      my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight'; 
       my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
                                                 localtime(time);
       if ($base eq "Midnight"){
@@ -954,7 +1206,8 @@
       } elsif ($base =~m/\d+/){
         $sec=0;$min=0;$hour=$base;
       };
-      $time = timelocal($sec,$min,$hour,$mday,$mon,$year)
+      #$mday="17"; #$mday="03"; #$mday="16"; #Temp!!
+      $time = timelocal($sec,$min,$hour,$mday,$mon,$year);
     }
     
     my $start = str2time( $startdate );
@@ -966,7 +1219,8 @@
 
 sub dbg {
     my $msg = shift;
-
+    my $time = scalar localtime;
+               $msg = $time.":".$msg."\n";
     if ( $opt{debug} ) {
         print STDERR $msg;
     }
@@ -1001,9 +1255,10 @@
     }
     my $i = keys %junkcount;
     if ( $i > 0 ) {
-        print("Junk Mails left in folder:\n");
-        print("-------------------------\n");
-        print("Count\tUser\n");
+                               print "\n\n";
+        print("\nJunk Mails left in folder:\n");
+        print("---------------------------\n\n");
+        print("\nCount\tUser\n");
         print("-------------------------\n");
         foreach my $thisuser (
             sort { $junkcount{$b} <=> $junkcount{$a} }
@@ -1033,7 +1288,7 @@
     foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} }
                 keys %found_viruses)
     {
-                       if (index($virus,"Sanesecurity")!=-1){
+                       if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){
                        print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n";
                } else {
                        print "Rejected $found_viruses{$virus}\t$virus\n";
@@ -1061,6 +1316,7 @@
     print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined;
     }
     print("---------------------------------------------\n\n");
+    print "\n\n";
 }
 
 sub show_Geoip_results
@@ -1078,38 +1334,40 @@
     } else {
       $percentthreshold = 0.5;
     }
-    print("Geoip results: (cutoff at $percentthreshold%) \n");
-    print("---------------------------------\n");
-    print("Country\tPercent\tCount\tRejected?\n");
-    print("---------------------------------\n");
-    foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
-                keys %found_countries)
-    {
-       $percent = $found_countries{$country} * 100 / $total_countries
-          if $total_countries;
-       $totalpercent = $totalpercent + $percent;
-       if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
-       if ( $percent >= $percentthreshold ) {
-           print "$country\t"
-             . sprintf( '%4.1f', $percent )
-             . "%\t$found_countries{$country}","\t$reject\n"
-             if $total_countries;
-       }       
-       
-    }
-    print("---------------------------------\n");
-    my ($showtotals);
-    if ($cdb->get('mailstats')){
-        $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); 
-    } else {
-      $showtotals = $true;
-    }
-
-    if ($showtotals){
-                       print "TOTALS\t$totalpercent%\t$total_countries\n";
-                       print("---------------------------------\n\n");
+    if ($total_countries > 0) {
+                       print("Geoip results: (cutoff at $percentthreshold%) \n");
+                       print("---------------------------------\n");
+                       print("Country\tPercent\tCount\tRejected?\n");
+                       print("---------------------------------\n");
+                       foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
+                                                                       keys %found_countries)
+                       {
+                                $percent = $found_countries{$country} * 100 / $total_countries
+                                               if $total_countries;
+                                $totalpercent = $totalpercent + $percent;
+                                if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
+                                if ( $percent >= $percentthreshold ) {
+                                                print "$country\t"
+                                                        . sprintf( '%4.1f', $percent )
+                                                        . "%\t$found_countries{$country}","\t$reject\n"
+                                                        if $total_countries;
+                                }      
+                                
+                       }
+                       print("---------------------------------\n");
+                       my ($showtotals);
+                       if ($cdb->get('mailstats')){
+                                       $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); 
+                       } else {
+                               $showtotals = $true;
+                       }
+
+                       if ($showtotals){
+                               print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n";
+                               print("---------------------------------\n\n");
+                       }
+                       print "\n";
                }
-               print "\n";
 }
 
 sub show_SARules_codes
@@ -1123,52 +1381,55 @@
     my ($percentthreshold);
     my ($defaultpercentthreshold);
     my ($totalpercent) = 0;
-
-    if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
-      $defaultpercentthreshold = $maxcutoff
-    } else {
-      $defaultpercentthreshold = $mincutoff
-    }
-    if ($cdb->get('mailstats')){
-        $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold;
-      } else {
-        $percentthreshold = $defaultpercentthreshold
-      }
-      
-    print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
-    print("---------------------------------------------\n");
-    print("Count\tPercent\tScore\t\t\n");
-    print("---------------------------------------------\n");
-    foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
-                keys %found_SARules)
-    {
-                       my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
-                               if $totalexamined;
-                       #$totalpercent = $totalpercent + $percent;
-                       my $avehits = $found_SARules{$SARule}{'totalhits'} /
-                       $found_SARules{$SARule}{'count'}
-                               if $found_SARules{$SARule}{'count'};
-                       if ( $percent >= $percentthreshold ) {
-                               print "$found_SARules{$SARule}{'count'}\t"
-                               . sprintf( '%4.1f', $percent ) . "%\t"
-                               . sprintf( '%4.1f', $avehits )
-                               . "\t$SARule\n"
+               
+               if ($sum_SARules > 0){
+                       
+                       if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
+                               $defaultpercentthreshold = $maxcutoff
+                       } else {
+                               $defaultpercentthreshold = $mincutoff
+                       }
+                       if ($cdb->get('mailstats')){
+                                       $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold;
+                               } else {
+                                       $percentthreshold = $defaultpercentthreshold
+                               }
+                               
+                       print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
+                       print("---------------------------------------------\n");
+                       print("Count\tPercent\tScore\t\t\n");
+                       print("---------------------------------------------\n");
+                       foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
+                                                                       keys %found_SARules)
+                       {
+                               my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
                                        if $totalexamined;
-}
-    }
-    print("---------------------------------------------\n");
-    my ($showtotals);
-    if ($cdb->get('mailstats')){
-        $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); 
-    } else {
-      $showtotals = $true;
-    }
-
-    if ($showtotals){
-                       print "$totalexamined\t(TOTALS)\n";
+                               #$totalpercent = $totalpercent + $percent;
+                               my $avehits = $found_SARules{$SARule}{'totalhits'} /
+                               $found_SARules{$SARule}{'count'}
+                                       if $found_SARules{$SARule}{'count'};
+                               if ( $percent >= $percentthreshold ) {
+                                       print "$found_SARules{$SARule}{'count'}\t"
+                                       . sprintf( '%4.1f', $percent ) . "%\t"
+                                       . sprintf( '%4.1f', $avehits )
+                                       . "\t$SARule\n"
+                                               if $totalexamined;
+       }
+                       }
                        print("---------------------------------------------\n");
+                       my ($showtotals);
+                       if ($cdb->get('mailstats')){
+                                       $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); 
+                       } else {
+                               $showtotals = $true;
+                       }
+
+                       if ($showtotals){
+                               print "$totalexamined\t(TOTALS)\n";
+                               print("---------------------------------------------\n");
+                       }
+                       print "\n";
                }
-               print "\n";
 
 
 }
@@ -1370,9 +1631,15 @@
            }
        $nhour++;
        }
-    $dbh->disconnect();
-    my $telapsed = time - $tstart;
-    print "Saved $reccount records in $telapsed sec.";
+       # and write out the log lines saved
+       
+       foreach my $logid (keys %LogLines){
+         
+               $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')");
+       }
+       $dbh->disconnect();
+       my $telapsed = time - $tstart;
+       print "Saved $reccount records in $telapsed sec.";
 }
 
 sub check_date_rec
@@ -1439,5 +1706,3 @@
     my $daterec = $sth->fetchrow_hashref();
     $daterec->{"dateid"};
  }
-
-