smeserver-mailstats/contribs9/smeserver-mailstats-1.1.bz9717-9716email_specific_stats.patch

diff -urN smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl
--- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl      2016-10-16 14:40:37.689999942 +0100
+++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl  2016-10-12 14:26:59.000000000 +0100
@@ -20,6 +20,10 @@
 # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection
 # bjr - 14Jun16 - make compatible with qpsmtpd 0.96
 # bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7)
+# bjr - 04Aug16 - Add code to log and count the blacklist RBL urls that have triggered, this (NFR) is Bugzilla 9717
+# bjr - 04Aug16 - Add code to expand the junkmail table to include daily ham and spam and deleted spam for each user - (NFR bugzilla 9716)
+# bjr - 05Aug16 - Add code to log remote relay incoming emails
+# bjr - 10Oct16 - Add code to show stats for the smeoptimizer package
 #
 #############################################################################
 #
@@ -45,6 +49,15 @@
 #                                               / HTMLPage - "yes"  / "no"  - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no"
 #
 #############################################################################
+#
+#
+#  TODO
+#
+# 1. Delete loglines records from any previous run of same table
+# 2. Add tracking LogId for each cont in the table
+# 3. Use link directory file to generate h1 / h2 tags for title and section headings
+# 4. Ditto for links to underlying data
+#
 
 # internal modules (part of core perl distribution)
 use strict;
@@ -60,6 +73,7 @@
 use Sys::Hostname;
 use Switch;
 use DBIx::Simple;
+use URI::URL;
 
 #use CGI;
 #use HTML::TextToHTML;
@@ -81,7 +95,7 @@
 
 #Configuration section
 my %opt = (
-    version => '0.7.5a',                         # please update at each change.
+    version => '0.7.10',                         # please update at each change.
     debug => 0,                                 # guess what ?
     sendmail => '/usr/sbin/sendmail',           # Path to sendmail stub
     from => 'spamfilter-stats',                 # Who is the mail from
@@ -96,7 +110,7 @@
 my $MAILMAN = "bounces";        #sender when mailman sending when orig is localhost
 my $DMARCDomain="dmarc";                               #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
 my $DMARCOkPattern="dmarc: pass";  #Pattern to use to detect DMARC approval
-
+my $localIPregexp = ".*((127\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(192\.168\.)).*";
 my $MinCol = 6;                 #Minimum column width
 my $HourColWidth = 16;            #Date and time column width
 
@@ -135,10 +149,12 @@
 my %found_SARules = ();
 my %junkcount = ();
 my %unrecog_plugin = ();
+my %blacklistURL = ();  #Count of use of each balcklist rhsbl
+my %usercounts = ();    #Count per received email of sucessful delivery, queued spam and deleted Spam, and rejected
 
 # replaced by...
-my %counts = ();                       #Hold all counts in 2-D matrix
-my @display = ();                    #used to switch on and off columns  - yes, no or auto for each category
+my %counts = ();                    #Hold all counts in 2-D matrix
+my @display = ();                   #used to switch on and off columns  - yes, no or auto for each category
 my @colwidth = ();                  #width of each column
                                     #(auto means only if non zero) - populated from possible db entries
 my @finaldisplay = ();              #final decision on display or not  - true or false
@@ -149,8 +165,9 @@
 my $CATWEBMAIL='WebMail';
 my $CATMAILMAN='Mailman';
 my $CATLOCAL='Local';
+my $CATRELAY="Relay";
 # border between where it came from and where it ended..
-my $countfromhere = 5;  #Temp - Check this not moved!!
+my $countfromhere = 6;  #Temp - Check this not moved!!
  
 my $CATVIRUS='Virus';
 my $CATRBLDNS='RBL/DNS';
@@ -166,7 +183,7 @@
 my $CATPERCENT='PERCENT';
 my $CATDMARC="DMARC Rej.";
 my $CATLOAD="Rej.Load";
-my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
+my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATRELAY,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
 my $GRANDTOTAL = '99';                #subs for count arrays, for grand total
 my $PERCENT = '98';                 # for column percentages
 
@@ -238,10 +255,21 @@
 
 #dbg("DMARC-EMAILS:".$DMARC_Report_emails); 
 
+# and setup list of local domains for spotting the local one in a list of email addresses (Remote station processing)
+use esmith::DomainsDB;
+my $d = esmith::DomainsDB->open_ro();
+my @domains = $d->keys();
+my $alldomains = "(";
+foreach my $dom (@domains){$alldomains .= $dom."|"}
+$alldomains .= ")";
+#print $alldomains;
+
 # Saving the Log lines processed
 my %LogLines = ();  #Save all the log lines processed for writing to the DB
 my %LogId = ();     #Save the Log Ids.
 my $CurrentLogId = ""; 
+my $Sequence = 0;
+
 
 # store the domain of interest. Every other records are stored in a 'Other' zone
 my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n";
@@ -306,6 +334,7 @@
 my $enableGeoiptable;
 my $enablejunkMailList;
 my $savedata;
+my $enableblacklist;  #Enabled according to setting in qpsmtpd
 if ($cdb->get('mailstats')){
                $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
                $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
@@ -319,12 +348,15 @@
                $enableGeoiptable = $true;
                $savedata = $false;
        }
-       $savedata = $false;  #TEMP!!
-
-#
-#---------------------------------------
-# Scan the qpsmtpd log file(s)
-#---------------------------------------
+       $enableblacklist = ($cdb->get('qpsmtpd')->prop("RHSBL") || "disabled") eq "enabled" || ($cdb->get('qpsmtpd')->prop("URIBL") || "disabled") eq "enabled";
+       #$savedata = $false;  #TEMP!!
+#if ($savedata){print STDERR "yes"} else {print STDERR "no"}
+
+my $makeHTMLemail = "no";
+#if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
+my $makeHTMLpage = "no";
+#if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
+#if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
 
 
 # Init the hashes
@@ -373,6 +405,12 @@
 
 my $count = -1; #for loop reduction in debugging mode
 
+#
+#---------------------------------------
+# Scan the qpsmtpd log file(s)
+#---------------------------------------
+
+
 my $CurrentMailId = "";
 
 LINE: while (<>) {
@@ -391,36 +429,38 @@
 
                #Count lines and skip out if debugging
                $count++;
-               last LINE if ($opt{debug} && $count >= 100000);
+               #last LINE if ($opt{debug} && $count >= 100);
     #dbg("REST:".$log);
+                       
 
                #Loglines to Saved String for later DB write
                if ($savedata) {
                        my $CurrentLine = $_;
                        $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/;
-                       if ($2 ne $CurrentMailId) {
-                               $CurrentLogId = $1."-".$2;
-                               $CurrentMailId = $2;
+                       my $l = length($CurrentLine);
+                       if ($l != 0){
+                               if (defined($2)){               #print STDERR "Undefined \$2:".$_.":".$count.":".$l;exit}
+                                       if ($2 ne $CurrentMailId) {
+                                               print "CL:$CurrentLine*\n" if !defined($1);
+                                               $CurrentLogId = $1."-".$2;
+                                               $CurrentMailId = $2;
+                                               $Sequence = 0;
+                                       } else {$Sequence++}
+                                       #$CurrentLogId .=":".$Sequence;
+                                       $LogLines{$CurrentLogId.":".$Sequence} = $_;
+                               }
                        }
-                       $LogLines{$CurrentLogId} = $_;
-                       #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
+                       #print STDERR $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
+                       #exit
                }
 
-               #Count lines and skip out if debugging
-               $count++;
-               last LINE if ($opt{debug} && $count >= 100);
-    #dbg("REST:".$log);
-                       
 
     # pull out spamasassin rule lists
     if ( $_ =~m/spamassassin: pass, Ham,(.*)</ )
     #if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
     {
-                dbg("SPAM:".$log);
-                
-                
+                #dbg("SPAM:".$log);
                 #New version does not seem to have spammassasin tests in logs 
-
                 #if (exists($2){  
                         #my (@SAtests) = split(',',$2);
                         #foreach my $SAtest (@SAtests) {
@@ -451,11 +491,6 @@
 
     #only select Logterse output
     next LINE unless m/logging::logterse:/;
-               
-               #Count lines and skip out if debugging
-               $count++;
-               last LINE if ($opt{debug} && $count >= 100000);
-    #dbg("REST:".$log);
 
     my $abstime = Time::TAI64::tai2unix($tai);
     my $abshour = floor( $abstime / 3600 );    # Hours since the epoch
@@ -465,35 +500,89 @@
     my (@log_items) = split $FS, $log_part;
 
     my (@timestamp_items) = split(' ',$timestamp_part);
-
+    
+               my $result= "rejected";  #Tag as rejected unti we know otherwise
     # we store the more recent recipient domain, for domain statistics
     # in fact, we only store the first recipient. Could be sort of headhache
     # to obtain precise stats with many recipients on more than one domain !
     my $proc = $timestamp_items[1] ;  #numeric Id for the email
+    my $emailnum = $proc; #proc gets modified later...
 
+    if ($emailnum == 23244) {
+                       dbg("TM0:".$timestamp_items[0]);
+                       dbg("TM1:".$timestamp_items[1]);
+                       dbg("TM2:".$timestamp_items[2]);
+                       dbg("TM3:".$timestamp_items[3]);
+                       dbg("LOG0:".$log_items[0]);
+                       dbg("LOG1:".$log_items[1]);
+                       dbg("LOG2:".$log_items[2]);
+                       dbg("LOG3:".$log_items[3]);
+                       dbg("LOG4:".$log_items[4]);
+                       dbg("LOG5:".$log_items[5]);
+                       dbg("LOG6:".$log_items[6]);
+                       dbg("LOG7:".$log_items[7]);
+                       dbg("IPregexp:".$localIPregexp);
+                       if (!test_for_private_ip($log_items[0])) {dbg("Log0 not found");}
+                       if (test_for_private_ip($log_items[2])){ dbg("Log2 match")} 
+                       if ($log_items[5] eq "queued") {dbg("LOG5 match")}
+               }
+               
     $totalexamined++;
 
-               #dbg("LOG1:".$log_items[1]);
-               #dbg("LOG3:".$log_items[3]);
+#              dbg("LOG8:".$log_items[8]);
                
     # first spot the fetchmail and local deliveries.
 
     # Spot from local workstation
     $localflag   = 0;
     $WebMailflag = 0;
-    if ( $log_items[1] =~ m/.*$DomainName.*/ ) {
+    if ( $log_items[1] =~ m/$DomainName/ ) {  #bjr
+                               #dbg("LOG1-Found:".$log_items[1]);
         $localsendtotal++;
         $counts{$abshour}{$CATLOCAL}++;
         $localflag = 1;
     }
+    
+    #Or a remote station    
+    elsif ((!test_for_private_ip($log_items[0])) and (test_for_private_ip($log_items[2])) and ($log_items[5] eq "queued"))
+    {
+                       #Remote user
+                       $localflag = 1;
+                       $counts{$abshour}{$CATRELAY}++;
+               }
+
+               elsif (($log_items[2] =~ m/$WebmailIP/) and (!test_for_private_ip($log_items[0]))) {
+                       #Webmail
+#                      if ($emailnum == 19608){
+                               dbg("WEBMAIL:");
+                               dbg("TM0:".$timestamp_items[0]);
+                               dbg("TM1:".$timestamp_items[1]);
+                               dbg("TM2:".$timestamp_items[2]);
+                               dbg("TM3:".$timestamp_items[3]);
+                               dbg("LOG0:".$log_items[0]);
+                               dbg("LOG1:".$log_items[1]);
+                               dbg("LOG2:".$log_items[2]);
+                               dbg("LOG3:".$log_items[3]);
+                               dbg("LOG4:".$log_items[4]);
+                               dbg("LOG5:".$log_items[5]);
+                               dbg("LOG6:".$log_items[6]);
+                               dbg("LOG7:".$log_items[7]);
+                               #exit;
+#                      }
+                       $localflag = 1;
+                       $WebMailsendtotal++;
+                       $counts{$abshour}{$CATWEBMAIL}++;
+                       $WebMailflag = 1;
+               }
 
     # see if from localhost
-    elsif ( $log_items[1] =~ m/.*$localhost.*/ ) {
+    elsif ( $log_items[1] =~ m/$localhost/ ) {
         # but not if it comes from fetchmail
-        if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { }
+        if ( $log_items[3] =~ m/$FETCHMAIL/ ) { }
         else {
+                                               $localflag = 1;
             # might still be from mailman here
-            if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) {
+            if ( $log_items[3] =~ m/$MAILMAN/ ) {
                 $mailmansendcount++;
                 $localsendtotal++;
                 $counts{$abshour}{$CATMAILMAN}++;
@@ -501,12 +590,10 @@
             }
             else {
                                                                #Or sent to the DMARC server
-                                                               dbg("LOG4:".$log_items[4]);
+                                                               #dbg("LOG4:".$log_items[4]);
                                                                #check for email address in $DMARC_Report_emails string
-                                                               #if ($log_items[4] =~ m/.*$DMARCDomain.*/) {
                                                                my $logemail = $log_items[4];
-                                                               #print STDERR "/",$log_items[4]."/\n";
-                                                               if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){
+                                                               if ((index($DMARC_Report_emails,$logemail)>=0) or ($logemail =~ m/$DMARCDomain/)){
                                                                        $localsendtotal++;
                                                                        $DMARCSendCount++;
                                                                        $localflag = 1;
@@ -514,10 +601,11 @@
                                                                else {
                                                                        #print STDERR "no match:.".$logemail;
                                                                        if (exists $log_items[8]){
-                                                                               dbg("LOG8:".$log_items[8]);
+                                                                               #dbg("LOG8:".$log_items[8]);
                                                                                # ignore incoming localhost spoofs
-                                                                               if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
+                                                                               if ( $log_items[8] =~ m/msg denied before queued/ ) { }
                                                                                else {
+                                                                                               #Webmail
                                                                                                $localflag = 1;
                                                                                                $WebMailsendtotal++;
                                                                                                $counts{$abshour}{$CATWEBMAIL}++;
@@ -536,12 +624,12 @@
     }
 
     # try to spot fetchmail emails
-    if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) {
-                               dbg("LOG0:".$log_items[0]);
+    if ( $log_items[0] =~ m/$FetchmailIP/ ) {
+                               #dbg("LOG0:".$log_items[0]);
         $localAccepttotal++;
         $counts{$abshour}{$CATFETCHMAIL}++;
     }
-    elsif ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) {
+    elsif ( $log_items[3] =~ m/$FETCHMAIL/ ) {
         $localAccepttotal++;
         $counts{$abshour}{$CATFETCHMAIL}++;
     }
@@ -549,12 +637,12 @@
 # and adjust for recipient field if not set-up by denying plugin - extract from deny msg
 
     if ( length( $log_items[4] ) == 0 ) {
-        dbg("LOG7:".$log_items[0]);
+        #dbg("LOG7:".$log_items[0]);
         if ( $log_items[5] eq 'check_goodrcptto' ) {
             if ( $log_items[7] gt "invalid recipient" ) {
                 $log_items[4] =
                   substr( $log_items[7], 18 );    #Leave only email address
-                dbg("LOG4:".$log_items[0]);
+                #dbg("LOG4:".$log_items[0]);
 
             }
         }
@@ -563,7 +651,7 @@
     #        if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) {
     # reduce to lc and process each e,mail if a list, pseperatedy commas
     my $recipientmail = lc( $log_items[4] );
-    dbg("LOG4:".$log_items[0]);    
+    #dbg("LOG4:".$log_items[0]);    
     if ( $recipientmail =~ m/.*,/ ) {
 
         #comma - split the line and deal with each domain
@@ -599,146 +687,179 @@
    # then categorise the result
 
 
-   if (exists $log_items[5]) {
+       if (exists $log_items[5]) {
 
-    if ($log_items[5] eq 'naughty') {
+               if ($log_items[5] eq 'naughty') {
                        my $rejreason = $log_items[7];
                        $rejreason = /.*(\(.*\)).*/;
-                       $rejreason = $1;
+                       if (!defined($1)){$rejreason = "unknown"}
+                       else {$rejreason = $1}
                        $found_qpcodes{$log_items[5]."-".$rejreason}++}
                else {$found_qpcodes{$log_items[5]}++}  ##Count different qpsmtpd result codes
 
-    #Check for badly formed lines (from earlier testing)
-
-   if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
 
-   elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
 
-   elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one.
+               elsif ($log_items[5] eq 'tnef2mime') { } #Not expecting this one.
 
-   elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
-            # and extract the spam score
-#            if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") 
-            if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") 
+               elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
+                                               # and extract the spam score
+               #            if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") 
+                                               if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)") 
                                                        {$rejectspamavg += $1}
-            mark_domain_rejected($proc);
-            next LINE
-   }
+                                               mark_domain_rejected($proc);
+               }
 
-   elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
-            #extract the virus name
-            if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
-            else {$found_viruses{$log_items[7]}++} #Some other message!!
-            dbg("LOG7:".$log_items[7]);
-            mark_domain_rejected($proc);
-            next LINE
-   }
+               elsif (($log_items[5] eq 'virus::clamav') or ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
+                                               #extract the virus name
+                                               if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
+                                               else {$found_viruses{$log_items[7]}++} #Some other message!!
+                                               #dbg("LOG7:".$log_items[7]);
+                                               mark_domain_rejected($proc);
+               }
 
-   elsif ($log_items[5] eq 'queued') { $Accepttotal++;
-            #extract the spam score
-            if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") {
-              $score = $1;
-#              print $log_items[8]."<".$score.">\n";
-              if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score}
-              else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score}
-            } else {
-              # no SA score - treat it as ham
-              $hamcount++;$counts{$abshour}{$CATHAM}++;
-            }
-            if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
-                    $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
-                    $currentrcptdomain{ $proc } = '' ;
-            }
-            next LINE
-   }
-   
+               elsif ($log_items[5] eq 'queued') { $Accepttotal++;
+                                               #extract the spam score
+                                               # Remove count for rejectred as it looks as if it might get through!!
+                                               $result= "queued";
+                                               if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") {
+                                                       $score = trim($1);
+                                                       if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric 
+                                                       {
+                                                               if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;}
+                                                               else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";}
+                                                       } else {
+                                                               print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n";
+                                                       }
+                                               } else {
+                                                       # no SA score - treat it as ham
+                                                       $hamcount++;$counts{$abshour}{$CATHAM}++;
+                                               }
+                                               if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
+                                                                               $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
+                                                                               $currentrcptdomain{ $proc } = '' ;
+                                               }
+               }
 
-   elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
 
-   elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
- 
-   elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
- 
-   elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
- 
-   elsif ($log_items[5] eq 'naughty') {
+               elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
+
+               elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
+
+               elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
+
+               elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
+
+               elsif ($log_items[5] eq 'naughty') {
                 #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify
                 if ($log_items[7] =~ m/(karma)/) {
-                        $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
+                        $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)}
                 elsif ($log_items[7] =~ m/(dnsbl)/){
-                        $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
+                        $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
                 elsif ($log_items[7] =~ m/(helo)/){
-                               $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+                               $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
                 else {
                         #Unidentified Naughty rejection
-                        $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE}
+                        $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++}
                }
-   elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-    elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
- 
-   elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-   elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
-  elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
+               elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
+
+               elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
 
         #Treat it as Unconf if not recognised
-        else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE}
+        else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++}
  } #Log[5] exists
 
-
-#   print "Unexpected failure string in log file: ".$log_items[5]."\n";  #Not detected
-#   next LINE
-
-
-
+               #Entry if not local send
+               if ($localflag == 0) {
+                       if (length($log_items[4]) > 0){
+                               # Need to check here for multiple email addresses
+                               my @emails = split(",",lc($log_items[4]));
+                               if (scalar(@emails) > 1) {
+                                       #Just pick the first local address to hang it on.
+                                       # TEMP - just go for the first address until I can work out how to spot the 1st "local" one
+                                       $usercounts{$emails[0]}{$result}++;
+                                       $usercounts{$emails[0]}{"proc"} = $proc;
+                                       #Compare with @domains array until we get a local one
+                                       my $gotone = $false;
+                                       foreach my $email (@emails){
+                                               #Extract the domain from the email address
+                                               my $fullemail = $email;
+                                               $email = s/.*\@(.*)$/$1/;
+                                               #and see if it is local
+                                               if ($email =~ m/$alldomains/){
+                                                       $usercounts{lc($fullemail)}{$result}++;
+                                                       $usercounts{lc($fullemail)}{"proc"} = $proc;
+                                                       $gotone = $true;
+                                                       last;
+                                               }
+                                       }
+                                       if (!$gotone) {
+                                                       $usercounts{'No internal email $proc'}{$result}++;
+                                                       $usercounts{'No internal email $proc'}{"proc"} = $proc;
+                                       }
+                                               
+                               } else {
+                                       $usercounts{lc($log_items[4])}{$result}++;
+                                       $usercounts{lc($log_items[4])}{"proc"} = $proc;
+                               }
+                       }
+               }
+   #exit if $emailnum == 15858;
+   
 }  #END OF MAIN LOOP
 
 #total up grand total Columns
@@ -750,7 +871,7 @@
            $counts{$GRANDTOTAL}{$categs[$ncateg]} += $counts{$nhour}{$categs[$ncateg]};
 
       # and total rows
-      if ( $ncateg < $categlen && $ncateg>=$countfromhere) {#skip initial columns of non final reasons
+      if ( $ncateg < $categlen and $ncateg>=$countfromhere) {#skip initial columns of non final reasons
         $counts{$nhour}{$categs[@categs-2]} += $counts{$nhour}{$categs[$ncateg]};
       }
       $ncateg++
@@ -815,7 +936,7 @@
 my $oldfh;
 
 #Open Sendmail if we are mailing it
-if ( $opt{'mail'} && !$disabled ) {
+if ( $opt{'mail'} and !$disabled ) {
     open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" )
       or die "Can't open sendmail: $!\n";
     print SENDMAIL "From: $opt{'from'}\n";
@@ -841,44 +962,44 @@
                my $oldFH = select $outputFH;
 
 
-    print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n";
-    print "----------------------------------------------------", "\n\n";
-
-    print "$0 Version : $opt{'version'}", "\n\n";
-    print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n";
+    print "SMEServer daily Anti-Virus and Spamfilter statistics from $hostname - ".strftime( "%F", localtime($start))."\n";
+    print "----------------------------------------------------------------------------------", "\n\n";
+    print "$0 Version : $opt{'version'}", "\n";
+    print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n";
     print "Period Ending    : ", strftime( "%c", localtime($end) ),   "\n";
-    print "\n";
-
-    print "Clam Version/DB Count/Last DB update: ",         `freshclam -V`."\n";
-    print "SpamAssassin Version : ", `spamassassin -V`."\n";
-    printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel;
+    print "Clam Version/DB Count/Last DB update: ",`freshclam -V`;
+    print "SpamAssassin Version : ",`spamassassin -V`;
+    printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,$SARejectLevel;
     if ($HighLogLevel) {
       printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n";
       printf "\tYou can set it this way:\n";
       printf "\tconfig setprop qpsmtpd LogLevel 6\n";
       printf "\tsignal-event email-update\n";
-      printf "\tsv t /var/service/qpsmtpd\n\n";
+      printf "\tsv t /var/service/qpsmtpd\n";
     }
-    print "\n\n";
     printf "Reporting Period : %.2f hrs\n", $hrsinperiod;
-    #print "----------------------------\n";
-    #print "\n";
-
     printf "All SMTP connections accepted:%8d          \n", $totalexamined;
-
     printf "Emails per hour              : %8.1f/hr\n", $emailperhour || 0;
-    print "\n";
     printf "Average spam score (accepted): %11.2f\n", $spamavg       || 0;
     printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0;
     printf "Average ham score            : %11.2f\n", $hamavg        || 0;
-    printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount      || 0;
-    if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
+    printf "Number of DMARC reporting emails sent:\t%11d (not shown on table)\n", $DMARCSendCount      || 0;
+    if ($hamcount != 0){ printf "Number of emails approved through DMARC:\t%11d (%3d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
+    
+    my $smeoptimizerprog = "/usr/local/smeoptimizer/SMEOptimizer.pl";
+    if (-e $smeoptimizerprog) { 
+                       #smeoptimizer installed - get result of status
+                       my @smeoptimizerlines = split(/\n/,`/usr/local/smeoptimizer/SMEOptimizer.pl -status`);
+                       print("SMEOptimizer status:\n");
+                       print("\t".$smeoptimizerlines[6]."\n");
+                       print("\t".$smeoptimizerlines[7]."\n");
+                       print("\t".$smeoptimizerlines[8]."\n");
+                       print("\t".$smeoptimizerlines[9]."\n");
+                       print("\t".$smeoptimizerlines[10]."\n");
+               }
+                       
         
-    print "\n\n";
-    print "\nStatistics by Hour\n";
-    print "-------------------\n";
-    #print "\n";
-
+    print "\nStatistics by Hour:\n";
     #
     # start by working out which colunns to show - tag the display array
     #
@@ -968,24 +1089,17 @@
     #
     # print it.
     #
-               my $makeHTMLemail = "no";
-    #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
-               my $makeHTMLpage = "no";
-               if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
-    #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
-
-    if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";}   #These lines mess up the HTML conversion ....
-    print $Titles."\n";
-    if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";}   #ditto
-    #$Line2 =~ s/-/a/g;
-    #print $Line2."\n";
-    #print "\n";
-    print $Values."\n";
+
+               print $Line1."\n";
+               #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";}   #These lines mess up the HTML conversion ....
+               print $Titles."\n";
+               #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";}   #ditto
+               print $Line2."\n";
+    print $Values;
     print $Line2."\n";
     print $Totals."\n";
     print $Percent."\n";
     print $Line1."\n";
-    print "\n";
 
     if ($localAccepttotal>0) {
       print "*Fetchml* means connections from Fetchmail delivering email\n";
@@ -1021,7 +1135,7 @@
     }
 
 
-    if ( !$RHSenabled || !$DNSenabled ) {
+    if ( !$RHSenabled or !$DNSenabled ) {
 
         # comment about RBL not set
         print
@@ -1114,11 +1228,15 @@
 
     if ($enableSARules) {show_SARules_codes();}
 
-    if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();}
+    if ($enableGeoiptable and $finaldisplay[$BadCountryCateg]){show_Geoip_results();}
 
     if ($enablejunkMailList) {List_Junkmail();}
+    
+    if ($enableblacklist) {show_blacklist_counts();}
+    
+    show_user_stats();
 
-    print "\nDone. Report generated in $telapsed sec.\n\n";
+    print "\nReport generated in $telapsed sec.\n";
 
     if ($savedata) { save_data(); }
     else
@@ -1128,25 +1246,26 @@
 
                select $oldFH;
                close $outputFH;
-               if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr}
-               if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){
+               if ($makeHTMLemail eq "no" or $makeHTMLemail eq "both") {print $tablestr}
+               if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" or $makeHTMLpage eq "yes"){
                        #Convert text to html and send it
                        require CGI;
                        require TextToHTML;
                        my $cgi = new CGI;
                        my $text = $tablestr;
-                       print $cgi->header();
                        my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20);
                        my $conv = new HTML::TextToHTML();
                        $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2,
                                                                                underline_length_tolerance=>1);
-                       my $html="<!DOCTYPE html> <html>\n";
+
+                       my $html = $cgi->header();
+                       $html .="<!DOCTYPE html> <html>\n";
                        $html .=  "<head><title>Mailstats -".strftime( "%F", localtime($start) )."</title>";
                        $html .=  "<link rel='stylesheet' type='text/css' href='mailstats.css' /></head>\n";
                        $html .= "<body>\n";
                        $html .= $conv->process_chunk($text);
                        $html .= "</body></html>\n";
-                       if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html}
+                       if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" ) {print $html}
                        #And drop it into a file
                        if ($makeHTMLpage eq "yes") {
                                my $filename = "mailstats.html";
@@ -1206,7 +1325,7 @@
       } elsif ($base =~m/\d+/){
         $sec=0;$min=0;$hour=$base;
       };
-      #$mday="17"; #$mday="03"; #$mday="16"; #Temp!!
+      #$mday="05"; #$mday="03"; #$mday="16"; #Temp!!
       $time = timelocal($sec,$min,$hour,$mday,$mon,$year);
     }
     
@@ -1255,10 +1374,9 @@
     }
     my $i = keys %junkcount;
     if ( $i > 0 ) {
-                               print "\n\n";
         print("\nJunk Mails left in folder:\n");
-        print("---------------------------\n\n");
-        print("\nCount\tUser\n");
+        print("---------------------------\n");
+        print("Count\tUser\n");
         print("-------------------------\n");
         foreach my $thisuser (
             sort { $junkcount{$b} <=> $junkcount{$a} }
@@ -1282,20 +1400,20 @@
 #
 
 {
-
-    print("Virus Statistics by name:\n");
-    print("------------------------------------------------------------------------\n");
+               my $line = "------------------------------------------------------------------------\n";
+    print("\nVirus Statistics by name:\n");
+    print($line);
     foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} }
                 keys %found_viruses)
     {
-                       if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){
+                       if (index($virus,"Sanesecurity") !=-1 or index($virus,"UNOFFICIAL") !=-1){
                        print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n";
                } else {
                        print "Rejected $found_viruses{$virus}\t$virus\n";
                }
                        
     }
-    print("------------------------------------------------------------------------\n\n");
+    print($line);
 }
 
 sub show_qpsmtpd_codes
@@ -1305,18 +1423,94 @@
 #
 
 {
-
-    print("Qpsmtpd codes league table:\n");
-    print("---------------------------------------------\n");
-    print("Count\tPercent\tReason\t\n");
-    print("---------------------------------------------\n");
+               my $line = "---------------------------------------------\n";
+    print("\nQpsmtpd codes league table:\n");
+    print($line);
+    print("Count\tPercent\tReason\n");
+    print($line);
     foreach my $qpcode (sort { $found_qpcodes{$b} <=> $found_qpcodes{$a} }
                 keys %found_qpcodes)
     {
-    print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined;
+    print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t\t$qpcode\n" if $totalexamined;
+    }
+    print($line);
+}
+
+sub  trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
+
+sub get_domain 
+{      #my $url= URI->new(shift);
+       #my $domain = $url->host;
+       my $url = shift;
+       $url =~ s!^.*https?://(?:www\.)?!!i;
+       $url =~ s!/.*!!;
+       $url =~ s/[\?\#\:].*//;
+       my $domain = trim($url);
+       return $domain;
+}
+
+sub show_blacklist_counts
+
+#
+# Show a sorted league table of the blacklist URL counts
+#
+
+{
+               my $line = "------------------\n";
+         print("\nBlacklist details:\n");
+    print($line);
+    if ($cdb->get('qpsmtpd')->prop("RHSBL") eq "enabled") {print "RBLLIST:".$cdb->get('qpsmtpd')->prop("RBLList")."\n";}
+    if ($cdb->get('qpsmtpd')->prop("URIBL") eq "enabled") {print "UBLLIST:".$cdb->get('qpsmtpd')->prop("UBLList")."\n";}
+    if (!$cdb->get('qpsmtpd')->prop("SBLList") eq "") {print "SBLLIST:".$cdb->get('qpsmtpd')->prop("SBLList")."\n";}
+    print($line);
+    print("Count\tURL\n");
+    print($line);
+    foreach my $blcode (sort { $blacklistURL{$b} <=> $blacklistURL{$a} }
+                keys %blacklistURL)
+    {
+                       print sprintf('%3u',$blacklistURL{$blcode})."\t$blcode\n";
     }
-    print("---------------------------------------------\n\n");
-    print "\n\n";
+    print($line);
+}
+
+
+sub show_user_stats
+
+#
+# Show a sorted league table of the user counts
+#
+
+{
+               #Compute totals for each entry
+               my $grandtotals=0;
+               my $totalqueued=0;
+               my $totalspam=0;
+               my $totalrejected=0;
+               foreach my $user (keys %usercounts){
+                       $usercounts{$user}{"queued"} = 0 if !(exists $usercounts{$user}{"queued"});
+                       $usercounts{$user}{"rejected"} = 0 if !(exists $usercounts{$user}{"rejected"});
+                       $usercounts{$user}{"spam"} = 0 if !(exists $usercounts{$user}{"spam"});
+                       $usercounts{$user}{"totals"} = $usercounts{$user}{"queued"}+$usercounts{$user}{"rejected"}+$usercounts{$user}{"spam"};
+                       $grandtotals += $usercounts{$user}{"totals"};
+                       $totalspam += $usercounts{$user}{"spam"};
+                       $totalqueued += $usercounts{$user}{"queued"};
+                       $totalrejected += $usercounts{$user}{"rejected"};
+               }
+               my $line = "--------------------------------------------------\n";
+    print("\nStatistics by email address received:\n");
+    print($line);
+    print("Queued\tRejected\tSpam tagged\tEmail Address\n");
+    print($line);
+    foreach my $user (sort { $usercounts{$b}{"totals"} <=> $usercounts{$a}{"totals"} }
+                keys %usercounts)
+    {
+                       print sprintf('%3u',$usercounts{$user}{"queued"})."\t".sprintf('%3u',$usercounts{$user}{"rejected"})."\t\t".sprintf('%3u',$usercounts{$user}{"spam"})."\t\t$user\n";
+    }  
+    print($line);
+               print sprintf('%3u',$totalqueued)."\t".sprintf('%3u',$totalrejected)."\t\t".sprintf('%3u',$totalspam)."\n";
+    print($line);
+               
+    
 }
 
 sub show_Geoip_results
@@ -1335,10 +1529,11 @@
       $percentthreshold = 0.5;
     }
     if ($total_countries > 0) {
-                       print("Geoip results: (cutoff at $percentthreshold%) \n");
-                       print("---------------------------------\n");
+                       my $line = "---------------------------------------------\n";
+                       print("\nGeoip results: (cutoff at $percentthreshold%) \n");
+                       print($line);
                        print("Country\tPercent\tCount\tRejected?\n");
-                       print("---------------------------------\n");
+                       print($line);
                        foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
                                                                        keys %found_countries)
                        {
@@ -1347,14 +1542,14 @@
                                 $totalpercent = $totalpercent + $percent;
                                 if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
                                 if ( $percent >= $percentthreshold ) {
-                                                print "$country\t"
+                                                print "$country\t\t"
                                                         . sprintf( '%4.1f', $percent )
-                                                        . "%\t$found_countries{$country}","\t$reject\n"
+                                                        . "%\t\t$found_countries{$country}","\t$reject\n"
                                                         if $total_countries;
                                 }      
                                 
                        }
-                       print("---------------------------------\n");
+                       print($line);
                        my ($showtotals);
                        if ($cdb->get('mailstats')){
                                        $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); 
@@ -1363,10 +1558,9 @@
                        }
 
                        if ($showtotals){
-                               print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n";
-                               print("---------------------------------\n\n");
+                               print "TOTALS\t\t".sprintf("%4.1f",$totalpercent)."%\t\t$total_countries\n";
+                               print($line);
                        }
-                       print "\n";
                }
 }
 
@@ -1384,7 +1578,7 @@
                
                if ($sum_SARules > 0){
                        
-                       if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
+                       if ($totalexamined >0 and $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
                                $defaultpercentthreshold = $maxcutoff
                        } else {
                                $defaultpercentthreshold = $mincutoff
@@ -1394,17 +1588,15 @@
                                } else {
                                        $percentthreshold = $defaultpercentthreshold
                                }
-                               
-                       print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
-                       print("---------------------------------------------\n");
+                       my $line = "---------------------------------------------\n";
+                       print("\nSpamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
+                       print($line);
                        print("Count\tPercent\tScore\t\t\n");
-                       print("---------------------------------------------\n");
+                       print($line);
                        foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
                                                                        keys %found_SARules)
                        {
-                               my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
-                                       if $totalexamined;
-                               #$totalpercent = $totalpercent + $percent;
+                               my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined if $totalexamined;
                                my $avehits = $found_SARules{$SARule}{'totalhits'} /
                                $found_SARules{$SARule}{'count'}
                                        if $found_SARules{$SARule}{'count'};
@@ -1416,7 +1608,7 @@
                                                if $totalexamined;
        }
                        }
-                       print("---------------------------------------------\n");
+                       print($line);
                        my ($showtotals);
                        if ($cdb->get('mailstats')){
                                        $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); 
@@ -1426,7 +1618,7 @@
 
                        if ($showtotals){
                                print "$totalexamined\t(TOTALS)\n";
-                               print("---------------------------------------------\n");
+                       print($line);
                        }
                        print "\n";
                }
@@ -1631,15 +1823,25 @@
            }
        $nhour++;
        }
-       # and write out the log lines saved
-       
-       foreach my $logid (keys %LogLines){
-         
-               $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')");
+       # and write out the log lines saved - only if html wanted
+       if ($makeHTMLemail eq 'yes' or $makeHTMLemail eq 'both' or $makeHTMLpage eq 'yes'){
+               foreach my $logid (keys %LogLines){
+                       $reccount++; 
+                       #Extract from keys
+                       my $extract = $logid;
+                       $extract =~/^(.*)-(.*):(.*)$/;
+                       my $Log64n = $1;
+                       my $LogMailId = $2;
+                       my $LogSeq = $3;
+                       my $LogLine = $dbh->quote($LogLines{$logid});
+                       my $sql = "INSERT INTO LogData (Log64n,MailID,Sequence,LogStr) VALUES ('";
+                       $sql .= $Log64n."','".$LogMailId."','".$LogSeq."',".$LogLine.")";
+                       $dbh->do($sql) or die($sql);
+               }
+               $dbh->disconnect();
+               $telapsed = time - $tstart;
+               print "Saved $reccount records in $telapsed sec.";
        }
-       $dbh->disconnect();
-       my $telapsed = time - $tstart;
-       print "Saved $reccount records in $telapsed sec.";
 }
 
 sub check_date_rec
@@ -1706,3 +1908,47 @@
     my $daterec = $sth->fetchrow_hashref();
     $daterec->{"dateid"};
  }
+ 
+ sub dump_entries
+ {
+       my $msg = shift;
+       #dbg($msg); 
+       #dbg("TM0:".$timestamp_items[0]);
+       #dbg("TM1:".$timestamp_items[1]);
+       #dbg("TM2:".$timestamp_items[2]);
+       #dbg("TM3:".$timestamp_items[3]);
+       #dbg("LOG0:".$log_items[0]);
+       #dbg("LOG1:".$log_items[1]);
+       #dbg("LOG2:".$log_items[2]);
+       #dbg("LOG3:".$log_items[3]);
+       #dbg("LOG4:".$log_items[4]);
+       #dbg("LOG5:".$log_items[5]);
+       #dbg("LOG6:".$log_items[6]);
+       #dbg("LOG7:".$log_items[7]);
+       #if ($opt{debug} == 1){exit;}
+}
+
+#sub test_for_private_ip {
+       #use NetAddr::IP;
+       #my $ip = shift;
+       #$ip =~ s/^\D*(([0-9]{1,3}\.){3}[0-9]{1,3}).*/$1/e;
+       #print "\nIP:$ip";
+       #my $nip = NetAddr::IP->new($ip);
+       #if ($nip){
+               #if ( $nip->is_rfc1918() ){
+                               #return 1;
+               #} else { return 0} 
+       #} else { return 0}
+#}
+
+
+sub test_for_private_ip {
+     use NetAddr::IP;
+     $_ = shift;
+     return unless /(\d+\.\d+\.\d+\.\d+)/;
+     my $ip = NetAddr::IP->new($1);
+     return unless $ip;
+     return $ip->is_rfc1918();
+}
+
+