/[smecontribs]/rpms/smeserver-mailstats/contribs9/smeserver-mailstats-1.1.bz9717-9716email_specific_stats.patch
ViewVC logotype

Annotation of /rpms/smeserver-mailstats/contribs9/smeserver-mailstats-1.1.bz9717-9716email_specific_stats.patch

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (hide annotations) (download)
Mon Oct 17 10:21:42 2016 UTC (8 years ago) by brianread
Branch: MAIN
CVS Tags: smeserver-mailstats-1_1-12_el6_sme, smeserver-mailstats-1_1-11_el6_sme, smeserver-mailstats-1_1-9_el6_sme, smeserver-mailstats-1_1-10_el6_sme, HEAD
Add patch

1 brianread 1.1 diff -urN smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl
2     --- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl 2016-10-16 14:40:37.689999942 +0100
3     +++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl 2016-10-12 14:26:59.000000000 +0100
4     @@ -20,6 +20,10 @@
5     # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection
6     # bjr - 14Jun16 - make compatible with qpsmtpd 0.96
7     # bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7)
8     +# bjr - 04Aug16 - Add code to log and count the blacklist RBL urls that have triggered, this (NFR) is Bugzilla 9717
9     +# bjr - 04Aug16 - Add code to expand the junkmail table to include daily ham and spam and deleted spam for each user - (NFR bugzilla 9716)
10     +# bjr - 05Aug16 - Add code to log remote relay incoming emails
11     +# bjr - 10Oct16 - Add code to show stats for the smeoptimizer package
12     #
13     #############################################################################
14     #
15     @@ -45,6 +49,15 @@
16     # / HTMLPage - "yes" / "no" - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no"
17     #
18     #############################################################################
19     +#
20     +#
21     +# TODO
22     +#
23     +# 1. Delete loglines records from any previous run of same table
24     +# 2. Add tracking LogId for each cont in the table
25     +# 3. Use link directory file to generate h1 / h2 tags for title and section headings
26     +# 4. Ditto for links to underlying data
27     +#
28    
29     # internal modules (part of core perl distribution)
30     use strict;
31     @@ -60,6 +73,7 @@
32     use Sys::Hostname;
33     use Switch;
34     use DBIx::Simple;
35     +use URI::URL;
36    
37     #use CGI;
38     #use HTML::TextToHTML;
39     @@ -81,7 +95,7 @@
40    
41     #Configuration section
42     my %opt = (
43     - version => '0.7.5a', # please update at each change.
44     + version => '0.7.10', # please update at each change.
45     debug => 0, # guess what ?
46     sendmail => '/usr/sbin/sendmail', # Path to sendmail stub
47     from => 'spamfilter-stats', # Who is the mail from
48     @@ -96,7 +110,7 @@
49     my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost
50     my $DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
51     my $DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval
52     -
53     +my $localIPregexp = ".*((127\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(192\.168\.)).*";
54     my $MinCol = 6; #Minimum column width
55     my $HourColWidth = 16; #Date and time column width
56    
57     @@ -135,10 +149,12 @@
58     my %found_SARules = ();
59     my %junkcount = ();
60     my %unrecog_plugin = ();
61     +my %blacklistURL = (); #Count of use of each balcklist rhsbl
62     +my %usercounts = (); #Count per received email of sucessful delivery, queued spam and deleted Spam, and rejected
63    
64     # replaced by...
65     -my %counts = (); #Hold all counts in 2-D matrix
66     -my @display = (); #used to switch on and off columns - yes, no or auto for each category
67     +my %counts = (); #Hold all counts in 2-D matrix
68     +my @display = (); #used to switch on and off columns - yes, no or auto for each category
69     my @colwidth = (); #width of each column
70     #(auto means only if non zero) - populated from possible db entries
71     my @finaldisplay = (); #final decision on display or not - true or false
72     @@ -149,8 +165,9 @@
73     my $CATWEBMAIL='WebMail';
74     my $CATMAILMAN='Mailman';
75     my $CATLOCAL='Local';
76     +my $CATRELAY="Relay";
77     # border between where it came from and where it ended..
78     -my $countfromhere = 5; #Temp - Check this not moved!!
79     +my $countfromhere = 6; #Temp - Check this not moved!!
80    
81     my $CATVIRUS='Virus';
82     my $CATRBLDNS='RBL/DNS';
83     @@ -166,7 +183,7 @@
84     my $CATPERCENT='PERCENT';
85     my $CATDMARC="DMARC Rej.";
86     my $CATLOAD="Rej.Load";
87     -my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
88     +my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATRELAY,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
89     my $GRANDTOTAL = '99'; #subs for count arrays, for grand total
90     my $PERCENT = '98'; # for column percentages
91    
92     @@ -238,10 +255,21 @@
93    
94     #dbg("DMARC-EMAILS:".$DMARC_Report_emails);
95    
96     +# and setup list of local domains for spotting the local one in a list of email addresses (Remote station processing)
97     +use esmith::DomainsDB;
98     +my $d = esmith::DomainsDB->open_ro();
99     +my @domains = $d->keys();
100     +my $alldomains = "(";
101     +foreach my $dom (@domains){$alldomains .= $dom."|"}
102     +$alldomains .= ")";
103     +#print $alldomains;
104     +
105     # Saving the Log lines processed
106     my %LogLines = (); #Save all the log lines processed for writing to the DB
107     my %LogId = (); #Save the Log Ids.
108     my $CurrentLogId = "";
109     +my $Sequence = 0;
110     +
111    
112     # store the domain of interest. Every other records are stored in a 'Other' zone
113     my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n";
114     @@ -306,6 +334,7 @@
115     my $enableGeoiptable;
116     my $enablejunkMailList;
117     my $savedata;
118     +my $enableblacklist; #Enabled according to setting in qpsmtpd
119     if ($cdb->get('mailstats')){
120     $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
121     $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
122     @@ -319,12 +348,15 @@
123     $enableGeoiptable = $true;
124     $savedata = $false;
125     }
126     - $savedata = $false; #TEMP!!
127     -
128     -#
129     -#---------------------------------------
130     -# Scan the qpsmtpd log file(s)
131     -#---------------------------------------
132     + $enableblacklist = ($cdb->get('qpsmtpd')->prop("RHSBL") || "disabled") eq "enabled" || ($cdb->get('qpsmtpd')->prop("URIBL") || "disabled") eq "enabled";
133     + #$savedata = $false; #TEMP!!
134     +#if ($savedata){print STDERR "yes"} else {print STDERR "no"}
135     +
136     +my $makeHTMLemail = "no";
137     +#if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
138     +my $makeHTMLpage = "no";
139     +#if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
140     +#if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
141    
142    
143     # Init the hashes
144     @@ -373,6 +405,12 @@
145    
146     my $count = -1; #for loop reduction in debugging mode
147    
148     +#
149     +#---------------------------------------
150     +# Scan the qpsmtpd log file(s)
151     +#---------------------------------------
152     +
153     +
154     my $CurrentMailId = "";
155    
156     LINE: while (<>) {
157     @@ -391,36 +429,38 @@
158    
159     #Count lines and skip out if debugging
160     $count++;
161     - last LINE if ($opt{debug} && $count >= 100000);
162     + #last LINE if ($opt{debug} && $count >= 100);
163     #dbg("REST:".$log);
164     +
165    
166     #Loglines to Saved String for later DB write
167     if ($savedata) {
168     my $CurrentLine = $_;
169     $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/;
170     - if ($2 ne $CurrentMailId) {
171     - $CurrentLogId = $1."-".$2;
172     - $CurrentMailId = $2;
173     + my $l = length($CurrentLine);
174     + if ($l != 0){
175     + if (defined($2)){ #print STDERR "Undefined \$2:".$_.":".$count.":".$l;exit}
176     + if ($2 ne $CurrentMailId) {
177     + print "CL:$CurrentLine*\n" if !defined($1);
178     + $CurrentLogId = $1."-".$2;
179     + $CurrentMailId = $2;
180     + $Sequence = 0;
181     + } else {$Sequence++}
182     + #$CurrentLogId .=":".$Sequence;
183     + $LogLines{$CurrentLogId.":".$Sequence} = $_;
184     + }
185     }
186     - $LogLines{$CurrentLogId} = $_;
187     - #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
188     + #print STDERR $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
189     + #exit
190     }
191    
192     - #Count lines and skip out if debugging
193     - $count++;
194     - last LINE if ($opt{debug} && $count >= 100);
195     - #dbg("REST:".$log);
196     -
197    
198     # pull out spamasassin rule lists
199     if ( $_ =~m/spamassassin: pass, Ham,(.*)</ )
200     #if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
201     {
202     - dbg("SPAM:".$log);
203     -
204     -
205     + #dbg("SPAM:".$log);
206     #New version does not seem to have spammassasin tests in logs
207     -
208     #if (exists($2){
209     #my (@SAtests) = split(',',$2);
210     #foreach my $SAtest (@SAtests) {
211     @@ -451,11 +491,6 @@
212    
213     #only select Logterse output
214     next LINE unless m/logging::logterse:/;
215     -
216     - #Count lines and skip out if debugging
217     - $count++;
218     - last LINE if ($opt{debug} && $count >= 100000);
219     - #dbg("REST:".$log);
220    
221     my $abstime = Time::TAI64::tai2unix($tai);
222     my $abshour = floor( $abstime / 3600 ); # Hours since the epoch
223     @@ -465,35 +500,89 @@
224     my (@log_items) = split $FS, $log_part;
225    
226     my (@timestamp_items) = split(' ',$timestamp_part);
227     -
228     +
229     + my $result= "rejected"; #Tag as rejected unti we know otherwise
230     # we store the more recent recipient domain, for domain statistics
231     # in fact, we only store the first recipient. Could be sort of headhache
232     # to obtain precise stats with many recipients on more than one domain !
233     my $proc = $timestamp_items[1] ; #numeric Id for the email
234     + my $emailnum = $proc; #proc gets modified later...
235    
236     + if ($emailnum == 23244) {
237     + dbg("TM0:".$timestamp_items[0]);
238     + dbg("TM1:".$timestamp_items[1]);
239     + dbg("TM2:".$timestamp_items[2]);
240     + dbg("TM3:".$timestamp_items[3]);
241     + dbg("LOG0:".$log_items[0]);
242     + dbg("LOG1:".$log_items[1]);
243     + dbg("LOG2:".$log_items[2]);
244     + dbg("LOG3:".$log_items[3]);
245     + dbg("LOG4:".$log_items[4]);
246     + dbg("LOG5:".$log_items[5]);
247     + dbg("LOG6:".$log_items[6]);
248     + dbg("LOG7:".$log_items[7]);
249     + dbg("IPregexp:".$localIPregexp);
250     + if (!test_for_private_ip($log_items[0])) {dbg("Log0 not found");}
251     + if (test_for_private_ip($log_items[2])){ dbg("Log2 match")}
252     + if ($log_items[5] eq "queued") {dbg("LOG5 match")}
253     + }
254     +
255     $totalexamined++;
256    
257     - #dbg("LOG1:".$log_items[1]);
258     - #dbg("LOG3:".$log_items[3]);
259     +# dbg("LOG8:".$log_items[8]);
260    
261     # first spot the fetchmail and local deliveries.
262    
263     # Spot from local workstation
264     $localflag = 0;
265     $WebMailflag = 0;
266     - if ( $log_items[1] =~ m/.*$DomainName.*/ ) {
267     + if ( $log_items[1] =~ m/$DomainName/ ) { #bjr
268     + #dbg("LOG1-Found:".$log_items[1]);
269     $localsendtotal++;
270     $counts{$abshour}{$CATLOCAL}++;
271     $localflag = 1;
272     }
273     +
274     + #Or a remote station
275     + elsif ((!test_for_private_ip($log_items[0])) and (test_for_private_ip($log_items[2])) and ($log_items[5] eq "queued"))
276     + {
277     + #Remote user
278     + $localflag = 1;
279     + $counts{$abshour}{$CATRELAY}++;
280     + }
281     +
282     + elsif (($log_items[2] =~ m/$WebmailIP/) and (!test_for_private_ip($log_items[0]))) {
283     + #Webmail
284     +# if ($emailnum == 19608){
285     + dbg("WEBMAIL:");
286     + dbg("TM0:".$timestamp_items[0]);
287     + dbg("TM1:".$timestamp_items[1]);
288     + dbg("TM2:".$timestamp_items[2]);
289     + dbg("TM3:".$timestamp_items[3]);
290     + dbg("LOG0:".$log_items[0]);
291     + dbg("LOG1:".$log_items[1]);
292     + dbg("LOG2:".$log_items[2]);
293     + dbg("LOG3:".$log_items[3]);
294     + dbg("LOG4:".$log_items[4]);
295     + dbg("LOG5:".$log_items[5]);
296     + dbg("LOG6:".$log_items[6]);
297     + dbg("LOG7:".$log_items[7]);
298     + #exit;
299     +# }
300     + $localflag = 1;
301     + $WebMailsendtotal++;
302     + $counts{$abshour}{$CATWEBMAIL}++;
303     + $WebMailflag = 1;
304     + }
305    
306     # see if from localhost
307     - elsif ( $log_items[1] =~ m/.*$localhost.*/ ) {
308     + elsif ( $log_items[1] =~ m/$localhost/ ) {
309     # but not if it comes from fetchmail
310     - if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { }
311     + if ( $log_items[3] =~ m/$FETCHMAIL/ ) { }
312     else {
313     + $localflag = 1;
314     # might still be from mailman here
315     - if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) {
316     + if ( $log_items[3] =~ m/$MAILMAN/ ) {
317     $mailmansendcount++;
318     $localsendtotal++;
319     $counts{$abshour}{$CATMAILMAN}++;
320     @@ -501,12 +590,10 @@
321     }
322     else {
323     #Or sent to the DMARC server
324     - dbg("LOG4:".$log_items[4]);
325     + #dbg("LOG4:".$log_items[4]);
326     #check for email address in $DMARC_Report_emails string
327     - #if ($log_items[4] =~ m/.*$DMARCDomain.*/) {
328     my $logemail = $log_items[4];
329     - #print STDERR "/",$log_items[4]."/\n";
330     - if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){
331     + if ((index($DMARC_Report_emails,$logemail)>=0) or ($logemail =~ m/$DMARCDomain/)){
332     $localsendtotal++;
333     $DMARCSendCount++;
334     $localflag = 1;
335     @@ -514,10 +601,11 @@
336     else {
337     #print STDERR "no match:.".$logemail;
338     if (exists $log_items[8]){
339     - dbg("LOG8:".$log_items[8]);
340     + #dbg("LOG8:".$log_items[8]);
341     # ignore incoming localhost spoofs
342     - if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
343     + if ( $log_items[8] =~ m/msg denied before queued/ ) { }
344     else {
345     + #Webmail
346     $localflag = 1;
347     $WebMailsendtotal++;
348     $counts{$abshour}{$CATWEBMAIL}++;
349     @@ -536,12 +624,12 @@
350     }
351    
352     # try to spot fetchmail emails
353     - if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) {
354     - dbg("LOG0:".$log_items[0]);
355     + if ( $log_items[0] =~ m/$FetchmailIP/ ) {
356     + #dbg("LOG0:".$log_items[0]);
357     $localAccepttotal++;
358     $counts{$abshour}{$CATFETCHMAIL}++;
359     }
360     - elsif ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) {
361     + elsif ( $log_items[3] =~ m/$FETCHMAIL/ ) {
362     $localAccepttotal++;
363     $counts{$abshour}{$CATFETCHMAIL}++;
364     }
365     @@ -549,12 +637,12 @@
366     # and adjust for recipient field if not set-up by denying plugin - extract from deny msg
367    
368     if ( length( $log_items[4] ) == 0 ) {
369     - dbg("LOG7:".$log_items[0]);
370     + #dbg("LOG7:".$log_items[0]);
371     if ( $log_items[5] eq 'check_goodrcptto' ) {
372     if ( $log_items[7] gt "invalid recipient" ) {
373     $log_items[4] =
374     substr( $log_items[7], 18 ); #Leave only email address
375     - dbg("LOG4:".$log_items[0]);
376     + #dbg("LOG4:".$log_items[0]);
377    
378     }
379     }
380     @@ -563,7 +651,7 @@
381     # if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) {
382     # reduce to lc and process each e,mail if a list, pseperatedy commas
383     my $recipientmail = lc( $log_items[4] );
384     - dbg("LOG4:".$log_items[0]);
385     + #dbg("LOG4:".$log_items[0]);
386     if ( $recipientmail =~ m/.*,/ ) {
387    
388     #comma - split the line and deal with each domain
389     @@ -599,146 +687,179 @@
390     # then categorise the result
391    
392    
393     - if (exists $log_items[5]) {
394     + if (exists $log_items[5]) {
395    
396     - if ($log_items[5] eq 'naughty') {
397     + if ($log_items[5] eq 'naughty') {
398     my $rejreason = $log_items[7];
399     $rejreason = /.*(\(.*\)).*/;
400     - $rejreason = $1;
401     + if (!defined($1)){$rejreason = "unknown"}
402     + else {$rejreason = $1}
403     $found_qpcodes{$log_items[5]."-".$rejreason}++}
404     else {$found_qpcodes{$log_items[5]}++} ##Count different qpsmtpd result codes
405    
406     - #Check for badly formed lines (from earlier testing)
407     -
408     - if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
409     + if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
410    
411     - elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
412     + elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
413    
414     - elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
415     + elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
416    
417     - elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
418     + elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
419    
420     - elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
421     + elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
422    
423     - elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
424     + elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
425    
426     - elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
427     + elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
428    
429     - elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
430     + elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
431    
432     - elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
433     + elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
434    
435     - elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
436     + elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
437    
438     - elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
439     + elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
440    
441     - elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
442     + elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
443    
444     - elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
445     + elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
446    
447     - elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE}
448     + elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)}
449    
450     - elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE}
451     + elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)}
452    
453     - elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
454     + elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
455    
456     - elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
457     + elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
458    
459     - elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
460     + elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
461    
462     - elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc);next LINE}
463     + elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc)}
464    
465     - elsif ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one.
466     + elsif ($log_items[5] eq 'tnef2mime') { } #Not expecting this one.
467    
468     - elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
469     - # and extract the spam score
470     -# if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)")
471     - if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)")
472     + elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
473     + # and extract the spam score
474     + # if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)")
475     + if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)")
476     {$rejectspamavg += $1}
477     - mark_domain_rejected($proc);
478     - next LINE
479     - }
480     + mark_domain_rejected($proc);
481     + }
482    
483     - elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
484     - #extract the virus name
485     - if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
486     - else {$found_viruses{$log_items[7]}++} #Some other message!!
487     - dbg("LOG7:".$log_items[7]);
488     - mark_domain_rejected($proc);
489     - next LINE
490     - }
491     + elsif (($log_items[5] eq 'virus::clamav') or ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
492     + #extract the virus name
493     + if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
494     + else {$found_viruses{$log_items[7]}++} #Some other message!!
495     + #dbg("LOG7:".$log_items[7]);
496     + mark_domain_rejected($proc);
497     + }
498    
499     - elsif ($log_items[5] eq 'queued') { $Accepttotal++;
500     - #extract the spam score
501     - if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") {
502     - $score = $1;
503     -# print $log_items[8]."<".$score.">\n";
504     - if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score}
505     - else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score}
506     - } else {
507     - # no SA score - treat it as ham
508     - $hamcount++;$counts{$abshour}{$CATHAM}++;
509     - }
510     - if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
511     - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
512     - $currentrcptdomain{ $proc } = '' ;
513     - }
514     - next LINE
515     - }
516     -
517     + elsif ($log_items[5] eq 'queued') { $Accepttotal++;
518     + #extract the spam score
519     + # Remove count for rejectred as it looks as if it might get through!!
520     + $result= "queued";
521     + if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") {
522     + $score = trim($1);
523     + if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric
524     + {
525     + if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;}
526     + else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";}
527     + } else {
528     + print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n";
529     + }
530     + } else {
531     + # no SA score - treat it as ham
532     + $hamcount++;$counts{$abshour}{$CATHAM}++;
533     + }
534     + if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
535     + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
536     + $currentrcptdomain{ $proc } = '' ;
537     + }
538     + }
539    
540     - elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
541    
542     - elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
543     -
544     - elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
545     -
546     - elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
547     -
548     - elsif ($log_items[5] eq 'naughty') {
549     + elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
550     +
551     + elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
552     +
553     + elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
554     +
555     + elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
556     +
557     + elsif ($log_items[5] eq 'naughty') {
558     #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify
559     if ($log_items[7] =~ m/(karma)/) {
560     - $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
561     + $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)}
562     elsif ($log_items[7] =~ m/(dnsbl)/){
563     - $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
564     + $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
565     elsif ($log_items[7] =~ m/(helo)/){
566     - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
567     + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
568     else {
569     #Unidentified Naughty rejection
570     - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE}
571     + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++}
572     }
573     - elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
574     + elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
575    
576     - elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE}
577     + elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc)}
578    
579     - elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
580     + elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)}
581    
582     - elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE}
583     + elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc)}
584    
585     - elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
586     + elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
587    
588     - elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
589     + elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
590    
591     - elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
592     + elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
593    
594     - elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
595     -
596     - elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
597     + elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
598    
599     - elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
600     + elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
601    
602     - elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
603     + elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
604     +
605     + elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
606    
607     #Treat it as Unconf if not recognised
608     - else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE}
609     + else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++}
610     } #Log[5] exists
611    
612     -
613     -# print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected
614     -# next LINE
615     -
616     -
617     -
618     + #Entry if not local send
619     + if ($localflag == 0) {
620     + if (length($log_items[4]) > 0){
621     + # Need to check here for multiple email addresses
622     + my @emails = split(",",lc($log_items[4]));
623     + if (scalar(@emails) > 1) {
624     + #Just pick the first local address to hang it on.
625     + # TEMP - just go for the first address until I can work out how to spot the 1st "local" one
626     + $usercounts{$emails[0]}{$result}++;
627     + $usercounts{$emails[0]}{"proc"} = $proc;
628     + #Compare with @domains array until we get a local one
629     + my $gotone = $false;
630     + foreach my $email (@emails){
631     + #Extract the domain from the email address
632     + my $fullemail = $email;
633     + $email = s/.*\@(.*)$/$1/;
634     + #and see if it is local
635     + if ($email =~ m/$alldomains/){
636     + $usercounts{lc($fullemail)}{$result}++;
637     + $usercounts{lc($fullemail)}{"proc"} = $proc;
638     + $gotone = $true;
639     + last;
640     + }
641     + }
642     + if (!$gotone) {
643     + $usercounts{'No internal email $proc'}{$result}++;
644     + $usercounts{'No internal email $proc'}{"proc"} = $proc;
645     + }
646     +
647     + } else {
648     + $usercounts{lc($log_items[4])}{$result}++;
649     + $usercounts{lc($log_items[4])}{"proc"} = $proc;
650     + }
651     + }
652     + }
653     + #exit if $emailnum == 15858;
654     +
655     } #END OF MAIN LOOP
656    
657     #total up grand total Columns
658     @@ -750,7 +871,7 @@
659     $counts{$GRANDTOTAL}{$categs[$ncateg]} += $counts{$nhour}{$categs[$ncateg]};
660    
661     # and total rows
662     - if ( $ncateg < $categlen && $ncateg>=$countfromhere) {#skip initial columns of non final reasons
663     + if ( $ncateg < $categlen and $ncateg>=$countfromhere) {#skip initial columns of non final reasons
664     $counts{$nhour}{$categs[@categs-2]} += $counts{$nhour}{$categs[$ncateg]};
665     }
666     $ncateg++
667     @@ -815,7 +936,7 @@
668     my $oldfh;
669    
670     #Open Sendmail if we are mailing it
671     -if ( $opt{'mail'} && !$disabled ) {
672     +if ( $opt{'mail'} and !$disabled ) {
673     open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" )
674     or die "Can't open sendmail: $!\n";
675     print SENDMAIL "From: $opt{'from'}\n";
676     @@ -841,44 +962,44 @@
677     my $oldFH = select $outputFH;
678    
679    
680     - print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n";
681     - print "----------------------------------------------------", "\n\n";
682     -
683     - print "$0 Version : $opt{'version'}", "\n\n";
684     - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n";
685     + print "SMEServer daily Anti-Virus and Spamfilter statistics from $hostname - ".strftime( "%F", localtime($start))."\n";
686     + print "----------------------------------------------------------------------------------", "\n\n";
687     + print "$0 Version : $opt{'version'}", "\n";
688     + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n";
689     print "Period Ending : ", strftime( "%c", localtime($end) ), "\n";
690     - print "\n";
691     -
692     - print "Clam Version/DB Count/Last DB update: ", `freshclam -V`."\n";
693     - print "SpamAssassin Version : ", `spamassassin -V`."\n";
694     - printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel;
695     + print "Clam Version/DB Count/Last DB update: ",`freshclam -V`;
696     + print "SpamAssassin Version : ",`spamassassin -V`;
697     + printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,$SARejectLevel;
698     if ($HighLogLevel) {
699     printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n";
700     printf "\tYou can set it this way:\n";
701     printf "\tconfig setprop qpsmtpd LogLevel 6\n";
702     printf "\tsignal-event email-update\n";
703     - printf "\tsv t /var/service/qpsmtpd\n\n";
704     + printf "\tsv t /var/service/qpsmtpd\n";
705     }
706     - print "\n\n";
707     printf "Reporting Period : %.2f hrs\n", $hrsinperiod;
708     - #print "----------------------------\n";
709     - #print "\n";
710     -
711     printf "All SMTP connections accepted:%8d \n", $totalexamined;
712     -
713     printf "Emails per hour : %8.1f/hr\n", $emailperhour || 0;
714     - print "\n";
715     printf "Average spam score (accepted): %11.2f\n", $spamavg || 0;
716     printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0;
717     printf "Average ham score : %11.2f\n", $hamavg || 0;
718     - printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount || 0;
719     - if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
720     + printf "Number of DMARC reporting emails sent:\t%11d (not shown on table)\n", $DMARCSendCount || 0;
721     + if ($hamcount != 0){ printf "Number of emails approved through DMARC:\t%11d (%3d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
722     +
723     + my $smeoptimizerprog = "/usr/local/smeoptimizer/SMEOptimizer.pl";
724     + if (-e $smeoptimizerprog) {
725     + #smeoptimizer installed - get result of status
726     + my @smeoptimizerlines = split(/\n/,`/usr/local/smeoptimizer/SMEOptimizer.pl -status`);
727     + print("SMEOptimizer status:\n");
728     + print("\t".$smeoptimizerlines[6]."\n");
729     + print("\t".$smeoptimizerlines[7]."\n");
730     + print("\t".$smeoptimizerlines[8]."\n");
731     + print("\t".$smeoptimizerlines[9]."\n");
732     + print("\t".$smeoptimizerlines[10]."\n");
733     + }
734     +
735    
736     - print "\n\n";
737     - print "\nStatistics by Hour\n";
738     - print "-------------------\n";
739     - #print "\n";
740     -
741     + print "\nStatistics by Hour:\n";
742     #
743     # start by working out which colunns to show - tag the display array
744     #
745     @@ -968,24 +1089,17 @@
746     #
747     # print it.
748     #
749     - my $makeHTMLemail = "no";
750     - #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
751     - my $makeHTMLpage = "no";
752     - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
753     - #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
754     -
755     - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion ....
756     - print $Titles."\n";
757     - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto
758     - #$Line2 =~ s/-/a/g;
759     - #print $Line2."\n";
760     - #print "\n";
761     - print $Values."\n";
762     +
763     + print $Line1."\n";
764     + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion ....
765     + print $Titles."\n";
766     + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto
767     + print $Line2."\n";
768     + print $Values;
769     print $Line2."\n";
770     print $Totals."\n";
771     print $Percent."\n";
772     print $Line1."\n";
773     - print "\n";
774    
775     if ($localAccepttotal>0) {
776     print "*Fetchml* means connections from Fetchmail delivering email\n";
777     @@ -1021,7 +1135,7 @@
778     }
779    
780    
781     - if ( !$RHSenabled || !$DNSenabled ) {
782     + if ( !$RHSenabled or !$DNSenabled ) {
783    
784     # comment about RBL not set
785     print
786     @@ -1114,11 +1228,15 @@
787    
788     if ($enableSARules) {show_SARules_codes();}
789    
790     - if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();}
791     + if ($enableGeoiptable and $finaldisplay[$BadCountryCateg]){show_Geoip_results();}
792    
793     if ($enablejunkMailList) {List_Junkmail();}
794     +
795     + if ($enableblacklist) {show_blacklist_counts();}
796     +
797     + show_user_stats();
798    
799     - print "\nDone. Report generated in $telapsed sec.\n\n";
800     + print "\nReport generated in $telapsed sec.\n";
801    
802     if ($savedata) { save_data(); }
803     else
804     @@ -1128,25 +1246,26 @@
805    
806     select $oldFH;
807     close $outputFH;
808     - if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr}
809     - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){
810     + if ($makeHTMLemail eq "no" or $makeHTMLemail eq "both") {print $tablestr}
811     + if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" or $makeHTMLpage eq "yes"){
812     #Convert text to html and send it
813     require CGI;
814     require TextToHTML;
815     my $cgi = new CGI;
816     my $text = $tablestr;
817     - print $cgi->header();
818     my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20);
819     my $conv = new HTML::TextToHTML();
820     $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2,
821     underline_length_tolerance=>1);
822     - my $html="<!DOCTYPE html> <html>\n";
823     +
824     + my $html = $cgi->header();
825     + $html .="<!DOCTYPE html> <html>\n";
826     $html .= "<head><title>Mailstats -".strftime( "%F", localtime($start) )."</title>";
827     $html .= "<link rel='stylesheet' type='text/css' href='mailstats.css' /></head>\n";
828     $html .= "<body>\n";
829     $html .= $conv->process_chunk($text);
830     $html .= "</body></html>\n";
831     - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html}
832     + if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" ) {print $html}
833     #And drop it into a file
834     if ($makeHTMLpage eq "yes") {
835     my $filename = "mailstats.html";
836     @@ -1206,7 +1325,7 @@
837     } elsif ($base =~m/\d+/){
838     $sec=0;$min=0;$hour=$base;
839     };
840     - #$mday="17"; #$mday="03"; #$mday="16"; #Temp!!
841     + #$mday="05"; #$mday="03"; #$mday="16"; #Temp!!
842     $time = timelocal($sec,$min,$hour,$mday,$mon,$year);
843     }
844    
845     @@ -1255,10 +1374,9 @@
846     }
847     my $i = keys %junkcount;
848     if ( $i > 0 ) {
849     - print "\n\n";
850     print("\nJunk Mails left in folder:\n");
851     - print("---------------------------\n\n");
852     - print("\nCount\tUser\n");
853     + print("---------------------------\n");
854     + print("Count\tUser\n");
855     print("-------------------------\n");
856     foreach my $thisuser (
857     sort { $junkcount{$b} <=> $junkcount{$a} }
858     @@ -1282,20 +1400,20 @@
859     #
860    
861     {
862     -
863     - print("Virus Statistics by name:\n");
864     - print("------------------------------------------------------------------------\n");
865     + my $line = "------------------------------------------------------------------------\n";
866     + print("\nVirus Statistics by name:\n");
867     + print($line);
868     foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} }
869     keys %found_viruses)
870     {
871     - if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){
872     + if (index($virus,"Sanesecurity") !=-1 or index($virus,"UNOFFICIAL") !=-1){
873     print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n";
874     } else {
875     print "Rejected $found_viruses{$virus}\t$virus\n";
876     }
877    
878     }
879     - print("------------------------------------------------------------------------\n\n");
880     + print($line);
881     }
882    
883     sub show_qpsmtpd_codes
884     @@ -1305,18 +1423,94 @@
885     #
886    
887     {
888     -
889     - print("Qpsmtpd codes league table:\n");
890     - print("---------------------------------------------\n");
891     - print("Count\tPercent\tReason\t\n");
892     - print("---------------------------------------------\n");
893     + my $line = "---------------------------------------------\n";
894     + print("\nQpsmtpd codes league table:\n");
895     + print($line);
896     + print("Count\tPercent\tReason\n");
897     + print($line);
898     foreach my $qpcode (sort { $found_qpcodes{$b} <=> $found_qpcodes{$a} }
899     keys %found_qpcodes)
900     {
901     - print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined;
902     + print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t\t$qpcode\n" if $totalexamined;
903     + }
904     + print($line);
905     +}
906     +
907     +sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
908     +
909     +sub get_domain
910     +{ #my $url= URI->new(shift);
911     + #my $domain = $url->host;
912     + my $url = shift;
913     + $url =~ s!^.*https?://(?:www\.)?!!i;
914     + $url =~ s!/.*!!;
915     + $url =~ s/[\?\#\:].*//;
916     + my $domain = trim($url);
917     + return $domain;
918     +}
919     +
920     +sub show_blacklist_counts
921     +
922     +#
923     +# Show a sorted league table of the blacklist URL counts
924     +#
925     +
926     +{
927     + my $line = "------------------\n";
928     + print("\nBlacklist details:\n");
929     + print($line);
930     + if ($cdb->get('qpsmtpd')->prop("RHSBL") eq "enabled") {print "RBLLIST:".$cdb->get('qpsmtpd')->prop("RBLList")."\n";}
931     + if ($cdb->get('qpsmtpd')->prop("URIBL") eq "enabled") {print "UBLLIST:".$cdb->get('qpsmtpd')->prop("UBLList")."\n";}
932     + if (!$cdb->get('qpsmtpd')->prop("SBLList") eq "") {print "SBLLIST:".$cdb->get('qpsmtpd')->prop("SBLList")."\n";}
933     + print($line);
934     + print("Count\tURL\n");
935     + print($line);
936     + foreach my $blcode (sort { $blacklistURL{$b} <=> $blacklistURL{$a} }
937     + keys %blacklistURL)
938     + {
939     + print sprintf('%3u',$blacklistURL{$blcode})."\t$blcode\n";
940     }
941     - print("---------------------------------------------\n\n");
942     - print "\n\n";
943     + print($line);
944     +}
945     +
946     +
947     +sub show_user_stats
948     +
949     +#
950     +# Show a sorted league table of the user counts
951     +#
952     +
953     +{
954     + #Compute totals for each entry
955     + my $grandtotals=0;
956     + my $totalqueued=0;
957     + my $totalspam=0;
958     + my $totalrejected=0;
959     + foreach my $user (keys %usercounts){
960     + $usercounts{$user}{"queued"} = 0 if !(exists $usercounts{$user}{"queued"});
961     + $usercounts{$user}{"rejected"} = 0 if !(exists $usercounts{$user}{"rejected"});
962     + $usercounts{$user}{"spam"} = 0 if !(exists $usercounts{$user}{"spam"});
963     + $usercounts{$user}{"totals"} = $usercounts{$user}{"queued"}+$usercounts{$user}{"rejected"}+$usercounts{$user}{"spam"};
964     + $grandtotals += $usercounts{$user}{"totals"};
965     + $totalspam += $usercounts{$user}{"spam"};
966     + $totalqueued += $usercounts{$user}{"queued"};
967     + $totalrejected += $usercounts{$user}{"rejected"};
968     + }
969     + my $line = "--------------------------------------------------\n";
970     + print("\nStatistics by email address received:\n");
971     + print($line);
972     + print("Queued\tRejected\tSpam tagged\tEmail Address\n");
973     + print($line);
974     + foreach my $user (sort { $usercounts{$b}{"totals"} <=> $usercounts{$a}{"totals"} }
975     + keys %usercounts)
976     + {
977     + print sprintf('%3u',$usercounts{$user}{"queued"})."\t".sprintf('%3u',$usercounts{$user}{"rejected"})."\t\t".sprintf('%3u',$usercounts{$user}{"spam"})."\t\t$user\n";
978     + }
979     + print($line);
980     + print sprintf('%3u',$totalqueued)."\t".sprintf('%3u',$totalrejected)."\t\t".sprintf('%3u',$totalspam)."\n";
981     + print($line);
982     +
983     +
984     }
985    
986     sub show_Geoip_results
987     @@ -1335,10 +1529,11 @@
988     $percentthreshold = 0.5;
989     }
990     if ($total_countries > 0) {
991     - print("Geoip results: (cutoff at $percentthreshold%) \n");
992     - print("---------------------------------\n");
993     + my $line = "---------------------------------------------\n";
994     + print("\nGeoip results: (cutoff at $percentthreshold%) \n");
995     + print($line);
996     print("Country\tPercent\tCount\tRejected?\n");
997     - print("---------------------------------\n");
998     + print($line);
999     foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
1000     keys %found_countries)
1001     {
1002     @@ -1347,14 +1542,14 @@
1003     $totalpercent = $totalpercent + $percent;
1004     if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
1005     if ( $percent >= $percentthreshold ) {
1006     - print "$country\t"
1007     + print "$country\t\t"
1008     . sprintf( '%4.1f', $percent )
1009     - . "%\t$found_countries{$country}","\t$reject\n"
1010     + . "%\t\t$found_countries{$country}","\t$reject\n"
1011     if $total_countries;
1012     }
1013    
1014     }
1015     - print("---------------------------------\n");
1016     + print($line);
1017     my ($showtotals);
1018     if ($cdb->get('mailstats')){
1019     $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
1020     @@ -1363,10 +1558,9 @@
1021     }
1022    
1023     if ($showtotals){
1024     - print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n";
1025     - print("---------------------------------\n\n");
1026     + print "TOTALS\t\t".sprintf("%4.1f",$totalpercent)."%\t\t$total_countries\n";
1027     + print($line);
1028     }
1029     - print "\n";
1030     }
1031     }
1032    
1033     @@ -1384,7 +1578,7 @@
1034    
1035     if ($sum_SARules > 0){
1036    
1037     - if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
1038     + if ($totalexamined >0 and $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
1039     $defaultpercentthreshold = $maxcutoff
1040     } else {
1041     $defaultpercentthreshold = $mincutoff
1042     @@ -1394,17 +1588,15 @@
1043     } else {
1044     $percentthreshold = $defaultpercentthreshold
1045     }
1046     -
1047     - print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
1048     - print("---------------------------------------------\n");
1049     + my $line = "---------------------------------------------\n";
1050     + print("\nSpamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
1051     + print($line);
1052     print("Count\tPercent\tScore\t\t\n");
1053     - print("---------------------------------------------\n");
1054     + print($line);
1055     foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
1056     keys %found_SARules)
1057     {
1058     - my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
1059     - if $totalexamined;
1060     - #$totalpercent = $totalpercent + $percent;
1061     + my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined if $totalexamined;
1062     my $avehits = $found_SARules{$SARule}{'totalhits'} /
1063     $found_SARules{$SARule}{'count'}
1064     if $found_SARules{$SARule}{'count'};
1065     @@ -1416,7 +1608,7 @@
1066     if $totalexamined;
1067     }
1068     }
1069     - print("---------------------------------------------\n");
1070     + print($line);
1071     my ($showtotals);
1072     if ($cdb->get('mailstats')){
1073     $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
1074     @@ -1426,7 +1618,7 @@
1075    
1076     if ($showtotals){
1077     print "$totalexamined\t(TOTALS)\n";
1078     - print("---------------------------------------------\n");
1079     + print($line);
1080     }
1081     print "\n";
1082     }
1083     @@ -1631,15 +1823,25 @@
1084     }
1085     $nhour++;
1086     }
1087     - # and write out the log lines saved
1088     -
1089     - foreach my $logid (keys %LogLines){
1090     -
1091     - $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')");
1092     + # and write out the log lines saved - only if html wanted
1093     + if ($makeHTMLemail eq 'yes' or $makeHTMLemail eq 'both' or $makeHTMLpage eq 'yes'){
1094     + foreach my $logid (keys %LogLines){
1095     + $reccount++;
1096     + #Extract from keys
1097     + my $extract = $logid;
1098     + $extract =~/^(.*)-(.*):(.*)$/;
1099     + my $Log64n = $1;
1100     + my $LogMailId = $2;
1101     + my $LogSeq = $3;
1102     + my $LogLine = $dbh->quote($LogLines{$logid});
1103     + my $sql = "INSERT INTO LogData (Log64n,MailID,Sequence,LogStr) VALUES ('";
1104     + $sql .= $Log64n."','".$LogMailId."','".$LogSeq."',".$LogLine.")";
1105     + $dbh->do($sql) or die($sql);
1106     + }
1107     + $dbh->disconnect();
1108     + $telapsed = time - $tstart;
1109     + print "Saved $reccount records in $telapsed sec.";
1110     }
1111     - $dbh->disconnect();
1112     - my $telapsed = time - $tstart;
1113     - print "Saved $reccount records in $telapsed sec.";
1114     }
1115    
1116     sub check_date_rec
1117     @@ -1706,3 +1908,47 @@
1118     my $daterec = $sth->fetchrow_hashref();
1119     $daterec->{"dateid"};
1120     }
1121     +
1122     + sub dump_entries
1123     + {
1124     + my $msg = shift;
1125     + #dbg($msg);
1126     + #dbg("TM0:".$timestamp_items[0]);
1127     + #dbg("TM1:".$timestamp_items[1]);
1128     + #dbg("TM2:".$timestamp_items[2]);
1129     + #dbg("TM3:".$timestamp_items[3]);
1130     + #dbg("LOG0:".$log_items[0]);
1131     + #dbg("LOG1:".$log_items[1]);
1132     + #dbg("LOG2:".$log_items[2]);
1133     + #dbg("LOG3:".$log_items[3]);
1134     + #dbg("LOG4:".$log_items[4]);
1135     + #dbg("LOG5:".$log_items[5]);
1136     + #dbg("LOG6:".$log_items[6]);
1137     + #dbg("LOG7:".$log_items[7]);
1138     + #if ($opt{debug} == 1){exit;}
1139     +}
1140     +
1141     +#sub test_for_private_ip {
1142     + #use NetAddr::IP;
1143     + #my $ip = shift;
1144     + #$ip =~ s/^\D*(([0-9]{1,3}\.){3}[0-9]{1,3}).*/$1/e;
1145     + #print "\nIP:$ip";
1146     + #my $nip = NetAddr::IP->new($ip);
1147     + #if ($nip){
1148     + #if ( $nip->is_rfc1918() ){
1149     + #return 1;
1150     + #} else { return 0}
1151     + #} else { return 0}
1152     +#}
1153     +
1154     +
1155     +sub test_for_private_ip {
1156     + use NetAddr::IP;
1157     + $_ = shift;
1158     + return unless /(\d+\.\d+\.\d+\.\d+)/;
1159     + my $ip = NetAddr::IP->new($1);
1160     + return unless $ip;
1161     + return $ip->is_rfc1918();
1162     +}
1163     +
1164     +

admin@koozali.org
ViewVC Help
Powered by ViewVC 1.2.1 RSS 2.0 feed