/[smecontribs]/rpms/smeserver-mailstats/contribs9/smeserver-mailstats-1.1.bz9717-9716email_specific_stats.patch
ViewVC logotype

Contents of /rpms/smeserver-mailstats/contribs9/smeserver-mailstats-1.1.bz9717-9716email_specific_stats.patch

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (show annotations) (download)
Mon Oct 17 10:21:42 2016 UTC (8 years ago) by brianread
Branch: MAIN
CVS Tags: smeserver-mailstats-1_1-12_el6_sme, smeserver-mailstats-1_1-11_el6_sme, smeserver-mailstats-1_1-9_el6_sme, smeserver-mailstats-1_1-10_el6_sme, HEAD
Add patch

1 diff -urN smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl
2 --- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl 2016-10-16 14:40:37.689999942 +0100
3 +++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl 2016-10-12 14:26:59.000000000 +0100
4 @@ -20,6 +20,10 @@
5 # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection
6 # bjr - 14Jun16 - make compatible with qpsmtpd 0.96
7 # bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7)
8 +# bjr - 04Aug16 - Add code to log and count the blacklist RBL urls that have triggered, this (NFR) is Bugzilla 9717
9 +# bjr - 04Aug16 - Add code to expand the junkmail table to include daily ham and spam and deleted spam for each user - (NFR bugzilla 9716)
10 +# bjr - 05Aug16 - Add code to log remote relay incoming emails
11 +# bjr - 10Oct16 - Add code to show stats for the smeoptimizer package
12 #
13 #############################################################################
14 #
15 @@ -45,6 +49,15 @@
16 # / HTMLPage - "yes" / "no" - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no"
17 #
18 #############################################################################
19 +#
20 +#
21 +# TODO
22 +#
23 +# 1. Delete loglines records from any previous run of same table
24 +# 2. Add tracking LogId for each cont in the table
25 +# 3. Use link directory file to generate h1 / h2 tags for title and section headings
26 +# 4. Ditto for links to underlying data
27 +#
28
29 # internal modules (part of core perl distribution)
30 use strict;
31 @@ -60,6 +73,7 @@
32 use Sys::Hostname;
33 use Switch;
34 use DBIx::Simple;
35 +use URI::URL;
36
37 #use CGI;
38 #use HTML::TextToHTML;
39 @@ -81,7 +95,7 @@
40
41 #Configuration section
42 my %opt = (
43 - version => '0.7.5a', # please update at each change.
44 + version => '0.7.10', # please update at each change.
45 debug => 0, # guess what ?
46 sendmail => '/usr/sbin/sendmail', # Path to sendmail stub
47 from => 'spamfilter-stats', # Who is the mail from
48 @@ -96,7 +110,7 @@
49 my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost
50 my $DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
51 my $DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval
52 -
53 +my $localIPregexp = ".*((127\.)|(10\.)|(172\.1[6-9]\.)|(172\.2[0-9]\.)|(172\.3[0-1]\.)|(192\.168\.)).*";
54 my $MinCol = 6; #Minimum column width
55 my $HourColWidth = 16; #Date and time column width
56
57 @@ -135,10 +149,12 @@
58 my %found_SARules = ();
59 my %junkcount = ();
60 my %unrecog_plugin = ();
61 +my %blacklistURL = (); #Count of use of each balcklist rhsbl
62 +my %usercounts = (); #Count per received email of sucessful delivery, queued spam and deleted Spam, and rejected
63
64 # replaced by...
65 -my %counts = (); #Hold all counts in 2-D matrix
66 -my @display = (); #used to switch on and off columns - yes, no or auto for each category
67 +my %counts = (); #Hold all counts in 2-D matrix
68 +my @display = (); #used to switch on and off columns - yes, no or auto for each category
69 my @colwidth = (); #width of each column
70 #(auto means only if non zero) - populated from possible db entries
71 my @finaldisplay = (); #final decision on display or not - true or false
72 @@ -149,8 +165,9 @@
73 my $CATWEBMAIL='WebMail';
74 my $CATMAILMAN='Mailman';
75 my $CATLOCAL='Local';
76 +my $CATRELAY="Relay";
77 # border between where it came from and where it ended..
78 -my $countfromhere = 5; #Temp - Check this not moved!!
79 +my $countfromhere = 6; #Temp - Check this not moved!!
80
81 my $CATVIRUS='Virus';
82 my $CATRBLDNS='RBL/DNS';
83 @@ -166,7 +183,7 @@
84 my $CATPERCENT='PERCENT';
85 my $CATDMARC="DMARC Rej.";
86 my $CATLOAD="Rej.Load";
87 -my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
88 +my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATRELAY,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
89 my $GRANDTOTAL = '99'; #subs for count arrays, for grand total
90 my $PERCENT = '98'; # for column percentages
91
92 @@ -238,10 +255,21 @@
93
94 #dbg("DMARC-EMAILS:".$DMARC_Report_emails);
95
96 +# and setup list of local domains for spotting the local one in a list of email addresses (Remote station processing)
97 +use esmith::DomainsDB;
98 +my $d = esmith::DomainsDB->open_ro();
99 +my @domains = $d->keys();
100 +my $alldomains = "(";
101 +foreach my $dom (@domains){$alldomains .= $dom."|"}
102 +$alldomains .= ")";
103 +#print $alldomains;
104 +
105 # Saving the Log lines processed
106 my %LogLines = (); #Save all the log lines processed for writing to the DB
107 my %LogId = (); #Save the Log Ids.
108 my $CurrentLogId = "";
109 +my $Sequence = 0;
110 +
111
112 # store the domain of interest. Every other records are stored in a 'Other' zone
113 my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n";
114 @@ -306,6 +334,7 @@
115 my $enableGeoiptable;
116 my $enablejunkMailList;
117 my $savedata;
118 +my $enableblacklist; #Enabled according to setting in qpsmtpd
119 if ($cdb->get('mailstats')){
120 $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
121 $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
122 @@ -319,12 +348,15 @@
123 $enableGeoiptable = $true;
124 $savedata = $false;
125 }
126 - $savedata = $false; #TEMP!!
127 -
128 -#
129 -#---------------------------------------
130 -# Scan the qpsmtpd log file(s)
131 -#---------------------------------------
132 + $enableblacklist = ($cdb->get('qpsmtpd')->prop("RHSBL") || "disabled") eq "enabled" || ($cdb->get('qpsmtpd')->prop("URIBL") || "disabled") eq "enabled";
133 + #$savedata = $false; #TEMP!!
134 +#if ($savedata){print STDERR "yes"} else {print STDERR "no"}
135 +
136 +my $makeHTMLemail = "no";
137 +#if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
138 +my $makeHTMLpage = "no";
139 +#if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
140 +#if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
141
142
143 # Init the hashes
144 @@ -373,6 +405,12 @@
145
146 my $count = -1; #for loop reduction in debugging mode
147
148 +#
149 +#---------------------------------------
150 +# Scan the qpsmtpd log file(s)
151 +#---------------------------------------
152 +
153 +
154 my $CurrentMailId = "";
155
156 LINE: while (<>) {
157 @@ -391,36 +429,38 @@
158
159 #Count lines and skip out if debugging
160 $count++;
161 - last LINE if ($opt{debug} && $count >= 100000);
162 + #last LINE if ($opt{debug} && $count >= 100);
163 #dbg("REST:".$log);
164 +
165
166 #Loglines to Saved String for later DB write
167 if ($savedata) {
168 my $CurrentLine = $_;
169 $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/;
170 - if ($2 ne $CurrentMailId) {
171 - $CurrentLogId = $1."-".$2;
172 - $CurrentMailId = $2;
173 + my $l = length($CurrentLine);
174 + if ($l != 0){
175 + if (defined($2)){ #print STDERR "Undefined \$2:".$_.":".$count.":".$l;exit}
176 + if ($2 ne $CurrentMailId) {
177 + print "CL:$CurrentLine*\n" if !defined($1);
178 + $CurrentLogId = $1."-".$2;
179 + $CurrentMailId = $2;
180 + $Sequence = 0;
181 + } else {$Sequence++}
182 + #$CurrentLogId .=":".$Sequence;
183 + $LogLines{$CurrentLogId.":".$Sequence} = $_;
184 + }
185 }
186 - $LogLines{$CurrentLogId} = $_;
187 - #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
188 + #print STDERR $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
189 + #exit
190 }
191
192 - #Count lines and skip out if debugging
193 - $count++;
194 - last LINE if ($opt{debug} && $count >= 100);
195 - #dbg("REST:".$log);
196 -
197
198 # pull out spamasassin rule lists
199 if ( $_ =~m/spamassassin: pass, Ham,(.*)</ )
200 #if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
201 {
202 - dbg("SPAM:".$log);
203 -
204 -
205 + #dbg("SPAM:".$log);
206 #New version does not seem to have spammassasin tests in logs
207 -
208 #if (exists($2){
209 #my (@SAtests) = split(',',$2);
210 #foreach my $SAtest (@SAtests) {
211 @@ -451,11 +491,6 @@
212
213 #only select Logterse output
214 next LINE unless m/logging::logterse:/;
215 -
216 - #Count lines and skip out if debugging
217 - $count++;
218 - last LINE if ($opt{debug} && $count >= 100000);
219 - #dbg("REST:".$log);
220
221 my $abstime = Time::TAI64::tai2unix($tai);
222 my $abshour = floor( $abstime / 3600 ); # Hours since the epoch
223 @@ -465,35 +500,89 @@
224 my (@log_items) = split $FS, $log_part;
225
226 my (@timestamp_items) = split(' ',$timestamp_part);
227 -
228 +
229 + my $result= "rejected"; #Tag as rejected unti we know otherwise
230 # we store the more recent recipient domain, for domain statistics
231 # in fact, we only store the first recipient. Could be sort of headhache
232 # to obtain precise stats with many recipients on more than one domain !
233 my $proc = $timestamp_items[1] ; #numeric Id for the email
234 + my $emailnum = $proc; #proc gets modified later...
235
236 + if ($emailnum == 23244) {
237 + dbg("TM0:".$timestamp_items[0]);
238 + dbg("TM1:".$timestamp_items[1]);
239 + dbg("TM2:".$timestamp_items[2]);
240 + dbg("TM3:".$timestamp_items[3]);
241 + dbg("LOG0:".$log_items[0]);
242 + dbg("LOG1:".$log_items[1]);
243 + dbg("LOG2:".$log_items[2]);
244 + dbg("LOG3:".$log_items[3]);
245 + dbg("LOG4:".$log_items[4]);
246 + dbg("LOG5:".$log_items[5]);
247 + dbg("LOG6:".$log_items[6]);
248 + dbg("LOG7:".$log_items[7]);
249 + dbg("IPregexp:".$localIPregexp);
250 + if (!test_for_private_ip($log_items[0])) {dbg("Log0 not found");}
251 + if (test_for_private_ip($log_items[2])){ dbg("Log2 match")}
252 + if ($log_items[5] eq "queued") {dbg("LOG5 match")}
253 + }
254 +
255 $totalexamined++;
256
257 - #dbg("LOG1:".$log_items[1]);
258 - #dbg("LOG3:".$log_items[3]);
259 +# dbg("LOG8:".$log_items[8]);
260
261 # first spot the fetchmail and local deliveries.
262
263 # Spot from local workstation
264 $localflag = 0;
265 $WebMailflag = 0;
266 - if ( $log_items[1] =~ m/.*$DomainName.*/ ) {
267 + if ( $log_items[1] =~ m/$DomainName/ ) { #bjr
268 + #dbg("LOG1-Found:".$log_items[1]);
269 $localsendtotal++;
270 $counts{$abshour}{$CATLOCAL}++;
271 $localflag = 1;
272 }
273 +
274 + #Or a remote station
275 + elsif ((!test_for_private_ip($log_items[0])) and (test_for_private_ip($log_items[2])) and ($log_items[5] eq "queued"))
276 + {
277 + #Remote user
278 + $localflag = 1;
279 + $counts{$abshour}{$CATRELAY}++;
280 + }
281 +
282 + elsif (($log_items[2] =~ m/$WebmailIP/) and (!test_for_private_ip($log_items[0]))) {
283 + #Webmail
284 +# if ($emailnum == 19608){
285 + dbg("WEBMAIL:");
286 + dbg("TM0:".$timestamp_items[0]);
287 + dbg("TM1:".$timestamp_items[1]);
288 + dbg("TM2:".$timestamp_items[2]);
289 + dbg("TM3:".$timestamp_items[3]);
290 + dbg("LOG0:".$log_items[0]);
291 + dbg("LOG1:".$log_items[1]);
292 + dbg("LOG2:".$log_items[2]);
293 + dbg("LOG3:".$log_items[3]);
294 + dbg("LOG4:".$log_items[4]);
295 + dbg("LOG5:".$log_items[5]);
296 + dbg("LOG6:".$log_items[6]);
297 + dbg("LOG7:".$log_items[7]);
298 + #exit;
299 +# }
300 + $localflag = 1;
301 + $WebMailsendtotal++;
302 + $counts{$abshour}{$CATWEBMAIL}++;
303 + $WebMailflag = 1;
304 + }
305
306 # see if from localhost
307 - elsif ( $log_items[1] =~ m/.*$localhost.*/ ) {
308 + elsif ( $log_items[1] =~ m/$localhost/ ) {
309 # but not if it comes from fetchmail
310 - if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { }
311 + if ( $log_items[3] =~ m/$FETCHMAIL/ ) { }
312 else {
313 + $localflag = 1;
314 # might still be from mailman here
315 - if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) {
316 + if ( $log_items[3] =~ m/$MAILMAN/ ) {
317 $mailmansendcount++;
318 $localsendtotal++;
319 $counts{$abshour}{$CATMAILMAN}++;
320 @@ -501,12 +590,10 @@
321 }
322 else {
323 #Or sent to the DMARC server
324 - dbg("LOG4:".$log_items[4]);
325 + #dbg("LOG4:".$log_items[4]);
326 #check for email address in $DMARC_Report_emails string
327 - #if ($log_items[4] =~ m/.*$DMARCDomain.*/) {
328 my $logemail = $log_items[4];
329 - #print STDERR "/",$log_items[4]."/\n";
330 - if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){
331 + if ((index($DMARC_Report_emails,$logemail)>=0) or ($logemail =~ m/$DMARCDomain/)){
332 $localsendtotal++;
333 $DMARCSendCount++;
334 $localflag = 1;
335 @@ -514,10 +601,11 @@
336 else {
337 #print STDERR "no match:.".$logemail;
338 if (exists $log_items[8]){
339 - dbg("LOG8:".$log_items[8]);
340 + #dbg("LOG8:".$log_items[8]);
341 # ignore incoming localhost spoofs
342 - if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
343 + if ( $log_items[8] =~ m/msg denied before queued/ ) { }
344 else {
345 + #Webmail
346 $localflag = 1;
347 $WebMailsendtotal++;
348 $counts{$abshour}{$CATWEBMAIL}++;
349 @@ -536,12 +624,12 @@
350 }
351
352 # try to spot fetchmail emails
353 - if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) {
354 - dbg("LOG0:".$log_items[0]);
355 + if ( $log_items[0] =~ m/$FetchmailIP/ ) {
356 + #dbg("LOG0:".$log_items[0]);
357 $localAccepttotal++;
358 $counts{$abshour}{$CATFETCHMAIL}++;
359 }
360 - elsif ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) {
361 + elsif ( $log_items[3] =~ m/$FETCHMAIL/ ) {
362 $localAccepttotal++;
363 $counts{$abshour}{$CATFETCHMAIL}++;
364 }
365 @@ -549,12 +637,12 @@
366 # and adjust for recipient field if not set-up by denying plugin - extract from deny msg
367
368 if ( length( $log_items[4] ) == 0 ) {
369 - dbg("LOG7:".$log_items[0]);
370 + #dbg("LOG7:".$log_items[0]);
371 if ( $log_items[5] eq 'check_goodrcptto' ) {
372 if ( $log_items[7] gt "invalid recipient" ) {
373 $log_items[4] =
374 substr( $log_items[7], 18 ); #Leave only email address
375 - dbg("LOG4:".$log_items[0]);
376 + #dbg("LOG4:".$log_items[0]);
377
378 }
379 }
380 @@ -563,7 +651,7 @@
381 # if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) {
382 # reduce to lc and process each e,mail if a list, pseperatedy commas
383 my $recipientmail = lc( $log_items[4] );
384 - dbg("LOG4:".$log_items[0]);
385 + #dbg("LOG4:".$log_items[0]);
386 if ( $recipientmail =~ m/.*,/ ) {
387
388 #comma - split the line and deal with each domain
389 @@ -599,146 +687,179 @@
390 # then categorise the result
391
392
393 - if (exists $log_items[5]) {
394 + if (exists $log_items[5]) {
395
396 - if ($log_items[5] eq 'naughty') {
397 + if ($log_items[5] eq 'naughty') {
398 my $rejreason = $log_items[7];
399 $rejreason = /.*(\(.*\)).*/;
400 - $rejreason = $1;
401 + if (!defined($1)){$rejreason = "unknown"}
402 + else {$rejreason = $1}
403 $found_qpcodes{$log_items[5]."-".$rejreason}++}
404 else {$found_qpcodes{$log_items[5]}++} ##Count different qpsmtpd result codes
405
406 - #Check for badly formed lines (from earlier testing)
407 -
408 - if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
409 + if ($log_items[5] eq 'check_earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
410
411 - elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
412 + elsif ($log_items[5] eq 'check_relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
413
414 - elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
415 + elsif ($log_items[5] eq 'check_norelay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
416
417 - elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
418 + elsif ($log_items[5] eq 'require_resolvable_fromhost') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
419
420 - elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
421 + elsif ($log_items[5] eq 'check_basicheaders') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
422
423 - elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
424 + elsif ($log_items[5] eq 'rhsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
425
426 - elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
427 + elsif ($log_items[5] eq 'dnsbl') { $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
428
429 - elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
430 + elsif ($log_items[5] eq 'check_badmailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
431
432 - elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
433 + elsif ($log_items[5] eq 'check_badrcptto_patterns') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
434
435 - elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
436 + elsif ($log_items[5] eq 'check_badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
437
438 - elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
439 + elsif ($log_items[5] eq 'check_spamhelo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
440
441 - elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
442 + elsif ($log_items[5] eq 'check_goodrcptto extn') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
443
444 - elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
445 + elsif ($log_items[5] eq 'rcpt_ok') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
446
447 - elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE}
448 + elsif ($log_items[5] eq 'pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)}
449
450 - elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc);next LINE}
451 + elsif ($log_items[5] eq 'virus::pattern_filter') { $PatternFilterCount++;$counts{$abshour}{$CATEXECUT}++;mark_domain_rejected($proc)}
452
453 - elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
454 + elsif ($log_items[5] eq 'check_goodrcptto') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
455
456 - elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
457 + elsif ($log_items[5] eq 'check_smtp_forward') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
458
459 - elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
460 + elsif ($log_items[5] eq 'count_unrecognized_commands') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
461
462 - elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc);next LINE}
463 + elsif ($log_items[5] eq 'check_badcountries') {$MiscDenyCount++;$counts{$abshour}{$CATBADCOUNTRIES}++;mark_domain_rejected($proc)}
464
465 - elsif ($log_items[5] eq 'tnef2mime') { next LINE} #Not expecting this one.
466 + elsif ($log_items[5] eq 'tnef2mime') { } #Not expecting this one.
467
468 - elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
469 - # and extract the spam score
470 -# if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)")
471 - if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)")
472 + elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
473 + # and extract the spam score
474 + # if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)")
475 + if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)")
476 {$rejectspamavg += $1}
477 - mark_domain_rejected($proc);
478 - next LINE
479 - }
480 + mark_domain_rejected($proc);
481 + }
482
483 - elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
484 - #extract the virus name
485 - if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
486 - else {$found_viruses{$log_items[7]}++} #Some other message!!
487 - dbg("LOG7:".$log_items[7]);
488 - mark_domain_rejected($proc);
489 - next LINE
490 - }
491 + elsif (($log_items[5] eq 'virus::clamav') or ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
492 + #extract the virus name
493 + if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
494 + else {$found_viruses{$log_items[7]}++} #Some other message!!
495 + #dbg("LOG7:".$log_items[7]);
496 + mark_domain_rejected($proc);
497 + }
498
499 - elsif ($log_items[5] eq 'queued') { $Accepttotal++;
500 - #extract the spam score
501 - if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") {
502 - $score = $1;
503 -# print $log_items[8]."<".$score.">\n";
504 - if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score}
505 - else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score}
506 - } else {
507 - # no SA score - treat it as ham
508 - $hamcount++;$counts{$abshour}{$CATHAM}++;
509 - }
510 - if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
511 - $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
512 - $currentrcptdomain{ $proc } = '' ;
513 - }
514 - next LINE
515 - }
516 -
517 + elsif ($log_items[5] eq 'queued') { $Accepttotal++;
518 + #extract the spam score
519 + # Remove count for rejectred as it looks as if it might get through!!
520 + $result= "queued";
521 + if ($log_items[8] =~ ".*score=([+-]?\\d+\.?\\d*).* required=([0-9\.]+)") {
522 + $score = trim($1);
523 + if ($score =~ /^[+-]?\d+\.?\d*$/ ) #check its numeric
524 + {
525 + if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score;}
526 + else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score;$result= "spam";}
527 + } else {
528 + print "Unexpected non numeric found in $proc:".$log_items[8]."($score)\n";
529 + }
530 + } else {
531 + # no SA score - treat it as ham
532 + $hamcount++;$counts{$abshour}{$CATHAM}++;
533 + }
534 + if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
535 + $byrcptdomain{ $currentrcptdomain{ $proc } }{ 'accept' }++ ;
536 + $currentrcptdomain{ $proc } = '' ;
537 + }
538 + }
539
540 - elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
541
542 - elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
543 -
544 - elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
545 -
546 - elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
547 -
548 - elsif ($log_items[5] eq 'naughty') {
549 + elsif ($log_items[5] eq 'tls') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
550 +
551 + elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
552 +
553 + elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
554 +
555 + elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
556 +
557 + elsif ($log_items[5] eq 'naughty') {
558 #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify
559 if ($log_items[7] =~ m/(karma)/) {
560 - $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
561 + $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)}
562 elsif ($log_items[7] =~ m/(dnsbl)/){
563 - $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
564 + $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);$blacklistURL{get_domain($log_items[7])}++}
565 elsif ($log_items[7] =~ m/(helo)/){
566 - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
567 + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
568 else {
569 #Unidentified Naughty rejection
570 - $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE}
571 + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++}
572 }
573 - elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
574 + elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
575
576 - elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE}
577 + elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc)}
578
579 - elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
580 + elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc)}
581
582 - elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE}
583 + elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc)}
584
585 - elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
586 + elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
587
588 - elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
589 + elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
590
591 - elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
592 + elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
593
594 - elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
595 -
596 - elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
597 + elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
598
599 - elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
600 + elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
601
602 - elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
603 + elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
604 +
605 + elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc)}
606
607 #Treat it as Unconf if not recognised
608 - else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE}
609 + else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++}
610 } #Log[5] exists
611
612 -
613 -# print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected
614 -# next LINE
615 -
616 -
617 -
618 + #Entry if not local send
619 + if ($localflag == 0) {
620 + if (length($log_items[4]) > 0){
621 + # Need to check here for multiple email addresses
622 + my @emails = split(",",lc($log_items[4]));
623 + if (scalar(@emails) > 1) {
624 + #Just pick the first local address to hang it on.
625 + # TEMP - just go for the first address until I can work out how to spot the 1st "local" one
626 + $usercounts{$emails[0]}{$result}++;
627 + $usercounts{$emails[0]}{"proc"} = $proc;
628 + #Compare with @domains array until we get a local one
629 + my $gotone = $false;
630 + foreach my $email (@emails){
631 + #Extract the domain from the email address
632 + my $fullemail = $email;
633 + $email = s/.*\@(.*)$/$1/;
634 + #and see if it is local
635 + if ($email =~ m/$alldomains/){
636 + $usercounts{lc($fullemail)}{$result}++;
637 + $usercounts{lc($fullemail)}{"proc"} = $proc;
638 + $gotone = $true;
639 + last;
640 + }
641 + }
642 + if (!$gotone) {
643 + $usercounts{'No internal email $proc'}{$result}++;
644 + $usercounts{'No internal email $proc'}{"proc"} = $proc;
645 + }
646 +
647 + } else {
648 + $usercounts{lc($log_items[4])}{$result}++;
649 + $usercounts{lc($log_items[4])}{"proc"} = $proc;
650 + }
651 + }
652 + }
653 + #exit if $emailnum == 15858;
654 +
655 } #END OF MAIN LOOP
656
657 #total up grand total Columns
658 @@ -750,7 +871,7 @@
659 $counts{$GRANDTOTAL}{$categs[$ncateg]} += $counts{$nhour}{$categs[$ncateg]};
660
661 # and total rows
662 - if ( $ncateg < $categlen && $ncateg>=$countfromhere) {#skip initial columns of non final reasons
663 + if ( $ncateg < $categlen and $ncateg>=$countfromhere) {#skip initial columns of non final reasons
664 $counts{$nhour}{$categs[@categs-2]} += $counts{$nhour}{$categs[$ncateg]};
665 }
666 $ncateg++
667 @@ -815,7 +936,7 @@
668 my $oldfh;
669
670 #Open Sendmail if we are mailing it
671 -if ( $opt{'mail'} && !$disabled ) {
672 +if ( $opt{'mail'} and !$disabled ) {
673 open( SENDMAIL, "|$opt{'sendmail'} -oi -t -odq" )
674 or die "Can't open sendmail: $!\n";
675 print SENDMAIL "From: $opt{'from'}\n";
676 @@ -841,44 +962,44 @@
677 my $oldFH = select $outputFH;
678
679
680 - print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n";
681 - print "----------------------------------------------------", "\n\n";
682 -
683 - print "$0 Version : $opt{'version'}", "\n\n";
684 - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n";
685 + print "SMEServer daily Anti-Virus and Spamfilter statistics from $hostname - ".strftime( "%F", localtime($start))."\n";
686 + print "----------------------------------------------------------------------------------", "\n\n";
687 + print "$0 Version : $opt{'version'}", "\n";
688 + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n";
689 print "Period Ending : ", strftime( "%c", localtime($end) ), "\n";
690 - print "\n";
691 -
692 - print "Clam Version/DB Count/Last DB update: ", `freshclam -V`."\n";
693 - print "SpamAssassin Version : ", `spamassassin -V`."\n";
694 - printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel;
695 + print "Clam Version/DB Count/Last DB update: ",`freshclam -V`;
696 + print "SpamAssassin Version : ",`spamassassin -V`;
697 + printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,$SARejectLevel;
698 if ($HighLogLevel) {
699 printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n";
700 printf "\tYou can set it this way:\n";
701 printf "\tconfig setprop qpsmtpd LogLevel 6\n";
702 printf "\tsignal-event email-update\n";
703 - printf "\tsv t /var/service/qpsmtpd\n\n";
704 + printf "\tsv t /var/service/qpsmtpd\n";
705 }
706 - print "\n\n";
707 printf "Reporting Period : %.2f hrs\n", $hrsinperiod;
708 - #print "----------------------------\n";
709 - #print "\n";
710 -
711 printf "All SMTP connections accepted:%8d \n", $totalexamined;
712 -
713 printf "Emails per hour : %8.1f/hr\n", $emailperhour || 0;
714 - print "\n";
715 printf "Average spam score (accepted): %11.2f\n", $spamavg || 0;
716 printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0;
717 printf "Average ham score : %11.2f\n", $hamavg || 0;
718 - printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount || 0;
719 - if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
720 + printf "Number of DMARC reporting emails sent:\t%11d (not shown on table)\n", $DMARCSendCount || 0;
721 + if ($hamcount != 0){ printf "Number of emails approved through DMARC:\t%11d (%3d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
722 +
723 + my $smeoptimizerprog = "/usr/local/smeoptimizer/SMEOptimizer.pl";
724 + if (-e $smeoptimizerprog) {
725 + #smeoptimizer installed - get result of status
726 + my @smeoptimizerlines = split(/\n/,`/usr/local/smeoptimizer/SMEOptimizer.pl -status`);
727 + print("SMEOptimizer status:\n");
728 + print("\t".$smeoptimizerlines[6]."\n");
729 + print("\t".$smeoptimizerlines[7]."\n");
730 + print("\t".$smeoptimizerlines[8]."\n");
731 + print("\t".$smeoptimizerlines[9]."\n");
732 + print("\t".$smeoptimizerlines[10]."\n");
733 + }
734 +
735
736 - print "\n\n";
737 - print "\nStatistics by Hour\n";
738 - print "-------------------\n";
739 - #print "\n";
740 -
741 + print "\nStatistics by Hour:\n";
742 #
743 # start by working out which colunns to show - tag the display array
744 #
745 @@ -968,24 +1089,17 @@
746 #
747 # print it.
748 #
749 - my $makeHTMLemail = "no";
750 - #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
751 - my $makeHTMLpage = "no";
752 - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
753 - #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
754 -
755 - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion ....
756 - print $Titles."\n";
757 - if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto
758 - #$Line2 =~ s/-/a/g;
759 - #print $Line2."\n";
760 - #print "\n";
761 - print $Values."\n";
762 +
763 + print $Line1."\n";
764 + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion ....
765 + print $Titles."\n";
766 + #if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto
767 + print $Line2."\n";
768 + print $Values;
769 print $Line2."\n";
770 print $Totals."\n";
771 print $Percent."\n";
772 print $Line1."\n";
773 - print "\n";
774
775 if ($localAccepttotal>0) {
776 print "*Fetchml* means connections from Fetchmail delivering email\n";
777 @@ -1021,7 +1135,7 @@
778 }
779
780
781 - if ( !$RHSenabled || !$DNSenabled ) {
782 + if ( !$RHSenabled or !$DNSenabled ) {
783
784 # comment about RBL not set
785 print
786 @@ -1114,11 +1228,15 @@
787
788 if ($enableSARules) {show_SARules_codes();}
789
790 - if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();}
791 + if ($enableGeoiptable and $finaldisplay[$BadCountryCateg]){show_Geoip_results();}
792
793 if ($enablejunkMailList) {List_Junkmail();}
794 +
795 + if ($enableblacklist) {show_blacklist_counts();}
796 +
797 + show_user_stats();
798
799 - print "\nDone. Report generated in $telapsed sec.\n\n";
800 + print "\nReport generated in $telapsed sec.\n";
801
802 if ($savedata) { save_data(); }
803 else
804 @@ -1128,25 +1246,26 @@
805
806 select $oldFH;
807 close $outputFH;
808 - if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr}
809 - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){
810 + if ($makeHTMLemail eq "no" or $makeHTMLemail eq "both") {print $tablestr}
811 + if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" or $makeHTMLpage eq "yes"){
812 #Convert text to html and send it
813 require CGI;
814 require TextToHTML;
815 my $cgi = new CGI;
816 my $text = $tablestr;
817 - print $cgi->header();
818 my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20);
819 my $conv = new HTML::TextToHTML();
820 $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2,
821 underline_length_tolerance=>1);
822 - my $html="<!DOCTYPE html> <html>\n";
823 +
824 + my $html = $cgi->header();
825 + $html .="<!DOCTYPE html> <html>\n";
826 $html .= "<head><title>Mailstats -".strftime( "%F", localtime($start) )."</title>";
827 $html .= "<link rel='stylesheet' type='text/css' href='mailstats.css' /></head>\n";
828 $html .= "<body>\n";
829 $html .= $conv->process_chunk($text);
830 $html .= "</body></html>\n";
831 - if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html}
832 + if ($makeHTMLemail eq "yes" or $makeHTMLemail eq "both" ) {print $html}
833 #And drop it into a file
834 if ($makeHTMLpage eq "yes") {
835 my $filename = "mailstats.html";
836 @@ -1206,7 +1325,7 @@
837 } elsif ($base =~m/\d+/){
838 $sec=0;$min=0;$hour=$base;
839 };
840 - #$mday="17"; #$mday="03"; #$mday="16"; #Temp!!
841 + #$mday="05"; #$mday="03"; #$mday="16"; #Temp!!
842 $time = timelocal($sec,$min,$hour,$mday,$mon,$year);
843 }
844
845 @@ -1255,10 +1374,9 @@
846 }
847 my $i = keys %junkcount;
848 if ( $i > 0 ) {
849 - print "\n\n";
850 print("\nJunk Mails left in folder:\n");
851 - print("---------------------------\n\n");
852 - print("\nCount\tUser\n");
853 + print("---------------------------\n");
854 + print("Count\tUser\n");
855 print("-------------------------\n");
856 foreach my $thisuser (
857 sort { $junkcount{$b} <=> $junkcount{$a} }
858 @@ -1282,20 +1400,20 @@
859 #
860
861 {
862 -
863 - print("Virus Statistics by name:\n");
864 - print("------------------------------------------------------------------------\n");
865 + my $line = "------------------------------------------------------------------------\n";
866 + print("\nVirus Statistics by name:\n");
867 + print($line);
868 foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} }
869 keys %found_viruses)
870 {
871 - if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){
872 + if (index($virus,"Sanesecurity") !=-1 or index($virus,"UNOFFICIAL") !=-1){
873 print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n";
874 } else {
875 print "Rejected $found_viruses{$virus}\t$virus\n";
876 }
877
878 }
879 - print("------------------------------------------------------------------------\n\n");
880 + print($line);
881 }
882
883 sub show_qpsmtpd_codes
884 @@ -1305,18 +1423,94 @@
885 #
886
887 {
888 -
889 - print("Qpsmtpd codes league table:\n");
890 - print("---------------------------------------------\n");
891 - print("Count\tPercent\tReason\t\n");
892 - print("---------------------------------------------\n");
893 + my $line = "---------------------------------------------\n";
894 + print("\nQpsmtpd codes league table:\n");
895 + print($line);
896 + print("Count\tPercent\tReason\n");
897 + print($line);
898 foreach my $qpcode (sort { $found_qpcodes{$b} <=> $found_qpcodes{$a} }
899 keys %found_qpcodes)
900 {
901 - print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined;
902 + print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t\t$qpcode\n" if $totalexamined;
903 + }
904 + print($line);
905 +}
906 +
907 +sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
908 +
909 +sub get_domain
910 +{ #my $url= URI->new(shift);
911 + #my $domain = $url->host;
912 + my $url = shift;
913 + $url =~ s!^.*https?://(?:www\.)?!!i;
914 + $url =~ s!/.*!!;
915 + $url =~ s/[\?\#\:].*//;
916 + my $domain = trim($url);
917 + return $domain;
918 +}
919 +
920 +sub show_blacklist_counts
921 +
922 +#
923 +# Show a sorted league table of the blacklist URL counts
924 +#
925 +
926 +{
927 + my $line = "------------------\n";
928 + print("\nBlacklist details:\n");
929 + print($line);
930 + if ($cdb->get('qpsmtpd')->prop("RHSBL") eq "enabled") {print "RBLLIST:".$cdb->get('qpsmtpd')->prop("RBLList")."\n";}
931 + if ($cdb->get('qpsmtpd')->prop("URIBL") eq "enabled") {print "UBLLIST:".$cdb->get('qpsmtpd')->prop("UBLList")."\n";}
932 + if (!$cdb->get('qpsmtpd')->prop("SBLList") eq "") {print "SBLLIST:".$cdb->get('qpsmtpd')->prop("SBLList")."\n";}
933 + print($line);
934 + print("Count\tURL\n");
935 + print($line);
936 + foreach my $blcode (sort { $blacklistURL{$b} <=> $blacklistURL{$a} }
937 + keys %blacklistURL)
938 + {
939 + print sprintf('%3u',$blacklistURL{$blcode})."\t$blcode\n";
940 }
941 - print("---------------------------------------------\n\n");
942 - print "\n\n";
943 + print($line);
944 +}
945 +
946 +
947 +sub show_user_stats
948 +
949 +#
950 +# Show a sorted league table of the user counts
951 +#
952 +
953 +{
954 + #Compute totals for each entry
955 + my $grandtotals=0;
956 + my $totalqueued=0;
957 + my $totalspam=0;
958 + my $totalrejected=0;
959 + foreach my $user (keys %usercounts){
960 + $usercounts{$user}{"queued"} = 0 if !(exists $usercounts{$user}{"queued"});
961 + $usercounts{$user}{"rejected"} = 0 if !(exists $usercounts{$user}{"rejected"});
962 + $usercounts{$user}{"spam"} = 0 if !(exists $usercounts{$user}{"spam"});
963 + $usercounts{$user}{"totals"} = $usercounts{$user}{"queued"}+$usercounts{$user}{"rejected"}+$usercounts{$user}{"spam"};
964 + $grandtotals += $usercounts{$user}{"totals"};
965 + $totalspam += $usercounts{$user}{"spam"};
966 + $totalqueued += $usercounts{$user}{"queued"};
967 + $totalrejected += $usercounts{$user}{"rejected"};
968 + }
969 + my $line = "--------------------------------------------------\n";
970 + print("\nStatistics by email address received:\n");
971 + print($line);
972 + print("Queued\tRejected\tSpam tagged\tEmail Address\n");
973 + print($line);
974 + foreach my $user (sort { $usercounts{$b}{"totals"} <=> $usercounts{$a}{"totals"} }
975 + keys %usercounts)
976 + {
977 + print sprintf('%3u',$usercounts{$user}{"queued"})."\t".sprintf('%3u',$usercounts{$user}{"rejected"})."\t\t".sprintf('%3u',$usercounts{$user}{"spam"})."\t\t$user\n";
978 + }
979 + print($line);
980 + print sprintf('%3u',$totalqueued)."\t".sprintf('%3u',$totalrejected)."\t\t".sprintf('%3u',$totalspam)."\n";
981 + print($line);
982 +
983 +
984 }
985
986 sub show_Geoip_results
987 @@ -1335,10 +1529,11 @@
988 $percentthreshold = 0.5;
989 }
990 if ($total_countries > 0) {
991 - print("Geoip results: (cutoff at $percentthreshold%) \n");
992 - print("---------------------------------\n");
993 + my $line = "---------------------------------------------\n";
994 + print("\nGeoip results: (cutoff at $percentthreshold%) \n");
995 + print($line);
996 print("Country\tPercent\tCount\tRejected?\n");
997 - print("---------------------------------\n");
998 + print($line);
999 foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
1000 keys %found_countries)
1001 {
1002 @@ -1347,14 +1542,14 @@
1003 $totalpercent = $totalpercent + $percent;
1004 if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
1005 if ( $percent >= $percentthreshold ) {
1006 - print "$country\t"
1007 + print "$country\t\t"
1008 . sprintf( '%4.1f', $percent )
1009 - . "%\t$found_countries{$country}","\t$reject\n"
1010 + . "%\t\t$found_countries{$country}","\t$reject\n"
1011 if $total_countries;
1012 }
1013
1014 }
1015 - print("---------------------------------\n");
1016 + print($line);
1017 my ($showtotals);
1018 if ($cdb->get('mailstats')){
1019 $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
1020 @@ -1363,10 +1558,9 @@
1021 }
1022
1023 if ($showtotals){
1024 - print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n";
1025 - print("---------------------------------\n\n");
1026 + print "TOTALS\t\t".sprintf("%4.1f",$totalpercent)."%\t\t$total_countries\n";
1027 + print($line);
1028 }
1029 - print "\n";
1030 }
1031 }
1032
1033 @@ -1384,7 +1578,7 @@
1034
1035 if ($sum_SARules > 0){
1036
1037 - if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
1038 + if ($totalexamined >0 and $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
1039 $defaultpercentthreshold = $maxcutoff
1040 } else {
1041 $defaultpercentthreshold = $mincutoff
1042 @@ -1394,17 +1588,15 @@
1043 } else {
1044 $percentthreshold = $defaultpercentthreshold
1045 }
1046 -
1047 - print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
1048 - print("---------------------------------------------\n");
1049 + my $line = "---------------------------------------------\n";
1050 + print("\nSpamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
1051 + print($line);
1052 print("Count\tPercent\tScore\t\t\n");
1053 - print("---------------------------------------------\n");
1054 + print($line);
1055 foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
1056 keys %found_SARules)
1057 {
1058 - my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
1059 - if $totalexamined;
1060 - #$totalpercent = $totalpercent + $percent;
1061 + my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined if $totalexamined;
1062 my $avehits = $found_SARules{$SARule}{'totalhits'} /
1063 $found_SARules{$SARule}{'count'}
1064 if $found_SARules{$SARule}{'count'};
1065 @@ -1416,7 +1608,7 @@
1066 if $totalexamined;
1067 }
1068 }
1069 - print("---------------------------------------------\n");
1070 + print($line);
1071 my ($showtotals);
1072 if ($cdb->get('mailstats')){
1073 $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
1074 @@ -1426,7 +1618,7 @@
1075
1076 if ($showtotals){
1077 print "$totalexamined\t(TOTALS)\n";
1078 - print("---------------------------------------------\n");
1079 + print($line);
1080 }
1081 print "\n";
1082 }
1083 @@ -1631,15 +1823,25 @@
1084 }
1085 $nhour++;
1086 }
1087 - # and write out the log lines saved
1088 -
1089 - foreach my $logid (keys %LogLines){
1090 -
1091 - $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')");
1092 + # and write out the log lines saved - only if html wanted
1093 + if ($makeHTMLemail eq 'yes' or $makeHTMLemail eq 'both' or $makeHTMLpage eq 'yes'){
1094 + foreach my $logid (keys %LogLines){
1095 + $reccount++;
1096 + #Extract from keys
1097 + my $extract = $logid;
1098 + $extract =~/^(.*)-(.*):(.*)$/;
1099 + my $Log64n = $1;
1100 + my $LogMailId = $2;
1101 + my $LogSeq = $3;
1102 + my $LogLine = $dbh->quote($LogLines{$logid});
1103 + my $sql = "INSERT INTO LogData (Log64n,MailID,Sequence,LogStr) VALUES ('";
1104 + $sql .= $Log64n."','".$LogMailId."','".$LogSeq."',".$LogLine.")";
1105 + $dbh->do($sql) or die($sql);
1106 + }
1107 + $dbh->disconnect();
1108 + $telapsed = time - $tstart;
1109 + print "Saved $reccount records in $telapsed sec.";
1110 }
1111 - $dbh->disconnect();
1112 - my $telapsed = time - $tstart;
1113 - print "Saved $reccount records in $telapsed sec.";
1114 }
1115
1116 sub check_date_rec
1117 @@ -1706,3 +1908,47 @@
1118 my $daterec = $sth->fetchrow_hashref();
1119 $daterec->{"dateid"};
1120 }
1121 +
1122 + sub dump_entries
1123 + {
1124 + my $msg = shift;
1125 + #dbg($msg);
1126 + #dbg("TM0:".$timestamp_items[0]);
1127 + #dbg("TM1:".$timestamp_items[1]);
1128 + #dbg("TM2:".$timestamp_items[2]);
1129 + #dbg("TM3:".$timestamp_items[3]);
1130 + #dbg("LOG0:".$log_items[0]);
1131 + #dbg("LOG1:".$log_items[1]);
1132 + #dbg("LOG2:".$log_items[2]);
1133 + #dbg("LOG3:".$log_items[3]);
1134 + #dbg("LOG4:".$log_items[4]);
1135 + #dbg("LOG5:".$log_items[5]);
1136 + #dbg("LOG6:".$log_items[6]);
1137 + #dbg("LOG7:".$log_items[7]);
1138 + #if ($opt{debug} == 1){exit;}
1139 +}
1140 +
1141 +#sub test_for_private_ip {
1142 + #use NetAddr::IP;
1143 + #my $ip = shift;
1144 + #$ip =~ s/^\D*(([0-9]{1,3}\.){3}[0-9]{1,3}).*/$1/e;
1145 + #print "\nIP:$ip";
1146 + #my $nip = NetAddr::IP->new($ip);
1147 + #if ($nip){
1148 + #if ( $nip->is_rfc1918() ){
1149 + #return 1;
1150 + #} else { return 0}
1151 + #} else { return 0}
1152 +#}
1153 +
1154 +
1155 +sub test_for_private_ip {
1156 + use NetAddr::IP;
1157 + $_ = shift;
1158 + return unless /(\d+\.\d+\.\d+\.\d+)/;
1159 + my $ip = NetAddr::IP->new($1);
1160 + return unless $ip;
1161 + return $ip->is_rfc1918();
1162 +}
1163 +
1164 +

admin@koozali.org
ViewVC Help
Powered by ViewVC 1.2.1 RSS 2.0 feed