/[smecontribs]/rpms/smeserver-mailstats/contribs10/smeserver-mailstats-1.1.bz9588.qpsmtpd0_96compatible.patch
ViewVC logotype

Contents of /rpms/smeserver-mailstats/contribs10/smeserver-mailstats-1.1.bz9588.qpsmtpd0_96compatible.patch

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (show annotations) (download)
Mon Apr 13 18:12:07 2020 UTC (4 years, 6 months ago) by brianr
Branch: MAIN
CVS Tags: smeserver-mailstats-1_1-16_el7_sme, smeserver-mailstats-1_1-13_el7_sme, smeserver-mailstats-1_1-17_el7_sme, smeserver-mailstats-1_1-14_el7_sme, smeserver-mailstats-1_1-11_el7_sme, smeserver-mailstats-1_1-12_el7_sme, smeserver-mailstats-1_1-15_el7_sme, HEAD
Initial import

1 diff -Nur smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl
2 --- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl 2016-07-02 04:29:21.671000000 -0400
3 +++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl 2016-07-02 04:39:34.037000000 -0400
4 @@ -18,6 +18,8 @@
5 # bjr - 19Jun15 - Add totals for the League tables
6 # bjr and Unnilennium - 08Apr16 - Add in else for unrecognised plugin detection
7 # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection
8 +# bjr - 14Jun16 - make compatible with qpsmtpd 0.96
9 +# bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7)
10 #
11 #############################################################################
12 #
13 @@ -37,8 +39,10 @@
14 # / ShowLeagueTotals - Show totals row after league tables - (default is "yes")
15 # / DBHost - MySQL server hostname (default is "localhost").
16 # / DBPort - MySQL server post (default is "3306")
17 -# / Interval - "day", "week", "fortnight", "month", "99999" - last is number of seconds (default is day)
18 +# / Interval - "daily", "weekly", "fortnightly", "monthly", "99999" - last is number of hours (default is daily)
19 # / Base - "Midnight", "Midday", "Now", "99" hour (0-23) (default is midnight)
20 +# / HTMLEmail - "yes", "no", "both" - default is "No" - Send email in HTML
21 +# / HTMLPage - "yes" / "no" - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no"
22 #
23 #############################################################################
24
25 @@ -55,6 +59,10 @@
26 use esmith::DomainsDB;
27 use Sys::Hostname;
28 use Switch;
29 +use DBIx::Simple;
30 +
31 +#use CGI;
32 +#use HTML::TextToHTML;
33
34 my $hostname = hostname();
35 my $cdb = esmith::ConfigDB->open_ro or die "Couldn't open ConfigDB : $!\n";
36 @@ -73,12 +81,11 @@
37
38 #Configuration section
39 my %opt = (
40 - version => '0.6.29', # please update at each change.
41 + version => '0.7.5a', # please update at each change.
42 debug => 0, # guess what ?
43 sendmail => '/usr/sbin/sendmail', # Path to sendmail stub
44 from => 'spamfilter-stats', # Who is the mail from
45 - mail => # mailstats email recipient
46 - $cdb->get('mailstats')->prop('Email') || 'admin',
47 + mail => $cdb->get('mailstats')->prop('Email') || 'admin', # mailstats email recipient
48 timezone => `date +%z`,
49 );
50
51 @@ -87,8 +94,10 @@
52 my $localhost = 'localhost'; #Apparent sender for webmail
53 my $FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email
54 my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost
55 +my $DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
56 +my $DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval
57
58 -my $MinCol = 8; #Minimum column width
59 +my $MinCol = 6; #Minimum column width
60 my $HourColWidth = 16; #Date and time column width
61
62 my $SARulethresholdPercent = 10; #If Sa rules less than this of total emails, then cutoff reduced
63 @@ -116,6 +125,10 @@
64 my $totalexamined = 0; #total download + RBL etc
65 my $WebMailsendtotal = 0; #total from Webmail
66 my $mailmansendcount = 0; #total from mailman
67 +my $DMARCSendCount = 0; #total DMARC reporting emails sent (approx)
68 +my $DMARCOkCount = 0; #Total emails approved through DMARC
69 +
70 +
71
72 my %found_viruses = ();
73 my %found_qpcodes = ();
74 @@ -137,25 +150,35 @@
75 my $CATMAILMAN='Mailman';
76 my $CATLOCAL='Local';
77 # border between where it came from and where it ended..
78 -my $countfromhere = 5;
79 -
80 +my $countfromhere = 5; #Temp - Check this not moved!!
81 +
82 my $CATVIRUS='Virus';
83 my $CATRBLDNS='RBL/DNS';
84 my $CATEXECUT='Execut.';
85 my $CATNONCONF='Non.Conf.';
86 my $CATBADCOUNTRIES='Geoip.';
87 -my $BadCountryCateg=8; #Careful here this number could change if more added before.
88 +my $CATKARMA="Karma";
89 +
90 my $CATSPAMDEL='Del.Spam';
91 my $CATSPAM='Qued.Spam?';
92 my $CATHAM='Ham';
93 my $CATTOTALS='TOTALS';
94 my $CATPERCENT='PERCENT';
95 -my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
96 +my $CATDMARC="DMARC Rej.";
97 +my $CATLOAD="Rej.Load";
98 +my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
99 my $GRANDTOTAL = '99'; #subs for count arrays, for grand total
100 my $PERCENT = '98'; # for column percentages
101
102 my $categlen = @categs-2; #-2 to avoid the total and percent column
103
104 +#
105 +# Index for certain columns - check these do not move if we add columns
106 +#
107 +my $BadCountryCateg=9;
108 +my $DMARCcateg = 5; #Not used.
109 +my $KarmaCateg=$BadCountryCateg+3;
110 +
111 my $above15 = 0;
112 my $RBLcount = 0;
113 my $MiscDenyCount = 0;
114 @@ -187,6 +210,38 @@
115 my $morethanonercpt = 0 ; # count every 'second' recipients for a mail.
116 my $recipcount = 0; # count every recipient email address received.
117
118 +#
119 +#Load up the emails curreently stored for DMARC reporting - so that we cna spot the reports being sent.
120 +#Held in an slqite db, created by the DMARC perl lib.
121 +#
122 +my $dsn = "dbi:SQLite:dbname=/var/lib/qpsmtpd/dmarc/reports.sqlite"; #Taken from /etc/mail-dmarc.ini
123 +# doesn't seem to need
124 +my $user = "";
125 +my $pass = "";
126 +my $DMARC_Report_emails = ""; #Flat string of all email addresses
127 +
128 + if (my $dbix = DBIx::Simple->connect( $dsn, $user, $pass )){
129 + my $result = $dbix->query("select rua from report_policy_published;");
130 + $result->bind(my ($emailaddress));
131 + while ($result->fetch){
132 + #print STDERR "$emailaddress";
133 + #remember email from logterse entry has chevrons round it - so we add them here to guarantee the alighment of the match
134 + #Remove the mailto:
135 + $emailaddress =~ s/mailto://g;
136 + # and map any commas to ><
137 + $emailaddress =~ s/,/></g;
138 + $DMARC_Report_emails .= "<".$emailaddress.">\n"
139 + }
140 + $dbix->disconnect();
141 + } else { $DMARC_Report_emails = "None found - DB not opened"}
142 +
143 +
144 +#dbg("DMARC-EMAILS:".$DMARC_Report_emails);
145 +
146 +# Saving the Log lines processed
147 +my %LogLines = (); #Save all the log lines processed for writing to the DB
148 +my %LogId = (); #Save the Log Ids.
149 +my $CurrentLogId = "";
150
151 # store the domain of interest. Every other records are stored in a 'Other' zone
152 my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n";
153 @@ -212,6 +267,8 @@
154
155 my ( $start, $end ) = analysis_period();
156
157 +dbg("Time interval:".strftime("%a %b %e %H:%M:%S %Y", localtime($start))."->".strftime("%a %b %e %H:%M:%S %Y", localtime($end))."\n");
158 +
159 #
160 # First check current configuration for logging, DNS enable and Max threshold for spamassassin
161 #
162 @@ -243,6 +300,27 @@
163
164 }
165
166 +# get enable/disable subsections
167 +my $enableqpsmtpdcodes;
168 +my $enableSARules;
169 +my $enableGeoiptable;
170 +my $enablejunkMailList;
171 +my $savedata;
172 +if ($cdb->get('mailstats')){
173 + $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
174 + $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
175 + $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false;
176 + $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false;
177 + $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false;
178 + } else {
179 + $enableqpsmtpdcodes = $true;
180 + $enableSARules = $true;
181 + $enablejunkMailList = $true;
182 + $enableGeoiptable = $true;
183 + $savedata = $false;
184 + }
185 + $savedata = $false; #TEMP!!
186 +
187 #
188 #---------------------------------------
189 # Scan the qpsmtpd log file(s)
190 @@ -263,6 +341,7 @@
191 }
192 # and grand totals, percent and display status from db entries, and column widths
193 $ncateg = 0;
194 +my $colpadding = 0;
195 while ( $ncateg < @categs) {
196 $counts{$GRANDTOTAL}{$categs[$ncateg]} = 0;
197 $counts{$PERCENT}{$categs[$ncateg]} = 0;
198 @@ -273,11 +352,11 @@
199 $display[$ncateg] = 'auto'
200 }
201 if ($ncateg == 0) {
202 - $colwidth[$ncateg] = $HourColWidth
203 + $colwidth[$ncateg] = $HourColWidth + $colpadding;
204 } else {
205 - $colwidth[$ncateg] = length($categs[$ncateg])+1
206 + $colwidth[$ncateg] = length($categs[$ncateg])+1+$colpadding;
207 }
208 - if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol}
209 + if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol + $colpadding}
210 $ncateg++
211 }
212
213 @@ -292,39 +371,91 @@
214 }
215 @ARGV=@ARGV2;
216
217 +my $count = -1; #for loop reduction in debugging mode
218 +
219 +my $CurrentMailId = "";
220 +
221 LINE: while (<>) {
222 - my($tai,$log) = split(' ',$_,2);
223
224 + #print STDERR $starttai,$endtai,$_,"\n";
225 +
226
227 + next LINE if !(my($tai,$log) = split(' ',$_,2));
228 + #dbg("TAI:".$tai);
229 +
230 + #dbg("REST1:".$log);
231 +
232 #If date specified, only process lines matching date
233 next LINE if ( $tai lt $starttai );
234 next LINE if ( $tai gt $endtai );
235
236 + #Count lines and skip out if debugging
237 + $count++;
238 + last LINE if ($opt{debug} && $count >= 100000);
239 + #dbg("REST:".$log);
240 +
241 + #Loglines to Saved String for later DB write
242 + if ($savedata) {
243 + my $CurrentLine = $_;
244 + $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/;
245 + if ($2 ne $CurrentMailId) {
246 + $CurrentLogId = $1."-".$2;
247 + $CurrentMailId = $2;
248 + }
249 + $LogLines{$CurrentLogId} = $_;
250 + #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
251 + }
252 +
253 + #Count lines and skip out if debugging
254 + $count++;
255 + last LINE if ($opt{debug} && $count >= 100);
256 + #dbg("REST:".$log);
257 +
258 +
259 # pull out spamasassin rule lists
260 - if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
261 + if ( $_ =~m/spamassassin: pass, Ham,(.*)</ )
262 + #if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
263 {
264 - my (@SAtests) = split(',',$2);
265 - foreach my $SAtest (@SAtests) {
266 - if (!$SAtest eq "") {
267 - $found_SARules{$SAtest}{'count'}++;
268 - $found_SARules{$SAtest}{'totalhits'} += $1;
269 - $sum_SARules++
270 - }
271 - }
272 -
273 + dbg("SPAM:".$log);
274 +
275 +
276 + #New version does not seem to have spammassasin tests in logs
277 +
278 + #if (exists($2){
279 + #my (@SAtests) = split(',',$2);
280 + #foreach my $SAtest (@SAtests) {
281 + #if (!$SAtest eq "") {
282 + #$found_SARules{$SAtest}{'count'}++;
283 + #$found_SARules{$SAtest}{'totalhits'} += $1;
284 + #$sum_SARules++
285 + #}
286 + #}
287 + #}
288 +
289 }
290 -
291 +
292 +
293 #Pull out Geoip countries for analysis table
294 - if ( $_ =~m/check_badcountries plugin \(connect\): GeoIP Country: (.*)/ )
295 + if ( $_ =~m/check_badcountries: GeoIP Country: (.*)/ )
296 {
297 $found_countries{$1}++;
298 $total_countries++;
299 }
300 +
301 + #Pull out DMARC approvals
302 + if ( $_ =~m/.*$DMARCOkPattern.*/ )
303 + {
304 + $DMARCOkCount++;
305 + }
306 +
307
308 #only select Logterse output
309 - next LINE unless m/terse plugin/;
310 -
311 -
312 + next LINE unless m/logging::logterse:/;
313 +
314 + #Count lines and skip out if debugging
315 + $count++;
316 + last LINE if ($opt{debug} && $count >= 100000);
317 + #dbg("REST:".$log);
318
319 my $abstime = Time::TAI64::tai2unix($tai);
320 my $abshour = floor( $abstime / 3600 ); # Hours since the epoch
321 @@ -342,6 +473,9 @@
322
323 $totalexamined++;
324
325 + #dbg("LOG1:".$log_items[1]);
326 + #dbg("LOG3:".$log_items[3]);
327 +
328 # first spot the fetchmail and local deliveries.
329
330 # Spot from local workstation
331 @@ -355,11 +489,9 @@
332
333 # see if from localhost
334 elsif ( $log_items[1] =~ m/.*$localhost.*/ ) {
335 -
336 # but not if it comes from fetchmail
337 if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { }
338 else {
339 -
340 # might still be from mailman here
341 if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) {
342 $mailmansendcount++;
343 @@ -368,21 +500,44 @@
344 $localflag = 1;
345 }
346 else {
347 -
348 - # eliminate incoming localhost spoofs
349 - if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
350 - else {
351 - $localflag = 1;
352 - $WebMailsendtotal++;
353 - $counts{$abshour}{$CATWEBMAIL}++;
354 - $WebMailflag = 1;
355 - }
356 + #Or sent to the DMARC server
357 + dbg("LOG4:".$log_items[4]);
358 + #check for email address in $DMARC_Report_emails string
359 + #if ($log_items[4] =~ m/.*$DMARCDomain.*/) {
360 + my $logemail = $log_items[4];
361 + #print STDERR "/",$log_items[4]."/\n";
362 + if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){
363 + $localsendtotal++;
364 + $DMARCSendCount++;
365 + $localflag = 1;
366 + }
367 + else {
368 + #print STDERR "no match:.".$logemail;
369 + if (exists $log_items[8]){
370 + dbg("LOG8:".$log_items[8]);
371 + # ignore incoming localhost spoofs
372 + if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
373 + else {
374 + $localflag = 1;
375 + $WebMailsendtotal++;
376 + $counts{$abshour}{$CATWEBMAIL}++;
377 + $WebMailflag = 1;
378 + }
379 + }
380 + else {
381 + $localflag = 1;
382 + $WebMailsendtotal++;
383 + $counts{$abshour}{$CATWEBMAIL}++;
384 + $WebMailflag = 1;
385 + }
386 + }
387 }
388 }
389 }
390
391 # try to spot fetchmail emails
392 if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) {
393 + dbg("LOG0:".$log_items[0]);
394 $localAccepttotal++;
395 $counts{$abshour}{$CATFETCHMAIL}++;
396 }
397 @@ -394,10 +549,13 @@
398 # and adjust for recipient field if not set-up by denying plugin - extract from deny msg
399
400 if ( length( $log_items[4] ) == 0 ) {
401 + dbg("LOG7:".$log_items[0]);
402 if ( $log_items[5] eq 'check_goodrcptto' ) {
403 if ( $log_items[7] gt "invalid recipient" ) {
404 $log_items[4] =
405 - substr( $log_items[7], 18 ) #Leave only email address
406 + substr( $log_items[7], 18 ); #Leave only email address
407 + dbg("LOG4:".$log_items[0]);
408 +
409 }
410 }
411 }
412 @@ -405,6 +563,7 @@
413 # if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) {
414 # reduce to lc and process each e,mail if a list, pseperatedy commas
415 my $recipientmail = lc( $log_items[4] );
416 + dbg("LOG4:".$log_items[0]);
417 if ( $recipientmail =~ m/.*,/ ) {
418
419 #comma - split the line and deal with each domain
420 @@ -442,7 +601,12 @@
421
422 if (exists $log_items[5]) {
423
424 - $found_qpcodes{$log_items[5]}++; ##Count different qpsmtpd result codes
425 + if ($log_items[5] eq 'naughty') {
426 + my $rejreason = $log_items[7];
427 + $rejreason = /.*(\(.*\)).*/;
428 + $rejreason = $1;
429 + $found_qpcodes{$log_items[5]."-".$rejreason}++}
430 + else {$found_qpcodes{$log_items[5]}++} ##Count different qpsmtpd result codes
431
432 #Check for badly formed lines (from earlier testing)
433
434 @@ -488,27 +652,31 @@
435
436 elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
437 # and extract the spam score
438 - if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1}
439 +# if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)")
440 + if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)")
441 + {$rejectspamavg += $1}
442 mark_domain_rejected($proc);
443 next LINE
444 }
445
446 - elsif ($log_items[5] eq 'virus::clamav') { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
447 + elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
448 #extract the virus name
449 - if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;}
450 + if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
451 + else {$found_viruses{$log_items[7]}++} #Some other message!!
452 + dbg("LOG7:".$log_items[7]);
453 mark_domain_rejected($proc);
454 next LINE
455 }
456
457 elsif ($log_items[5] eq 'queued') { $Accepttotal++;
458 #extract the spam score
459 - if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") {
460 + if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") {
461 $score = $1;
462 # print $log_items[8]."<".$score.">\n";
463 if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score}
464 else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score}
465 } else {
466 - # no SA score - so it must be ham
467 + # no SA score - treat it as ham
468 $hamcount++;$counts{$abshour}{$CATHAM}++;
469 }
470 if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
471 @@ -523,15 +691,53 @@
472
473 elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
474
475 + elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
476 +
477 + elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
478 +
479 + elsif ($log_items[5] eq 'naughty') {
480 + #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify
481 + if ($log_items[7] =~ m/(karma)/) {
482 + $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
483 + elsif ($log_items[7] =~ m/(dnsbl)/){
484 + $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
485 + elsif ($log_items[7] =~ m/(helo)/){
486 + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
487 + else {
488 + #Unidentified Naughty rejection
489 + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE}
490 + }
491 + elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
492 +
493 + elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE}
494 +
495 + elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
496 +
497 + elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE}
498 +
499 + elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
500 +
501 + elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
502 +
503 + elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
504 +
505 + elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
506 +
507 + elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
508 +
509 + elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
510 +
511 + elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
512 +
513 #Treat it as Unconf if not recognised
514 else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE}
515 + } #Log[5] exists
516 +
517 +
518 +# print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected
519 +# next LINE
520
521 -/*
522 - print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected
523 - next LINE
524 -*/
525
526 - }
527
528 } #END OF MAIN LOOP
529
530 @@ -624,18 +830,28 @@
531 if ( !$disabled ) {
532
533 #Output results
534 +
535 + # NEW - save the print to a variable so that it can be processed into html.
536 + #
537 + #Save current output selection and divert into variable
538 + #
539 + my $output;
540 + my $tablestr="";
541 + open(my $outputFH, '>', \$tablestr) or die; # This shouldn't fail
542 + my $oldFH = select $outputFH;
543 +
544 +
545 print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n";
546 print "----------------------------------------------------", "\n\n";
547
548 print "$0 Version : $opt{'version'}", "\n\n";
549 - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n";
550 + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n";
551 print "Period Ending : ", strftime( "%c", localtime($end) ), "\n";
552 print "\n";
553
554 - print "Clam Version : ", `freshclam -V`;
555 - print "SpamAssassin Version : ", `spamassassin -V`;
556 - printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,
557 - $SARejectLevel;
558 + print "Clam Version/DB Count/Last DB update: ", `freshclam -V`."\n";
559 + print "SpamAssassin Version : ", `spamassassin -V`."\n";
560 + printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel;
561 if ($HighLogLevel) {
562 printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n";
563 printf "\tYou can set it this way:\n";
564 @@ -643,10 +859,10 @@
565 printf "\tsignal-event email-update\n";
566 printf "\tsv t /var/service/qpsmtpd\n\n";
567 }
568 - print "\n";
569 + print "\n\n";
570 printf "Reporting Period : %.2f hrs\n", $hrsinperiod;
571 - print "----------------------------\n";
572 - print "\n";
573 + #print "----------------------------\n";
574 + #print "\n";
575
576 printf "All SMTP connections accepted:%8d \n", $totalexamined;
577
578 @@ -655,8 +871,13 @@
579 printf "Average spam score (accepted): %11.2f\n", $spamavg || 0;
580 printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0;
581 printf "Average ham score : %11.2f\n", $hamavg || 0;
582 - print "\n";
583 - print "Statistics by Hour\n";
584 + printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount || 0;
585 + if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
586 +
587 + print "\n\n";
588 + print "\nStatistics by Hour\n";
589 + print "-------------------\n";
590 + #print "\n";
591
592 #
593 # start by working out which colunns to show - tag the display array
594 @@ -682,13 +903,13 @@
595
596
597 # and put together the print lines
598 - #
599 +
600 my $Line1; #Full Line across the page
601 my $Line2; #Broken Line across the page
602 my $Titles; #Column headers
603 my $Values; #Values
604 my $Totals; #Corresponding totals
605 - my $Percent; # and column percentages
606 + my $Percent; # and column percentages
607
608 my $hour = floor( $start / 3600 );
609 $Line1 = '';
610 @@ -706,7 +927,7 @@
611 $Line1 .= substr('---------------------',0,$colwidth[$ncateg]);
612 $Line2 .= substr('---------------------',0,$colwidth[$ncateg]-1);
613 $Line2 .= " ";
614 - $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])." ";
615 + $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])."|";
616 if ($ncateg == 0) {
617 $Totals .= substr('TOTALS ',0,$colwidth[$ncateg]-2);
618 $Percent .= substr('PERCENTAGES ',0,$colwidth[$ncateg]-1);
619 @@ -744,39 +965,58 @@
620 $hour++;
621 }
622
623 - # print it.
624 - print $Line1."\n";
625 + #
626 + # print it.
627 + #
628 + my $makeHTMLemail = "no";
629 + #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
630 + my $makeHTMLpage = "no";
631 + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
632 + #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
633 +
634 + if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion ....
635 print $Titles."\n";
636 - print $Line2."\n";
637 + if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto
638 + #$Line2 =~ s/-/a/g;
639 + #print $Line2."\n";
640 + #print "\n";
641 print $Values."\n";
642 print $Line2."\n";
643 print $Totals."\n";
644 print $Percent."\n";
645 print $Line1."\n";
646 -
647 + print "\n";
648
649 if ($localAccepttotal>0) {
650 print "*Fetchml* means connections from Fetchmail delivering email\n";
651 }
652 - print "*Local* means connections from workstations on local LAN.\n";
653 + print "*Local* means connections from workstations on local LAN.\n\n";
654 print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol";
655 - print " or email was to non existant address.\n";
656 + print " or email was to non existant address.\n\n";
657 +
658 + if ($finaldisplay[$KarmaCateg]){
659 + print "*Karma* means email was rejected based on the mailserver's previous activities.\n\n";
660 + }
661 +
662
663 if ($finaldisplay[$BadCountryCateg]){
664 $BadCountries = $cdb->get('qpsmtpd')->prop('BadCountries') || "*none*";
665 - print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n";
666 + print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n\n";
667 }
668
669 +
670 +
671 if (scalar keys %unrecog_plugin > 0){
672 #Show unrecog plugins found
673 print "*Unrecognised plugins found - categorised as Non-Conf\n";
674 foreach my $unrec (keys %unrecog_plugin){
675 print "\t$unrec\t($unrecog_plugin{$unrec})\n";
676 - }
677 + }
678 + print "\n";
679 }
680
681 if ($QueryNoLogTerse) {
682 - print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n";
683 + print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n\n";
684 # print " to enable it follow the instructions at .............................\n";
685 }
686
687 @@ -813,7 +1053,7 @@
688 # if ($Webmailsendtotal > 0) {print "If you have the mailman contrib installed, then the webmail totals might include some mailman emails\n"}
689
690 # time to do a 'by recipient domain' report
691 - print "\nIncoming mails by recipient domains usage\n";
692 + print "Incoming mails by recipient domains usage\n";
693 print "-----------------------------------------\n";
694 print
695 "Domains Type Total Denied XferErr Accept \%accept\n";
696 @@ -869,25 +1109,6 @@
697 show_virus_variants();
698 }
699
700 - # get enable/disable subsections
701 - my $enableqpsmtpdcodes;
702 - my $enableSARules;
703 - my $enableGeoiptable;
704 - my $enablejunkMailList;
705 - my $savedata;
706 - if ($cdb->get('mailstats')){
707 - $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
708 - $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
709 - $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false;
710 - $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false;
711 - $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false;
712 - } else {
713 - $enableqpsmtpdcodes = $true;
714 - $enableSARules = $true;
715 - $enablejunkMailList = $true;
716 - $enableGeoiptable = $true;
717 - $savedata = $false;
718 - }
719
720 if ($enableqpsmtpdcodes) {show_qpsmtpd_codes();}
721
722 @@ -905,8 +1126,39 @@
723 "config setprop mailstats SaveDataToMySQL yes\n";
724 }
725
726 + select $oldFH;
727 + close $outputFH;
728 + if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr}
729 + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){
730 + #Convert text to html and send it
731 + require CGI;
732 + require TextToHTML;
733 + my $cgi = new CGI;
734 + my $text = $tablestr;
735 + print $cgi->header();
736 + my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20);
737 + my $conv = new HTML::TextToHTML();
738 + $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2,
739 + underline_length_tolerance=>1);
740 + my $html="<!DOCTYPE html> <html>\n";
741 + $html .= "<head><title>Mailstats -".strftime( "%F", localtime($start) )."</title>";
742 + $html .= "<link rel='stylesheet' type='text/css' href='mailstats.css' /></head>\n";
743 + $html .= "<body>\n";
744 + $html .= $conv->process_chunk($text);
745 + $html .= "</body></html>\n";
746 + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html}
747 + #And drop it into a file
748 + if ($makeHTMLpage eq "yes") {
749 + my $filename = "mailstats.html";
750 + open(my $fh, '>', $filename) or die "Could not open file '$filename' $!";
751 + print $fh $html;
752 + close $fh;
753 + }
754 +
755 + }
756
757 - #Close Senmdmail if it was opened
758 +
759 + #Close Sendmail if it was opened
760 if ( $opt{'mail'} ) {
761 select $oldfh;
762 close(SENDMAIL);
763 @@ -934,17 +1186,17 @@
764
765 if ($cdb->get('mailstats'))
766 {
767 - my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily';
768 + my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily'; #"fortnightly"; #"daily";# #; TEMP!!
769 if ($interval eq "weekly") {
770 $secsininterval = 86400*7;
771 } elsif ($interval eq "fortnightly") {
772 $secsininterval = 86400*14;
773 } elsif ($interval eq "monthly") {
774 - $secsininterval = 86400;
775 + $secsininterval = 86400*30;
776 } elsif ($interval =~m/\d+/) {
777 $secsininterval = $interval*3600;
778 };
779 - my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight';
780 + my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight';
781 my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
782 localtime(time);
783 if ($base eq "Midnight"){
784 @@ -954,7 +1206,8 @@
785 } elsif ($base =~m/\d+/){
786 $sec=0;$min=0;$hour=$base;
787 };
788 - $time = timelocal($sec,$min,$hour,$mday,$mon,$year)
789 + #$mday="17"; #$mday="03"; #$mday="16"; #Temp!!
790 + $time = timelocal($sec,$min,$hour,$mday,$mon,$year);
791 }
792
793 my $start = str2time( $startdate );
794 @@ -966,7 +1219,8 @@
795
796 sub dbg {
797 my $msg = shift;
798 -
799 + my $time = scalar localtime;
800 + $msg = $time.":".$msg."\n";
801 if ( $opt{debug} ) {
802 print STDERR $msg;
803 }
804 @@ -1001,9 +1255,10 @@
805 }
806 my $i = keys %junkcount;
807 if ( $i > 0 ) {
808 - print("Junk Mails left in folder:\n");
809 - print("-------------------------\n");
810 - print("Count\tUser\n");
811 + print "\n\n";
812 + print("\nJunk Mails left in folder:\n");
813 + print("---------------------------\n\n");
814 + print("\nCount\tUser\n");
815 print("-------------------------\n");
816 foreach my $thisuser (
817 sort { $junkcount{$b} <=> $junkcount{$a} }
818 @@ -1033,7 +1288,7 @@
819 foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} }
820 keys %found_viruses)
821 {
822 - if (index($virus,"Sanesecurity")!=-1){
823 + if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){
824 print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n";
825 } else {
826 print "Rejected $found_viruses{$virus}\t$virus\n";
827 @@ -1061,6 +1316,7 @@
828 print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined;
829 }
830 print("---------------------------------------------\n\n");
831 + print "\n\n";
832 }
833
834 sub show_Geoip_results
835 @@ -1078,38 +1334,40 @@
836 } else {
837 $percentthreshold = 0.5;
838 }
839 - print("Geoip results: (cutoff at $percentthreshold%) \n");
840 - print("---------------------------------\n");
841 - print("Country\tPercent\tCount\tRejected?\n");
842 - print("---------------------------------\n");
843 - foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
844 - keys %found_countries)
845 - {
846 - $percent = $found_countries{$country} * 100 / $total_countries
847 - if $total_countries;
848 - $totalpercent = $totalpercent + $percent;
849 - if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
850 - if ( $percent >= $percentthreshold ) {
851 - print "$country\t"
852 - . sprintf( '%4.1f', $percent )
853 - . "%\t$found_countries{$country}","\t$reject\n"
854 - if $total_countries;
855 - }
856 -
857 - }
858 - print("---------------------------------\n");
859 - my ($showtotals);
860 - if ($cdb->get('mailstats')){
861 - $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
862 - } else {
863 - $showtotals = $true;
864 - }
865 -
866 - if ($showtotals){
867 - print "TOTALS\t$totalpercent%\t$total_countries\n";
868 - print("---------------------------------\n\n");
869 + if ($total_countries > 0) {
870 + print("Geoip results: (cutoff at $percentthreshold%) \n");
871 + print("---------------------------------\n");
872 + print("Country\tPercent\tCount\tRejected?\n");
873 + print("---------------------------------\n");
874 + foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
875 + keys %found_countries)
876 + {
877 + $percent = $found_countries{$country} * 100 / $total_countries
878 + if $total_countries;
879 + $totalpercent = $totalpercent + $percent;
880 + if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
881 + if ( $percent >= $percentthreshold ) {
882 + print "$country\t"
883 + . sprintf( '%4.1f', $percent )
884 + . "%\t$found_countries{$country}","\t$reject\n"
885 + if $total_countries;
886 + }
887 +
888 + }
889 + print("---------------------------------\n");
890 + my ($showtotals);
891 + if ($cdb->get('mailstats')){
892 + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
893 + } else {
894 + $showtotals = $true;
895 + }
896 +
897 + if ($showtotals){
898 + print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n";
899 + print("---------------------------------\n\n");
900 + }
901 + print "\n";
902 }
903 - print "\n";
904 }
905
906 sub show_SARules_codes
907 @@ -1123,52 +1381,55 @@
908 my ($percentthreshold);
909 my ($defaultpercentthreshold);
910 my ($totalpercent) = 0;
911 -
912 - if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
913 - $defaultpercentthreshold = $maxcutoff
914 - } else {
915 - $defaultpercentthreshold = $mincutoff
916 - }
917 - if ($cdb->get('mailstats')){
918 - $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold;
919 - } else {
920 - $percentthreshold = $defaultpercentthreshold
921 - }
922 -
923 - print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
924 - print("---------------------------------------------\n");
925 - print("Count\tPercent\tScore\t\t\n");
926 - print("---------------------------------------------\n");
927 - foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
928 - keys %found_SARules)
929 - {
930 - my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
931 - if $totalexamined;
932 - #$totalpercent = $totalpercent + $percent;
933 - my $avehits = $found_SARules{$SARule}{'totalhits'} /
934 - $found_SARules{$SARule}{'count'}
935 - if $found_SARules{$SARule}{'count'};
936 - if ( $percent >= $percentthreshold ) {
937 - print "$found_SARules{$SARule}{'count'}\t"
938 - . sprintf( '%4.1f', $percent ) . "%\t"
939 - . sprintf( '%4.1f', $avehits )
940 - . "\t$SARule\n"
941 +
942 + if ($sum_SARules > 0){
943 +
944 + if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
945 + $defaultpercentthreshold = $maxcutoff
946 + } else {
947 + $defaultpercentthreshold = $mincutoff
948 + }
949 + if ($cdb->get('mailstats')){
950 + $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold;
951 + } else {
952 + $percentthreshold = $defaultpercentthreshold
953 + }
954 +
955 + print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
956 + print("---------------------------------------------\n");
957 + print("Count\tPercent\tScore\t\t\n");
958 + print("---------------------------------------------\n");
959 + foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
960 + keys %found_SARules)
961 + {
962 + my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
963 if $totalexamined;
964 -}
965 - }
966 - print("---------------------------------------------\n");
967 - my ($showtotals);
968 - if ($cdb->get('mailstats')){
969 - $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
970 - } else {
971 - $showtotals = $true;
972 - }
973 -
974 - if ($showtotals){
975 - print "$totalexamined\t(TOTALS)\n";
976 + #$totalpercent = $totalpercent + $percent;
977 + my $avehits = $found_SARules{$SARule}{'totalhits'} /
978 + $found_SARules{$SARule}{'count'}
979 + if $found_SARules{$SARule}{'count'};
980 + if ( $percent >= $percentthreshold ) {
981 + print "$found_SARules{$SARule}{'count'}\t"
982 + . sprintf( '%4.1f', $percent ) . "%\t"
983 + . sprintf( '%4.1f', $avehits )
984 + . "\t$SARule\n"
985 + if $totalexamined;
986 + }
987 + }
988 print("---------------------------------------------\n");
989 + my ($showtotals);
990 + if ($cdb->get('mailstats')){
991 + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
992 + } else {
993 + $showtotals = $true;
994 + }
995 +
996 + if ($showtotals){
997 + print "$totalexamined\t(TOTALS)\n";
998 + print("---------------------------------------------\n");
999 + }
1000 + print "\n";
1001 }
1002 - print "\n";
1003
1004
1005 }
1006 @@ -1370,9 +1631,15 @@
1007 }
1008 $nhour++;
1009 }
1010 - $dbh->disconnect();
1011 - my $telapsed = time - $tstart;
1012 - print "Saved $reccount records in $telapsed sec.";
1013 + # and write out the log lines saved
1014 +
1015 + foreach my $logid (keys %LogLines){
1016 +
1017 + $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')");
1018 + }
1019 + $dbh->disconnect();
1020 + my $telapsed = time - $tstart;
1021 + print "Saved $reccount records in $telapsed sec.";
1022 }
1023
1024 sub check_date_rec
1025 @@ -1439,5 +1706,3 @@
1026 my $daterec = $sth->fetchrow_hashref();
1027 $daterec->{"dateid"};
1028 }
1029 -
1030 -

admin@koozali.org
ViewVC Help
Powered by ViewVC 1.2.1 RSS 2.0 feed