/[smecontribs]/rpms/smeserver-mailstats/contribs10/smeserver-mailstats-1.1.bz9588.qpsmtpd0_96compatible.patch
ViewVC logotype

Annotation of /rpms/smeserver-mailstats/contribs10/smeserver-mailstats-1.1.bz9588.qpsmtpd0_96compatible.patch

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.1 - (hide annotations) (download)
Mon Apr 13 18:12:07 2020 UTC (4 years, 6 months ago) by brianr
Branch: MAIN
CVS Tags: smeserver-mailstats-1_1-16_el7_sme, smeserver-mailstats-1_1-13_el7_sme, smeserver-mailstats-1_1-17_el7_sme, smeserver-mailstats-1_1-14_el7_sme, smeserver-mailstats-1_1-11_el7_sme, smeserver-mailstats-1_1-12_el7_sme, smeserver-mailstats-1_1-15_el7_sme, HEAD
Initial import

1 brianr 1.1 diff -Nur smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl
2     --- smeserver-mailstats-1.1.old/root/usr/bin/spamfilter-stats-7.pl 2016-07-02 04:29:21.671000000 -0400
3     +++ smeserver-mailstats-1.1/root/usr/bin/spamfilter-stats-7.pl 2016-07-02 04:39:34.037000000 -0400
4     @@ -18,6 +18,8 @@
5     # bjr - 19Jun15 - Add totals for the League tables
6     # bjr and Unnilennium - 08Apr16 - Add in else for unrecognised plugin detection
7     # bjr - 08Apr16 - Add in link for SaneSecurity "extra" virus detection
8     +# bjr - 14Jun16 - make compatible with qpsmtpd 0.96
9     +# bjr - 16Jun16 - Add code to create an html equivalent of the text email (v0.7)
10     #
11     #############################################################################
12     #
13     @@ -37,8 +39,10 @@
14     # / ShowLeagueTotals - Show totals row after league tables - (default is "yes")
15     # / DBHost - MySQL server hostname (default is "localhost").
16     # / DBPort - MySQL server post (default is "3306")
17     -# / Interval - "day", "week", "fortnight", "month", "99999" - last is number of seconds (default is day)
18     +# / Interval - "daily", "weekly", "fortnightly", "monthly", "99999" - last is number of hours (default is daily)
19     # / Base - "Midnight", "Midday", "Now", "99" hour (0-23) (default is midnight)
20     +# / HTMLEmail - "yes", "no", "both" - default is "No" - Send email in HTML
21     +# / HTMLPage - "yes" / "no" - default is "yes" if HTMLEmail is "yes" or "both" otherwise "no"
22     #
23     #############################################################################
24    
25     @@ -55,6 +59,10 @@
26     use esmith::DomainsDB;
27     use Sys::Hostname;
28     use Switch;
29     +use DBIx::Simple;
30     +
31     +#use CGI;
32     +#use HTML::TextToHTML;
33    
34     my $hostname = hostname();
35     my $cdb = esmith::ConfigDB->open_ro or die "Couldn't open ConfigDB : $!\n";
36     @@ -73,12 +81,11 @@
37    
38     #Configuration section
39     my %opt = (
40     - version => '0.6.29', # please update at each change.
41     + version => '0.7.5a', # please update at each change.
42     debug => 0, # guess what ?
43     sendmail => '/usr/sbin/sendmail', # Path to sendmail stub
44     from => 'spamfilter-stats', # Who is the mail from
45     - mail => # mailstats email recipient
46     - $cdb->get('mailstats')->prop('Email') || 'admin',
47     + mail => $cdb->get('mailstats')->prop('Email') || 'admin', # mailstats email recipient
48     timezone => `date +%z`,
49     );
50    
51     @@ -87,8 +94,10 @@
52     my $localhost = 'localhost'; #Apparent sender for webmail
53     my $FETCHMAIL = 'FETCHMAIL'; #Sender from fetchmail when Ip address not 127.0.0.200 - when qpsmtpd denies the email
54     my $MAILMAN = "bounces"; #sender when mailman sending when orig is localhost
55     +my $DMARCDomain="dmarc"; #Pattern to recognised DMARC sent emails (this not very reliable, as the email address could be anything)
56     +my $DMARCOkPattern="dmarc: pass"; #Pattern to use to detect DMARC approval
57    
58     -my $MinCol = 8; #Minimum column width
59     +my $MinCol = 6; #Minimum column width
60     my $HourColWidth = 16; #Date and time column width
61    
62     my $SARulethresholdPercent = 10; #If Sa rules less than this of total emails, then cutoff reduced
63     @@ -116,6 +125,10 @@
64     my $totalexamined = 0; #total download + RBL etc
65     my $WebMailsendtotal = 0; #total from Webmail
66     my $mailmansendcount = 0; #total from mailman
67     +my $DMARCSendCount = 0; #total DMARC reporting emails sent (approx)
68     +my $DMARCOkCount = 0; #Total emails approved through DMARC
69     +
70     +
71    
72     my %found_viruses = ();
73     my %found_qpcodes = ();
74     @@ -137,25 +150,35 @@
75     my $CATMAILMAN='Mailman';
76     my $CATLOCAL='Local';
77     # border between where it came from and where it ended..
78     -my $countfromhere = 5;
79     -
80     +my $countfromhere = 5; #Temp - Check this not moved!!
81     +
82     my $CATVIRUS='Virus';
83     my $CATRBLDNS='RBL/DNS';
84     my $CATEXECUT='Execut.';
85     my $CATNONCONF='Non.Conf.';
86     my $CATBADCOUNTRIES='Geoip.';
87     -my $BadCountryCateg=8; #Careful here this number could change if more added before.
88     +my $CATKARMA="Karma";
89     +
90     my $CATSPAMDEL='Del.Spam';
91     my $CATSPAM='Qued.Spam?';
92     my $CATHAM='Ham';
93     my $CATTOTALS='TOTALS';
94     my $CATPERCENT='PERCENT';
95     -my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
96     +my $CATDMARC="DMARC Rej.";
97     +my $CATLOAD="Rej.Load";
98     +my @categs = ($CATHOUR,$CATFETCHMAIL,$CATWEBMAIL,$CATMAILMAN,$CATLOCAL,$CATDMARC,$CATVIRUS,$CATRBLDNS,$CATEXECUT,$CATBADCOUNTRIES,$CATNONCONF,$CATLOAD,$CATKARMA,$CATSPAMDEL,$CATSPAM,$CATHAM,$CATTOTALS,$CATPERCENT);
99     my $GRANDTOTAL = '99'; #subs for count arrays, for grand total
100     my $PERCENT = '98'; # for column percentages
101    
102     my $categlen = @categs-2; #-2 to avoid the total and percent column
103    
104     +#
105     +# Index for certain columns - check these do not move if we add columns
106     +#
107     +my $BadCountryCateg=9;
108     +my $DMARCcateg = 5; #Not used.
109     +my $KarmaCateg=$BadCountryCateg+3;
110     +
111     my $above15 = 0;
112     my $RBLcount = 0;
113     my $MiscDenyCount = 0;
114     @@ -187,6 +210,38 @@
115     my $morethanonercpt = 0 ; # count every 'second' recipients for a mail.
116     my $recipcount = 0; # count every recipient email address received.
117    
118     +#
119     +#Load up the emails curreently stored for DMARC reporting - so that we cna spot the reports being sent.
120     +#Held in an slqite db, created by the DMARC perl lib.
121     +#
122     +my $dsn = "dbi:SQLite:dbname=/var/lib/qpsmtpd/dmarc/reports.sqlite"; #Taken from /etc/mail-dmarc.ini
123     +# doesn't seem to need
124     +my $user = "";
125     +my $pass = "";
126     +my $DMARC_Report_emails = ""; #Flat string of all email addresses
127     +
128     + if (my $dbix = DBIx::Simple->connect( $dsn, $user, $pass )){
129     + my $result = $dbix->query("select rua from report_policy_published;");
130     + $result->bind(my ($emailaddress));
131     + while ($result->fetch){
132     + #print STDERR "$emailaddress";
133     + #remember email from logterse entry has chevrons round it - so we add them here to guarantee the alighment of the match
134     + #Remove the mailto:
135     + $emailaddress =~ s/mailto://g;
136     + # and map any commas to ><
137     + $emailaddress =~ s/,/></g;
138     + $DMARC_Report_emails .= "<".$emailaddress.">\n"
139     + }
140     + $dbix->disconnect();
141     + } else { $DMARC_Report_emails = "None found - DB not opened"}
142     +
143     +
144     +#dbg("DMARC-EMAILS:".$DMARC_Report_emails);
145     +
146     +# Saving the Log lines processed
147     +my %LogLines = (); #Save all the log lines processed for writing to the DB
148     +my %LogId = (); #Save the Log Ids.
149     +my $CurrentLogId = "";
150    
151     # store the domain of interest. Every other records are stored in a 'Other' zone
152     my $ddb = esmith::DomainsDB->open_ro or die "Couldn't open DomainsDB : $!\n";
153     @@ -212,6 +267,8 @@
154    
155     my ( $start, $end ) = analysis_period();
156    
157     +dbg("Time interval:".strftime("%a %b %e %H:%M:%S %Y", localtime($start))."->".strftime("%a %b %e %H:%M:%S %Y", localtime($end))."\n");
158     +
159     #
160     # First check current configuration for logging, DNS enable and Max threshold for spamassassin
161     #
162     @@ -243,6 +300,27 @@
163    
164     }
165    
166     +# get enable/disable subsections
167     +my $enableqpsmtpdcodes;
168     +my $enableSARules;
169     +my $enableGeoiptable;
170     +my $enablejunkMailList;
171     +my $savedata;
172     +if ($cdb->get('mailstats')){
173     + $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
174     + $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
175     + $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false;
176     + $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false;
177     + $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false;
178     + } else {
179     + $enableqpsmtpdcodes = $true;
180     + $enableSARules = $true;
181     + $enablejunkMailList = $true;
182     + $enableGeoiptable = $true;
183     + $savedata = $false;
184     + }
185     + $savedata = $false; #TEMP!!
186     +
187     #
188     #---------------------------------------
189     # Scan the qpsmtpd log file(s)
190     @@ -263,6 +341,7 @@
191     }
192     # and grand totals, percent and display status from db entries, and column widths
193     $ncateg = 0;
194     +my $colpadding = 0;
195     while ( $ncateg < @categs) {
196     $counts{$GRANDTOTAL}{$categs[$ncateg]} = 0;
197     $counts{$PERCENT}{$categs[$ncateg]} = 0;
198     @@ -273,11 +352,11 @@
199     $display[$ncateg] = 'auto'
200     }
201     if ($ncateg == 0) {
202     - $colwidth[$ncateg] = $HourColWidth
203     + $colwidth[$ncateg] = $HourColWidth + $colpadding;
204     } else {
205     - $colwidth[$ncateg] = length($categs[$ncateg])+1
206     + $colwidth[$ncateg] = length($categs[$ncateg])+1+$colpadding;
207     }
208     - if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol}
209     + if ($colwidth[$ncateg] < $MinCol) {$colwidth[$ncateg] = $MinCol + $colpadding}
210     $ncateg++
211     }
212    
213     @@ -292,39 +371,91 @@
214     }
215     @ARGV=@ARGV2;
216    
217     +my $count = -1; #for loop reduction in debugging mode
218     +
219     +my $CurrentMailId = "";
220     +
221     LINE: while (<>) {
222     - my($tai,$log) = split(' ',$_,2);
223    
224     + #print STDERR $starttai,$endtai,$_,"\n";
225     +
226    
227     + next LINE if !(my($tai,$log) = split(' ',$_,2));
228     + #dbg("TAI:".$tai);
229     +
230     + #dbg("REST1:".$log);
231     +
232     #If date specified, only process lines matching date
233     next LINE if ( $tai lt $starttai );
234     next LINE if ( $tai gt $endtai );
235    
236     + #Count lines and skip out if debugging
237     + $count++;
238     + last LINE if ($opt{debug} && $count >= 100000);
239     + #dbg("REST:".$log);
240     +
241     + #Loglines to Saved String for later DB write
242     + if ($savedata) {
243     + my $CurrentLine = $_;
244     + $CurrentLine = /^\@([0-9a-z]*) ([0-9]*) .*$/;
245     + if ($2 ne $CurrentMailId) {
246     + $CurrentLogId = $1."-".$2;
247     + $CurrentMailId = $2;
248     + }
249     + $LogLines{$CurrentLogId} = $_;
250     + #print $CurrentLogId.":".$LogLines{$CurrentLogId}."\n";
251     + }
252     +
253     + #Count lines and skip out if debugging
254     + $count++;
255     + last LINE if ($opt{debug} && $count >= 100);
256     + #dbg("REST:".$log);
257     +
258     +
259     # pull out spamasassin rule lists
260     - if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
261     + if ( $_ =~m/spamassassin: pass, Ham,(.*)</ )
262     + #if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ )
263     {
264     - my (@SAtests) = split(',',$2);
265     - foreach my $SAtest (@SAtests) {
266     - if (!$SAtest eq "") {
267     - $found_SARules{$SAtest}{'count'}++;
268     - $found_SARules{$SAtest}{'totalhits'} += $1;
269     - $sum_SARules++
270     - }
271     - }
272     -
273     + dbg("SPAM:".$log);
274     +
275     +
276     + #New version does not seem to have spammassasin tests in logs
277     +
278     + #if (exists($2){
279     + #my (@SAtests) = split(',',$2);
280     + #foreach my $SAtest (@SAtests) {
281     + #if (!$SAtest eq "") {
282     + #$found_SARules{$SAtest}{'count'}++;
283     + #$found_SARules{$SAtest}{'totalhits'} += $1;
284     + #$sum_SARules++
285     + #}
286     + #}
287     + #}
288     +
289     }
290     -
291     +
292     +
293     #Pull out Geoip countries for analysis table
294     - if ( $_ =~m/check_badcountries plugin \(connect\): GeoIP Country: (.*)/ )
295     + if ( $_ =~m/check_badcountries: GeoIP Country: (.*)/ )
296     {
297     $found_countries{$1}++;
298     $total_countries++;
299     }
300     +
301     + #Pull out DMARC approvals
302     + if ( $_ =~m/.*$DMARCOkPattern.*/ )
303     + {
304     + $DMARCOkCount++;
305     + }
306     +
307    
308     #only select Logterse output
309     - next LINE unless m/terse plugin/;
310     -
311     -
312     + next LINE unless m/logging::logterse:/;
313     +
314     + #Count lines and skip out if debugging
315     + $count++;
316     + last LINE if ($opt{debug} && $count >= 100000);
317     + #dbg("REST:".$log);
318    
319     my $abstime = Time::TAI64::tai2unix($tai);
320     my $abshour = floor( $abstime / 3600 ); # Hours since the epoch
321     @@ -342,6 +473,9 @@
322    
323     $totalexamined++;
324    
325     + #dbg("LOG1:".$log_items[1]);
326     + #dbg("LOG3:".$log_items[3]);
327     +
328     # first spot the fetchmail and local deliveries.
329    
330     # Spot from local workstation
331     @@ -355,11 +489,9 @@
332    
333     # see if from localhost
334     elsif ( $log_items[1] =~ m/.*$localhost.*/ ) {
335     -
336     # but not if it comes from fetchmail
337     if ( $log_items[3] =~ m/.*$FETCHMAIL.*/ ) { }
338     else {
339     -
340     # might still be from mailman here
341     if ( $log_items[3] =~ m/.*$MAILMAN.*/ ) {
342     $mailmansendcount++;
343     @@ -368,21 +500,44 @@
344     $localflag = 1;
345     }
346     else {
347     -
348     - # eliminate incoming localhost spoofs
349     - if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
350     - else {
351     - $localflag = 1;
352     - $WebMailsendtotal++;
353     - $counts{$abshour}{$CATWEBMAIL}++;
354     - $WebMailflag = 1;
355     - }
356     + #Or sent to the DMARC server
357     + dbg("LOG4:".$log_items[4]);
358     + #check for email address in $DMARC_Report_emails string
359     + #if ($log_items[4] =~ m/.*$DMARCDomain.*/) {
360     + my $logemail = $log_items[4];
361     + #print STDERR "/",$log_items[4]."/\n";
362     + if ((index($DMARC_Report_emails,$logemail)>=0) || ($logemail =~ m/.*$DMARCDomain.*/)){
363     + $localsendtotal++;
364     + $DMARCSendCount++;
365     + $localflag = 1;
366     + }
367     + else {
368     + #print STDERR "no match:.".$logemail;
369     + if (exists $log_items[8]){
370     + dbg("LOG8:".$log_items[8]);
371     + # ignore incoming localhost spoofs
372     + if ( $log_items[8] =~ m/.*msg denied before queued.*/ ) { }
373     + else {
374     + $localflag = 1;
375     + $WebMailsendtotal++;
376     + $counts{$abshour}{$CATWEBMAIL}++;
377     + $WebMailflag = 1;
378     + }
379     + }
380     + else {
381     + $localflag = 1;
382     + $WebMailsendtotal++;
383     + $counts{$abshour}{$CATWEBMAIL}++;
384     + $WebMailflag = 1;
385     + }
386     + }
387     }
388     }
389     }
390    
391     # try to spot fetchmail emails
392     if ( $log_items[0] =~ m/.*$FetchmailIP.*/ ) {
393     + dbg("LOG0:".$log_items[0]);
394     $localAccepttotal++;
395     $counts{$abshour}{$CATFETCHMAIL}++;
396     }
397     @@ -394,10 +549,13 @@
398     # and adjust for recipient field if not set-up by denying plugin - extract from deny msg
399    
400     if ( length( $log_items[4] ) == 0 ) {
401     + dbg("LOG7:".$log_items[0]);
402     if ( $log_items[5] eq 'check_goodrcptto' ) {
403     if ( $log_items[7] gt "invalid recipient" ) {
404     $log_items[4] =
405     - substr( $log_items[7], 18 ) #Leave only email address
406     + substr( $log_items[7], 18 ); #Leave only email address
407     + dbg("LOG4:".$log_items[0]);
408     +
409     }
410     }
411     }
412     @@ -405,6 +563,7 @@
413     # if ( ( $currentrcptdomain{ $proc } || '' ) eq '' ) {
414     # reduce to lc and process each e,mail if a list, pseperatedy commas
415     my $recipientmail = lc( $log_items[4] );
416     + dbg("LOG4:".$log_items[0]);
417     if ( $recipientmail =~ m/.*,/ ) {
418    
419     #comma - split the line and deal with each domain
420     @@ -442,7 +601,12 @@
421    
422     if (exists $log_items[5]) {
423    
424     - $found_qpcodes{$log_items[5]}++; ##Count different qpsmtpd result codes
425     + if ($log_items[5] eq 'naughty') {
426     + my $rejreason = $log_items[7];
427     + $rejreason = /.*(\(.*\)).*/;
428     + $rejreason = $1;
429     + $found_qpcodes{$log_items[5]."-".$rejreason}++}
430     + else {$found_qpcodes{$log_items[5]}++} ##Count different qpsmtpd result codes
431    
432     #Check for badly formed lines (from earlier testing)
433    
434     @@ -488,27 +652,31 @@
435    
436     elsif ($log_items[5] eq 'spamassassin') { $above15++;$counts{$abshour}{$CATSPAMDEL}++;
437     # and extract the spam score
438     - if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)") {$rejectspamavg += $1}
439     +# if ($log_items[8] =~ "Yes, hits=(.*) required=([0-9\.]+)")
440     + if ($log_items[8] =~ "Yes, score=(.*) required=([0-9\.]+)")
441     + {$rejectspamavg += $1}
442     mark_domain_rejected($proc);
443     next LINE
444     }
445    
446     - elsif ($log_items[5] eq 'virus::clamav') { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
447     + elsif (($log_items[5] eq 'virus::clamav') || ($log_items[5] eq 'virus::clamdscan')) { $infectedcount++;$counts{$abshour}{$CATVIRUS}++;
448     #extract the virus name
449     - if ($log_items[7] =~ "Virus Found: (.*)" ) {$found_viruses{$1}++;}
450     + if ($log_items[7] =~ "Virus found: (.*)" ) {$found_viruses{$1}++;}
451     + else {$found_viruses{$log_items[7]}++} #Some other message!!
452     + dbg("LOG7:".$log_items[7]);
453     mark_domain_rejected($proc);
454     next LINE
455     }
456    
457     elsif ($log_items[5] eq 'queued') { $Accepttotal++;
458     #extract the spam score
459     - if ($log_items[8] =~ ".*hits=(.*) required=([0-9\.]+)") {
460     + if ($log_items[8] =~ ".*score=(.*) required=([0-9\.]+)") {
461     $score = $1;
462     # print $log_items[8]."<".$score.">\n";
463     if ($score < $SATagLevel) { $hamcount++;$counts{$abshour}{$CATHAM}++;$hamavg += $score}
464     else {$spamcount++;$counts{$abshour}{$CATSPAM}++;$spamavg += $score}
465     } else {
466     - # no SA score - so it must be ham
467     + # no SA score - treat it as ham
468     $hamcount++;$counts{$abshour}{$CATHAM}++;
469     }
470     if ( ( $currentrcptdomain{ $proc } || '' ) ne '' ) {
471     @@ -523,15 +691,53 @@
472    
473     elsif ($log_items[5] eq 'auth::auth_cvm_unix_local') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
474    
475     + elsif ($log_items[5] eq 'earlytalker') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
476     +
477     + elsif ($log_items[5] eq 'uribl') {$RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
478     +
479     + elsif ($log_items[5] eq 'naughty') {
480     + #Naughty plugin seems to span a number of rejection reasons - so we have to use the next but one log_item[7] to identify
481     + if ($log_items[7] =~ m/(karma)/) {
482     + $MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
483     + elsif ($log_items[7] =~ m/(dnsbl)/){
484     + $RBLcount++;$counts{$abshour}{$CATRBLDNS}++;mark_domain_rejected($proc);next LINE}
485     + elsif ($log_items[7] =~ m/(helo)/){
486     + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
487     + else {
488     + #Unidentified Naughty rejection
489     + $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]."-".$log_items[7]}++;next LINE}
490     + }
491     + elsif ($log_items[5] eq 'resolvable_fromhost') {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
492     +
493     + elsif ($log_items[5] eq 'loadcheck') {$MiscDenyCount++;$counts{$abshour}{$CATLOAD}++;mark_domain_rejected($proc);next LINE}
494     +
495     + elsif ($log_items[5] eq 'karma') {$MiscDenyCount++;$counts{$abshour}{$CATKARMA}++;mark_domain_rejected($proc);next LINE}
496     +
497     + elsif ($log_items[5] eq 'dmarc') {$MiscDenyCount++;$counts{$abshour}{$CATDMARC}++;mark_domain_rejected($proc);next LINE}
498     +
499     + elsif ($log_items[5] eq 'relay') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
500     +
501     + elsif ($log_items[5] eq 'headers') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
502     +
503     + elsif ($log_items[5] eq 'mailfrom') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
504     +
505     + elsif ($log_items[5] eq 'badrcptto') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
506     +
507     + elsif ($log_items[5] eq 'helo') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
508     +
509     + elsif ($log_items[5] eq 'check_smtp_forward') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
510     +
511     + elsif ($log_items[5] eq 'sender_permitted_from') { $MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);next LINE}
512     +
513     #Treat it as Unconf if not recognised
514     else {$MiscDenyCount++;$counts{$abshour}{$CATNONCONF}++;mark_domain_rejected($proc);$unrecog_plugin{$log_items[5]}++;next LINE}
515     + } #Log[5] exists
516     +
517     +
518     +# print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected
519     +# next LINE
520    
521     -/*
522     - print "Unexpected failure string in log file: ".$log_items[5]."\n"; #Not detected
523     - next LINE
524     -*/
525    
526     - }
527    
528     } #END OF MAIN LOOP
529    
530     @@ -624,18 +830,28 @@
531     if ( !$disabled ) {
532    
533     #Output results
534     +
535     + # NEW - save the print to a variable so that it can be processed into html.
536     + #
537     + #Save current output selection and divert into variable
538     + #
539     + my $output;
540     + my $tablestr="";
541     + open(my $outputFH, '>', \$tablestr) or die; # This shouldn't fail
542     + my $oldFH = select $outputFH;
543     +
544     +
545     print "SMEServer daily Anti-Virus and Spamfilter statistics", "\n";
546     print "----------------------------------------------------", "\n\n";
547    
548     print "$0 Version : $opt{'version'}", "\n\n";
549     - print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n";
550     + print "Period Beginning : ", strftime( "%c", localtime($start) ), "\n\n";
551     print "Period Ending : ", strftime( "%c", localtime($end) ), "\n";
552     print "\n";
553    
554     - print "Clam Version : ", `freshclam -V`;
555     - print "SpamAssassin Version : ", `spamassassin -V`;
556     - printf "Tag level: %3d; Reject level: %3d $warnnoreject\n", $SATagLevel,
557     - $SARejectLevel;
558     + print "Clam Version/DB Count/Last DB update: ", `freshclam -V`."\n";
559     + print "SpamAssassin Version : ", `spamassassin -V`."\n";
560     + printf "Tag level: %3d; Reject level: %3d $warnnoreject", $SATagLevel,$SARejectLevel;
561     if ($HighLogLevel) {
562     printf "*Loglevel is set to: ".$LogLevel. " - you only need it set to 6\n";
563     printf "\tYou can set it this way:\n";
564     @@ -643,10 +859,10 @@
565     printf "\tsignal-event email-update\n";
566     printf "\tsv t /var/service/qpsmtpd\n\n";
567     }
568     - print "\n";
569     + print "\n\n";
570     printf "Reporting Period : %.2f hrs\n", $hrsinperiod;
571     - print "----------------------------\n";
572     - print "\n";
573     + #print "----------------------------\n";
574     + #print "\n";
575    
576     printf "All SMTP connections accepted:%8d \n", $totalexamined;
577    
578     @@ -655,8 +871,13 @@
579     printf "Average spam score (accepted): %11.2f\n", $spamavg || 0;
580     printf "Average spam score (rejected): %11.2f\n", $rejectspamavg || 0;
581     printf "Average ham score : %11.2f\n", $hamavg || 0;
582     - print "\n";
583     - print "Statistics by Hour\n";
584     + printf "\nNumber of DMARC reporting emails sent: %11d (not shown on table)\n", $DMARCSendCount || 0;
585     + if ($hamcount != 0){ printf "Number of emails approved through DMARC: %11d (%4d%% of Ham count)\n", $DMARCOkCount|| 0,$DMARCOkCount*100/$hamcount || 0;}
586     +
587     + print "\n\n";
588     + print "\nStatistics by Hour\n";
589     + print "-------------------\n";
590     + #print "\n";
591    
592     #
593     # start by working out which colunns to show - tag the display array
594     @@ -682,13 +903,13 @@
595    
596    
597     # and put together the print lines
598     - #
599     +
600     my $Line1; #Full Line across the page
601     my $Line2; #Broken Line across the page
602     my $Titles; #Column headers
603     my $Values; #Values
604     my $Totals; #Corresponding totals
605     - my $Percent; # and column percentages
606     + my $Percent; # and column percentages
607    
608     my $hour = floor( $start / 3600 );
609     $Line1 = '';
610     @@ -706,7 +927,7 @@
611     $Line1 .= substr('---------------------',0,$colwidth[$ncateg]);
612     $Line2 .= substr('---------------------',0,$colwidth[$ncateg]-1);
613     $Line2 .= " ";
614     - $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])." ";
615     + $Titles .= sprintf('%'.($colwidth[$ncateg]-1).'s',$categs[$ncateg])."|";
616     if ($ncateg == 0) {
617     $Totals .= substr('TOTALS ',0,$colwidth[$ncateg]-2);
618     $Percent .= substr('PERCENTAGES ',0,$colwidth[$ncateg]-1);
619     @@ -744,39 +965,58 @@
620     $hour++;
621     }
622    
623     - # print it.
624     - print $Line1."\n";
625     + #
626     + # print it.
627     + #
628     + my $makeHTMLemail = "no";
629     + #if ($cdb->get('mailstats')){$makeHTMLemail = $cdb->get('mailstats')->prop('HTMLEmail') || "no"} #TEMP!!
630     + my $makeHTMLpage = "no";
631     + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both") {$makeHTMLpage = "yes"}
632     + #if ($cdb->get('mailstats')){$makeHTMLpage = $cdb->get('mailstats')->prop('HTMLPage') || "no"}
633     +
634     + if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line1."\n";} #These lines mess up the HTML conversion ....
635     print $Titles."\n";
636     - print $Line2."\n";
637     + if ($makeHTMLemail eq "no" && $makeHTMLpage eq "no"){print $Line2."\n";} #ditto
638     + #$Line2 =~ s/-/a/g;
639     + #print $Line2."\n";
640     + #print "\n";
641     print $Values."\n";
642     print $Line2."\n";
643     print $Totals."\n";
644     print $Percent."\n";
645     print $Line1."\n";
646     -
647     + print "\n";
648    
649     if ($localAccepttotal>0) {
650     print "*Fetchml* means connections from Fetchmail delivering email\n";
651     }
652     - print "*Local* means connections from workstations on local LAN.\n";
653     + print "*Local* means connections from workstations on local LAN.\n\n";
654     print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol";
655     - print " or email was to non existant address.\n";
656     + print " or email was to non existant address.\n\n";
657     +
658     + if ($finaldisplay[$KarmaCateg]){
659     + print "*Karma* means email was rejected based on the mailserver's previous activities.\n\n";
660     + }
661     +
662    
663     if ($finaldisplay[$BadCountryCateg]){
664     $BadCountries = $cdb->get('qpsmtpd')->prop('BadCountries') || "*none*";
665     - print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n";
666     + print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n\n";
667     }
668    
669     +
670     +
671     if (scalar keys %unrecog_plugin > 0){
672     #Show unrecog plugins found
673     print "*Unrecognised plugins found - categorised as Non-Conf\n";
674     foreach my $unrec (keys %unrecog_plugin){
675     print "\t$unrec\t($unrecog_plugin{$unrec})\n";
676     - }
677     + }
678     + print "\n";
679     }
680    
681     if ($QueryNoLogTerse) {
682     - print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n";
683     + print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n\n";
684     # print " to enable it follow the instructions at .............................\n";
685     }
686    
687     @@ -813,7 +1053,7 @@
688     # if ($Webmailsendtotal > 0) {print "If you have the mailman contrib installed, then the webmail totals might include some mailman emails\n"}
689    
690     # time to do a 'by recipient domain' report
691     - print "\nIncoming mails by recipient domains usage\n";
692     + print "Incoming mails by recipient domains usage\n";
693     print "-----------------------------------------\n";
694     print
695     "Domains Type Total Denied XferErr Accept \%accept\n";
696     @@ -869,25 +1109,6 @@
697     show_virus_variants();
698     }
699    
700     - # get enable/disable subsections
701     - my $enableqpsmtpdcodes;
702     - my $enableSARules;
703     - my $enableGeoiptable;
704     - my $enablejunkMailList;
705     - my $savedata;
706     - if ($cdb->get('mailstats')){
707     - $enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false;
708     - $enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false;
709     - $enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false;
710     - $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false;
711     - $savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false;
712     - } else {
713     - $enableqpsmtpdcodes = $true;
714     - $enableSARules = $true;
715     - $enablejunkMailList = $true;
716     - $enableGeoiptable = $true;
717     - $savedata = $false;
718     - }
719    
720     if ($enableqpsmtpdcodes) {show_qpsmtpd_codes();}
721    
722     @@ -905,8 +1126,39 @@
723     "config setprop mailstats SaveDataToMySQL yes\n";
724     }
725    
726     + select $oldFH;
727     + close $outputFH;
728     + if ($makeHTMLemail eq "no" || $makeHTMLemail eq "both") {print $tablestr}
729     + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" || $makeHTMLpage eq "yes"){
730     + #Convert text to html and send it
731     + require CGI;
732     + require TextToHTML;
733     + my $cgi = new CGI;
734     + my $text = $tablestr;
735     + print $cgi->header();
736     + my %paramhash = (default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>10,tab_width=>20);
737     + my $conv = new HTML::TextToHTML();
738     + $conv->args(default_link_dict=>'',make_tables=>1,preformat_trigger_lines=>2,preformat_whitespace_min=>2,
739     + underline_length_tolerance=>1);
740     + my $html="<!DOCTYPE html> <html>\n";
741     + $html .= "<head><title>Mailstats -".strftime( "%F", localtime($start) )."</title>";
742     + $html .= "<link rel='stylesheet' type='text/css' href='mailstats.css' /></head>\n";
743     + $html .= "<body>\n";
744     + $html .= $conv->process_chunk($text);
745     + $html .= "</body></html>\n";
746     + if ($makeHTMLemail eq "yes" || $makeHTMLemail eq "both" ) {print $html}
747     + #And drop it into a file
748     + if ($makeHTMLpage eq "yes") {
749     + my $filename = "mailstats.html";
750     + open(my $fh, '>', $filename) or die "Could not open file '$filename' $!";
751     + print $fh $html;
752     + close $fh;
753     + }
754     +
755     + }
756    
757     - #Close Senmdmail if it was opened
758     +
759     + #Close Sendmail if it was opened
760     if ( $opt{'mail'} ) {
761     select $oldfh;
762     close(SENDMAIL);
763     @@ -934,17 +1186,17 @@
764    
765     if ($cdb->get('mailstats'))
766     {
767     - my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily';
768     + my $interval = $cdb->get('mailstats')->prop('Interval') || 'daily'; #"fortnightly"; #"daily";# #; TEMP!!
769     if ($interval eq "weekly") {
770     $secsininterval = 86400*7;
771     } elsif ($interval eq "fortnightly") {
772     $secsininterval = 86400*14;
773     } elsif ($interval eq "monthly") {
774     - $secsininterval = 86400;
775     + $secsininterval = 86400*30;
776     } elsif ($interval =~m/\d+/) {
777     $secsininterval = $interval*3600;
778     };
779     - my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight';
780     + my $base = $cdb->get('mailstats')->prop('Base') || 'Midnight';
781     my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) =
782     localtime(time);
783     if ($base eq "Midnight"){
784     @@ -954,7 +1206,8 @@
785     } elsif ($base =~m/\d+/){
786     $sec=0;$min=0;$hour=$base;
787     };
788     - $time = timelocal($sec,$min,$hour,$mday,$mon,$year)
789     + #$mday="17"; #$mday="03"; #$mday="16"; #Temp!!
790     + $time = timelocal($sec,$min,$hour,$mday,$mon,$year);
791     }
792    
793     my $start = str2time( $startdate );
794     @@ -966,7 +1219,8 @@
795    
796     sub dbg {
797     my $msg = shift;
798     -
799     + my $time = scalar localtime;
800     + $msg = $time.":".$msg."\n";
801     if ( $opt{debug} ) {
802     print STDERR $msg;
803     }
804     @@ -1001,9 +1255,10 @@
805     }
806     my $i = keys %junkcount;
807     if ( $i > 0 ) {
808     - print("Junk Mails left in folder:\n");
809     - print("-------------------------\n");
810     - print("Count\tUser\n");
811     + print "\n\n";
812     + print("\nJunk Mails left in folder:\n");
813     + print("---------------------------\n\n");
814     + print("\nCount\tUser\n");
815     print("-------------------------\n");
816     foreach my $thisuser (
817     sort { $junkcount{$b} <=> $junkcount{$a} }
818     @@ -1033,7 +1288,7 @@
819     foreach my $virus (sort { $found_viruses{$b} <=> $found_viruses{$a} }
820     keys %found_viruses)
821     {
822     - if (index($virus,"Sanesecurity")!=-1){
823     + if (index($virus,"Sanesecurity") !=-1 || index($virus,"UNOFFICIAL") !=-1){
824     print "Rejected $found_viruses{$virus}\thttp://sane.mxuptime.com/s.aspx?id=$virus\n";
825     } else {
826     print "Rejected $found_viruses{$virus}\t$virus\n";
827     @@ -1061,6 +1316,7 @@
828     print "$found_qpcodes{$qpcode}\t".sprintf('%4.1f',$found_qpcodes{$qpcode}*100/$totalexamined)."%\t$qpcode\n" if $totalexamined;
829     }
830     print("---------------------------------------------\n\n");
831     + print "\n\n";
832     }
833    
834     sub show_Geoip_results
835     @@ -1078,38 +1334,40 @@
836     } else {
837     $percentthreshold = 0.5;
838     }
839     - print("Geoip results: (cutoff at $percentthreshold%) \n");
840     - print("---------------------------------\n");
841     - print("Country\tPercent\tCount\tRejected?\n");
842     - print("---------------------------------\n");
843     - foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
844     - keys %found_countries)
845     - {
846     - $percent = $found_countries{$country} * 100 / $total_countries
847     - if $total_countries;
848     - $totalpercent = $totalpercent + $percent;
849     - if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
850     - if ( $percent >= $percentthreshold ) {
851     - print "$country\t"
852     - . sprintf( '%4.1f', $percent )
853     - . "%\t$found_countries{$country}","\t$reject\n"
854     - if $total_countries;
855     - }
856     -
857     - }
858     - print("---------------------------------\n");
859     - my ($showtotals);
860     - if ($cdb->get('mailstats')){
861     - $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
862     - } else {
863     - $showtotals = $true;
864     - }
865     -
866     - if ($showtotals){
867     - print "TOTALS\t$totalpercent%\t$total_countries\n";
868     - print("---------------------------------\n\n");
869     + if ($total_countries > 0) {
870     + print("Geoip results: (cutoff at $percentthreshold%) \n");
871     + print("---------------------------------\n");
872     + print("Country\tPercent\tCount\tRejected?\n");
873     + print("---------------------------------\n");
874     + foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} }
875     + keys %found_countries)
876     + {
877     + $percent = $found_countries{$country} * 100 / $total_countries
878     + if $total_countries;
879     + $totalpercent = $totalpercent + $percent;
880     + if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";}
881     + if ( $percent >= $percentthreshold ) {
882     + print "$country\t"
883     + . sprintf( '%4.1f', $percent )
884     + . "%\t$found_countries{$country}","\t$reject\n"
885     + if $total_countries;
886     + }
887     +
888     + }
889     + print("---------------------------------\n");
890     + my ($showtotals);
891     + if ($cdb->get('mailstats')){
892     + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
893     + } else {
894     + $showtotals = $true;
895     + }
896     +
897     + if ($showtotals){
898     + print "TOTALS\t".sprintf("%4.1f",$totalpercent)."%\t$total_countries\n";
899     + print("---------------------------------\n\n");
900     + }
901     + print "\n";
902     }
903     - print "\n";
904     }
905    
906     sub show_SARules_codes
907     @@ -1123,52 +1381,55 @@
908     my ($percentthreshold);
909     my ($defaultpercentthreshold);
910     my ($totalpercent) = 0;
911     -
912     - if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
913     - $defaultpercentthreshold = $maxcutoff
914     - } else {
915     - $defaultpercentthreshold = $mincutoff
916     - }
917     - if ($cdb->get('mailstats')){
918     - $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold;
919     - } else {
920     - $percentthreshold = $defaultpercentthreshold
921     - }
922     -
923     - print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
924     - print("---------------------------------------------\n");
925     - print("Count\tPercent\tScore\t\t\n");
926     - print("---------------------------------------------\n");
927     - foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
928     - keys %found_SARules)
929     - {
930     - my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
931     - if $totalexamined;
932     - #$totalpercent = $totalpercent + $percent;
933     - my $avehits = $found_SARules{$SARule}{'totalhits'} /
934     - $found_SARules{$SARule}{'count'}
935     - if $found_SARules{$SARule}{'count'};
936     - if ( $percent >= $percentthreshold ) {
937     - print "$found_SARules{$SARule}{'count'}\t"
938     - . sprintf( '%4.1f', $percent ) . "%\t"
939     - . sprintf( '%4.1f', $avehits )
940     - . "\t$SARule\n"
941     +
942     + if ($sum_SARules > 0){
943     +
944     + if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) {
945     + $defaultpercentthreshold = $maxcutoff
946     + } else {
947     + $defaultpercentthreshold = $mincutoff
948     + }
949     + if ($cdb->get('mailstats')){
950     + $percentthreshold = $cdb->get('mailstats')->prop("SARulePercentThreshold") || $defaultpercentthreshold;
951     + } else {
952     + $percentthreshold = $defaultpercentthreshold
953     + }
954     +
955     + print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n");
956     + print("---------------------------------------------\n");
957     + print("Count\tPercent\tScore\t\t\n");
958     + print("---------------------------------------------\n");
959     + foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} }
960     + keys %found_SARules)
961     + {
962     + my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined
963     if $totalexamined;
964     -}
965     - }
966     - print("---------------------------------------------\n");
967     - my ($showtotals);
968     - if ($cdb->get('mailstats')){
969     - $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
970     - } else {
971     - $showtotals = $true;
972     - }
973     -
974     - if ($showtotals){
975     - print "$totalexamined\t(TOTALS)\n";
976     + #$totalpercent = $totalpercent + $percent;
977     + my $avehits = $found_SARules{$SARule}{'totalhits'} /
978     + $found_SARules{$SARule}{'count'}
979     + if $found_SARules{$SARule}{'count'};
980     + if ( $percent >= $percentthreshold ) {
981     + print "$found_SARules{$SARule}{'count'}\t"
982     + . sprintf( '%4.1f', $percent ) . "%\t"
983     + . sprintf( '%4.1f', $avehits )
984     + . "\t$SARule\n"
985     + if $totalexamined;
986     + }
987     + }
988     print("---------------------------------------------\n");
989     + my ($showtotals);
990     + if ($cdb->get('mailstats')){
991     + $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes");
992     + } else {
993     + $showtotals = $true;
994     + }
995     +
996     + if ($showtotals){
997     + print "$totalexamined\t(TOTALS)\n";
998     + print("---------------------------------------------\n");
999     + }
1000     + print "\n";
1001     }
1002     - print "\n";
1003    
1004    
1005     }
1006     @@ -1370,9 +1631,15 @@
1007     }
1008     $nhour++;
1009     }
1010     - $dbh->disconnect();
1011     - my $telapsed = time - $tstart;
1012     - print "Saved $reccount records in $telapsed sec.";
1013     + # and write out the log lines saved
1014     +
1015     + foreach my $logid (keys %LogLines){
1016     +
1017     + $dbh->do("INSERT INTO LogData (MailID,Sequence,LogStr) VALUES ('".$logid."','"."1','".$LogLines{$logid}."')");
1018     + }
1019     + $dbh->disconnect();
1020     + my $telapsed = time - $tstart;
1021     + print "Saved $reccount records in $telapsed sec.";
1022     }
1023    
1024     sub check_date_rec
1025     @@ -1439,5 +1706,3 @@
1026     my $daterec = $sth->fetchrow_hashref();
1027     $daterec->{"dateid"};
1028     }
1029     -
1030     -

admin@koozali.org
ViewVC Help
Powered by ViewVC 1.2.1 RSS 2.0 feed