1 |
diff -Nur smeserver-mailstats-1.1.bz8656.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1.bz8656/root/usr/bin/spamfilter-stats-7.pl |
2 |
--- smeserver-mailstats-1.1.bz8656.old/root/usr/bin/spamfilter-stats-7.pl 2015-06-25 21:00:48.064715149 +0200 |
3 |
+++ smeserver-mailstats-1.1.bz8656/root/usr/bin/spamfilter-stats-7.pl 2015-06-25 21:13:05.272509457 +0200 |
4 |
@@ -1,5 +1,3 @@ |
5 |
- |
6 |
- |
7 |
#!/usr/bin/perl -w |
8 |
|
9 |
############################################################################# |
10 |
@@ -13,8 +11,11 @@ |
11 |
# bjr - 02sept12 - Add in qpsmtpd failure code auth::auth_cvm_unix_local as per Bug 7089 |
12 |
# bjr - 10Jun15 - Sort out multiple files as input parameters as per bug 5613 |
13 |
# - Sort out geoip failure status as per Bug 4262 |
14 |
-# - change final message about the DB (it is created automatically these days by the rpm) |
15 |
-# |
16 |
+# - change final message about the DB (it is created automatically these days by the rpm) |
17 |
+# bjr - 17Jun15 - Add annotation showing Badcountries being eliminated |
18 |
+# - correct Spamfilter details extract, as per Bug 8656 |
19 |
+# - Add analysis table of Geoip results |
20 |
+# bjr - 19Jun15 - Add totals for the League tables |
21 |
# |
22 |
############################################################################# |
23 |
# |
24 |
@@ -25,10 +26,13 @@ |
25 |
# / <column header> ("yes"|"no"|"auto") - enable, supress or only show if nonzero |
26 |
# / QpsmtpdCodes ("enabled"|"disabled") |
27 |
# / SARules ("enabled"|"disabled") |
28 |
+# / GeoipTable ("enabled"|"disabled") |
29 |
+# / GeoipCutoffPercent (0.5%) - threshold to show Geoip country in league table |
30 |
# / JunkMailList ("enabled"|"disabled") |
31 |
# / SARulePercentThreshold (0.5) - threshold of SArules percentage for report cutoff |
32 |
# / Email (admin) - email to send report |
33 |
# / SaveDataToMySQL - save data to MySQL database (default is "no") |
34 |
+# / ShowLeagueTotals - Show totals row after league tables - (default is "yes") |
35 |
# / DBHost - MySQL server hostname (default is "localhost"). |
36 |
# / DBPort - MySQL server post (default is "3306") |
37 |
# / Interval - "day", "week", "fortnight", "month", "99999" - last is number of seconds (default is day) |
38 |
@@ -67,7 +71,7 @@ |
39 |
|
40 |
#Configuration section |
41 |
my %opt = ( |
42 |
- version => '0.6.27', # please update at each change. |
43 |
+ version => '0.6.28', # please update at each change. |
44 |
debug => 0, # guess what ? |
45 |
sendmail => '/usr/sbin/sendmail', # Path to sendmail stub |
46 |
from => 'spamfilter-stats', # Who is the mail from |
47 |
@@ -137,6 +141,7 @@ |
48 |
my $CATEXECUT='Execut.'; |
49 |
my $CATNONCONF='Non.Conf.'; |
50 |
my $CATBADCOUNTRIES='Geoip.'; |
51 |
+my $BadCountryCateg=8; #Careful here this number could change if more added before. |
52 |
my $CATSPAMDEL='Del.Spam'; |
53 |
my $CATSPAM='Qued.Spam?'; |
54 |
my $CATHAM='Ham'; |
55 |
@@ -158,6 +163,10 @@ |
56 |
my $warnnoreject = " "; |
57 |
my $rblnotset = ' '; |
58 |
|
59 |
+my %found_countries = (); |
60 |
+my $total_countries = 0; |
61 |
+my $BadCountries = ""; #From the DB |
62 |
+ |
63 |
my $FS = "\t"; # field separator used by logterse plugin |
64 |
my %log_items = ( "", "", "", "", "", "", "", "" ); |
65 |
my $score; |
66 |
@@ -233,7 +242,7 @@ |
67 |
|
68 |
# |
69 |
#--------------------------------------- |
70 |
-# Scan the qpsmtpd log file |
71 |
+# Scan the qpsmtpd log file(s) |
72 |
#--------------------------------------- |
73 |
|
74 |
|
75 |
@@ -277,12 +286,10 @@ |
76 |
my @ARGV2; |
77 |
foreach ( map { glob } @ARGV){ |
78 |
push(@ARGV2,($_)); |
79 |
-#print "\nFound $_"; |
80 |
} |
81 |
@ARGV=@ARGV2; |
82 |
|
83 |
LINE: while (<>) { |
84 |
-#print "\nLine:$_"; |
85 |
my($tai,$log) = split(' ',$_,2); |
86 |
|
87 |
|
88 |
@@ -291,10 +298,10 @@ |
89 |
next LINE if ( $tai gt $endtai ); |
90 |
|
91 |
# pull out spamasassin rule lists |
92 |
- if ( $_ =~m/spamassassin plugin: check_spam:.*hits=(.*), required.*tests=(.*)/ ) |
93 |
+ if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ ) |
94 |
{ |
95 |
- my ($SAtests) = split(',',$2); |
96 |
- foreach my $SAtest ($SAtests) { |
97 |
+ my (@SAtests) = split(',',$2); |
98 |
+ foreach my $SAtest (@SAtests) { |
99 |
if (!$SAtest eq "") { |
100 |
$found_SARules{$SAtest}{'count'}++; |
101 |
$found_SARules{$SAtest}{'totalhits'} += $1; |
102 |
@@ -303,10 +310,17 @@ |
103 |
} |
104 |
|
105 |
} |
106 |
+ |
107 |
+ #Pull out Geoip countries for analysis table |
108 |
+ if ( $_ =~m/check_badcountries plugin \(connect\): GeoIP Country: (.*)/ ) |
109 |
+ { |
110 |
+ $found_countries{$1}++; |
111 |
+ $total_countries++; |
112 |
+ } |
113 |
+ |
114 |
#only select Logterse output |
115 |
next LINE unless m/terse plugin/; |
116 |
|
117 |
-#print "\nFound $_"; |
118 |
|
119 |
|
120 |
my $abstime = Time::TAI64::tai2unix($tai); |
121 |
@@ -738,9 +752,13 @@ |
122 |
print "*Fetchml* means connections from Fetchmail delivering email\n"; |
123 |
} |
124 |
print "*Local* means connections from workstations on local LAN.\n"; |
125 |
- print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol.\n"; |
126 |
- print " or email was to non existant address.\n"; |
127 |
- print "\n"; |
128 |
+ print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol"; |
129 |
+ print " or email was to non existant address.\n"; |
130 |
+ |
131 |
+ if ($finaldisplay[$BadCountryCateg]){ |
132 |
+ $BadCountries = $cdb->get('qpsmtpd')->prop('BadCountries') || "*none*"; |
133 |
+ print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n"; |
134 |
+ } |
135 |
|
136 |
if ($QueryNoLogTerse) { |
137 |
print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; |
138 |
@@ -839,17 +857,20 @@ |
139 |
# get enable/disable subsections |
140 |
my $enableqpsmtpdcodes; |
141 |
my $enableSARules; |
142 |
+ my $enableGeoiptable; |
143 |
my $enablejunkMailList; |
144 |
my $savedata; |
145 |
if ($cdb->get('mailstats')){ |
146 |
$enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false; |
147 |
$enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false; |
148 |
$enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false; |
149 |
+ $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false; |
150 |
$savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false; |
151 |
} else { |
152 |
$enableqpsmtpdcodes = $true; |
153 |
$enableSARules = $true; |
154 |
$enablejunkMailList = $true; |
155 |
+ $enableGeoiptable = $true; |
156 |
$savedata = $false; |
157 |
} |
158 |
|
159 |
@@ -857,6 +878,8 @@ |
160 |
|
161 |
if ($enableSARules) {show_SARules_codes();} |
162 |
|
163 |
+ if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();} |
164 |
+ |
165 |
if ($enablejunkMailList) {List_Junkmail();} |
166 |
|
167 |
print "\nDone. Report generated in $telapsed sec.\n\n"; |
168 |
@@ -1020,6 +1043,55 @@ |
169 |
print("---------------------------------------------\n\n"); |
170 |
} |
171 |
|
172 |
+sub show_Geoip_results |
173 |
+# |
174 |
+# Show league table of GEoip results |
175 |
+# |
176 |
+{ |
177 |
+ |
178 |
+ my ($percentthreshold); |
179 |
+ my ($reject); |
180 |
+ my ($percent); |
181 |
+ my ($totalpercent)=0; |
182 |
+ if ($cdb->get('mailstats')){ |
183 |
+ $percentthreshold = $cdb->get('mailstats')->prop("GeoipCutoffPercent") || 0.5; |
184 |
+ } else { |
185 |
+ $percentthreshold = 0.5; |
186 |
+ } |
187 |
+ print("Geoip results: (cutoff at $percentthreshold%) \n"); |
188 |
+ print("---------------------------------\n"); |
189 |
+ print("Country\tPercent\tCount\tRejected?\n"); |
190 |
+ print("---------------------------------\n"); |
191 |
+ foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} } |
192 |
+ keys %found_countries) |
193 |
+ { |
194 |
+ $percent = $found_countries{$country} * 100 / $total_countries |
195 |
+ if $total_countries; |
196 |
+ $totalpercent = $totalpercent + $percent; |
197 |
+ if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";} |
198 |
+ if ( $percent >= $percentthreshold ) { |
199 |
+ print "$country\t" |
200 |
+ . sprintf( '%4.1f', $percent ) |
201 |
+ . "%\t$found_countries{$country}","\t$reject\n" |
202 |
+ if $total_countries; |
203 |
+ } |
204 |
+ |
205 |
+ } |
206 |
+ print("---------------------------------\n"); |
207 |
+ my ($showtotals); |
208 |
+ if ($cdb->get('mailstats')){ |
209 |
+ $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); |
210 |
+ } else { |
211 |
+ $showtotals = $true; |
212 |
+ } |
213 |
+ |
214 |
+ if ($showtotals){ |
215 |
+ print "TOTALS\t$totalpercent%\t$total_countries\n"; |
216 |
+ print("---------------------------------\n\n"); |
217 |
+ } |
218 |
+ print "\n"; |
219 |
+} |
220 |
+ |
221 |
sub show_SARules_codes |
222 |
|
223 |
# |
224 |
@@ -1028,9 +1100,9 @@ |
225 |
# |
226 |
|
227 |
{ |
228 |
- |
229 |
my ($percentthreshold); |
230 |
my ($defaultpercentthreshold); |
231 |
+ my ($totalpercent) = 0; |
232 |
|
233 |
if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { |
234 |
$defaultpercentthreshold = $maxcutoff |
235 |
@@ -1042,27 +1114,41 @@ |
236 |
} else { |
237 |
$percentthreshold = $defaultpercentthreshold |
238 |
} |
239 |
- print("Spamassassin Rules:\n"); |
240 |
+ |
241 |
+ print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n"); |
242 |
print("---------------------------------------------\n"); |
243 |
- print("Count\tPercent\tRule\t\n"); |
244 |
+ print("Count\tPercent\tScore\t\t\n"); |
245 |
print("---------------------------------------------\n"); |
246 |
foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} } |
247 |
keys %found_SARules) |
248 |
{ |
249 |
- my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined |
250 |
- if $totalexamined; |
251 |
- my $avehits = $found_SARules{$SARule}{'totalhits'} / |
252 |
- $found_SARules{$SARule}{'count'} |
253 |
- if $found_SARules{$SARule}{'count'}; |
254 |
- if ( $percent > $percentthreshold ) { |
255 |
- print "$found_SARules{$SARule}{'count'}\t" |
256 |
- . sprintf( '%4.1f', $percent ) . "%\t" |
257 |
- . sprintf( '%4.1f', $avehits ) |
258 |
- . "\t$SARule\n" |
259 |
- if $totalexamined; |
260 |
- } |
261 |
+ my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined |
262 |
+ if $totalexamined; |
263 |
+ #$totalpercent = $totalpercent + $percent; |
264 |
+ my $avehits = $found_SARules{$SARule}{'totalhits'} / |
265 |
+ $found_SARules{$SARule}{'count'} |
266 |
+ if $found_SARules{$SARule}{'count'}; |
267 |
+ if ( $percent >= $percentthreshold ) { |
268 |
+ print "$found_SARules{$SARule}{'count'}\t" |
269 |
+ . sprintf( '%4.1f', $percent ) . "%\t" |
270 |
+ . sprintf( '%4.1f', $avehits ) |
271 |
+ . "\t$SARule\n" |
272 |
+ if $totalexamined; |
273 |
+} |
274 |
} |
275 |
- print("---------------------------------------------\n\n"); |
276 |
+ print("---------------------------------------------\n"); |
277 |
+ my ($showtotals); |
278 |
+ if ($cdb->get('mailstats')){ |
279 |
+ $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); |
280 |
+ } else { |
281 |
+ $showtotals = $true; |
282 |
+ } |
283 |
+ |
284 |
+ if ($showtotals){ |
285 |
+ print "$totalexamined\t(TOTALS)\n"; |
286 |
+ print("---------------------------------------------\n"); |
287 |
+ } |
288 |
+ print "\n"; |
289 |
|
290 |
|
291 |
} |