1 |
unnilennium |
1.1 |
diff -Nur smeserver-mailstats-1.1.bz8656.old/root/usr/bin/spamfilter-stats-7.pl smeserver-mailstats-1.1.bz8656/root/usr/bin/spamfilter-stats-7.pl |
2 |
|
|
--- smeserver-mailstats-1.1.bz8656.old/root/usr/bin/spamfilter-stats-7.pl 2015-06-25 21:00:48.064715149 +0200 |
3 |
|
|
+++ smeserver-mailstats-1.1.bz8656/root/usr/bin/spamfilter-stats-7.pl 2015-06-25 21:13:05.272509457 +0200 |
4 |
|
|
@@ -1,5 +1,3 @@ |
5 |
|
|
- |
6 |
|
|
- |
7 |
|
|
#!/usr/bin/perl -w |
8 |
|
|
|
9 |
|
|
############################################################################# |
10 |
|
|
@@ -13,8 +11,11 @@ |
11 |
|
|
# bjr - 02sept12 - Add in qpsmtpd failure code auth::auth_cvm_unix_local as per Bug 7089 |
12 |
|
|
# bjr - 10Jun15 - Sort out multiple files as input parameters as per bug 5613 |
13 |
|
|
# - Sort out geoip failure status as per Bug 4262 |
14 |
|
|
-# - change final message about the DB (it is created automatically these days by the rpm) |
15 |
|
|
-# |
16 |
|
|
+# - change final message about the DB (it is created automatically these days by the rpm) |
17 |
|
|
+# bjr - 17Jun15 - Add annotation showing Badcountries being eliminated |
18 |
|
|
+# - correct Spamfilter details extract, as per Bug 8656 |
19 |
|
|
+# - Add analysis table of Geoip results |
20 |
|
|
+# bjr - 19Jun15 - Add totals for the League tables |
21 |
|
|
# |
22 |
|
|
############################################################################# |
23 |
|
|
# |
24 |
|
|
@@ -25,10 +26,13 @@ |
25 |
|
|
# / <column header> ("yes"|"no"|"auto") - enable, supress or only show if nonzero |
26 |
|
|
# / QpsmtpdCodes ("enabled"|"disabled") |
27 |
|
|
# / SARules ("enabled"|"disabled") |
28 |
|
|
+# / GeoipTable ("enabled"|"disabled") |
29 |
|
|
+# / GeoipCutoffPercent (0.5%) - threshold to show Geoip country in league table |
30 |
|
|
# / JunkMailList ("enabled"|"disabled") |
31 |
|
|
# / SARulePercentThreshold (0.5) - threshold of SArules percentage for report cutoff |
32 |
|
|
# / Email (admin) - email to send report |
33 |
|
|
# / SaveDataToMySQL - save data to MySQL database (default is "no") |
34 |
|
|
+# / ShowLeagueTotals - Show totals row after league tables - (default is "yes") |
35 |
|
|
# / DBHost - MySQL server hostname (default is "localhost"). |
36 |
|
|
# / DBPort - MySQL server post (default is "3306") |
37 |
|
|
# / Interval - "day", "week", "fortnight", "month", "99999" - last is number of seconds (default is day) |
38 |
|
|
@@ -67,7 +71,7 @@ |
39 |
|
|
|
40 |
|
|
#Configuration section |
41 |
|
|
my %opt = ( |
42 |
|
|
- version => '0.6.27', # please update at each change. |
43 |
|
|
+ version => '0.6.28', # please update at each change. |
44 |
|
|
debug => 0, # guess what ? |
45 |
|
|
sendmail => '/usr/sbin/sendmail', # Path to sendmail stub |
46 |
|
|
from => 'spamfilter-stats', # Who is the mail from |
47 |
|
|
@@ -137,6 +141,7 @@ |
48 |
|
|
my $CATEXECUT='Execut.'; |
49 |
|
|
my $CATNONCONF='Non.Conf.'; |
50 |
|
|
my $CATBADCOUNTRIES='Geoip.'; |
51 |
|
|
+my $BadCountryCateg=8; #Careful here this number could change if more added before. |
52 |
|
|
my $CATSPAMDEL='Del.Spam'; |
53 |
|
|
my $CATSPAM='Qued.Spam?'; |
54 |
|
|
my $CATHAM='Ham'; |
55 |
|
|
@@ -158,6 +163,10 @@ |
56 |
|
|
my $warnnoreject = " "; |
57 |
|
|
my $rblnotset = ' '; |
58 |
|
|
|
59 |
|
|
+my %found_countries = (); |
60 |
|
|
+my $total_countries = 0; |
61 |
|
|
+my $BadCountries = ""; #From the DB |
62 |
|
|
+ |
63 |
|
|
my $FS = "\t"; # field separator used by logterse plugin |
64 |
|
|
my %log_items = ( "", "", "", "", "", "", "", "" ); |
65 |
|
|
my $score; |
66 |
|
|
@@ -233,7 +242,7 @@ |
67 |
|
|
|
68 |
|
|
# |
69 |
|
|
#--------------------------------------- |
70 |
|
|
-# Scan the qpsmtpd log file |
71 |
|
|
+# Scan the qpsmtpd log file(s) |
72 |
|
|
#--------------------------------------- |
73 |
|
|
|
74 |
|
|
|
75 |
|
|
@@ -277,12 +286,10 @@ |
76 |
|
|
my @ARGV2; |
77 |
|
|
foreach ( map { glob } @ARGV){ |
78 |
|
|
push(@ARGV2,($_)); |
79 |
|
|
-#print "\nFound $_"; |
80 |
|
|
} |
81 |
|
|
@ARGV=@ARGV2; |
82 |
|
|
|
83 |
|
|
LINE: while (<>) { |
84 |
|
|
-#print "\nLine:$_"; |
85 |
|
|
my($tai,$log) = split(' ',$_,2); |
86 |
|
|
|
87 |
|
|
|
88 |
|
|
@@ -291,10 +298,10 @@ |
89 |
|
|
next LINE if ( $tai gt $endtai ); |
90 |
|
|
|
91 |
|
|
# pull out spamasassin rule lists |
92 |
|
|
- if ( $_ =~m/spamassassin plugin: check_spam:.*hits=(.*), required.*tests=(.*)/ ) |
93 |
|
|
+ if ( $_ =~m/spamassassin plugin.*: check_spam:.*hits=(.*), required.*tests=(.*)/ ) |
94 |
|
|
{ |
95 |
|
|
- my ($SAtests) = split(',',$2); |
96 |
|
|
- foreach my $SAtest ($SAtests) { |
97 |
|
|
+ my (@SAtests) = split(',',$2); |
98 |
|
|
+ foreach my $SAtest (@SAtests) { |
99 |
|
|
if (!$SAtest eq "") { |
100 |
|
|
$found_SARules{$SAtest}{'count'}++; |
101 |
|
|
$found_SARules{$SAtest}{'totalhits'} += $1; |
102 |
|
|
@@ -303,10 +310,17 @@ |
103 |
|
|
} |
104 |
|
|
|
105 |
|
|
} |
106 |
|
|
+ |
107 |
|
|
+ #Pull out Geoip countries for analysis table |
108 |
|
|
+ if ( $_ =~m/check_badcountries plugin \(connect\): GeoIP Country: (.*)/ ) |
109 |
|
|
+ { |
110 |
|
|
+ $found_countries{$1}++; |
111 |
|
|
+ $total_countries++; |
112 |
|
|
+ } |
113 |
|
|
+ |
114 |
|
|
#only select Logterse output |
115 |
|
|
next LINE unless m/terse plugin/; |
116 |
|
|
|
117 |
|
|
-#print "\nFound $_"; |
118 |
|
|
|
119 |
|
|
|
120 |
|
|
my $abstime = Time::TAI64::tai2unix($tai); |
121 |
|
|
@@ -738,9 +752,13 @@ |
122 |
|
|
print "*Fetchml* means connections from Fetchmail delivering email\n"; |
123 |
|
|
} |
124 |
|
|
print "*Local* means connections from workstations on local LAN.\n"; |
125 |
|
|
- print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol.\n"; |
126 |
|
|
- print " or email was to non existant address.\n"; |
127 |
|
|
- print "\n"; |
128 |
|
|
+ print "*Non\.Conf\.* means sending mailserver did not conform to correct protocol"; |
129 |
|
|
+ print " or email was to non existant address.\n"; |
130 |
|
|
+ |
131 |
|
|
+ if ($finaldisplay[$BadCountryCateg]){ |
132 |
|
|
+ $BadCountries = $cdb->get('qpsmtpd')->prop('BadCountries') || "*none*"; |
133 |
|
|
+ print "*Geoip\.*:Bad Countries mask is:".$BadCountries."\n"; |
134 |
|
|
+ } |
135 |
|
|
|
136 |
|
|
if ($QueryNoLogTerse) { |
137 |
|
|
print "* - as no records where found, it looks as though you may not have the *logterse* \nplugin running as part of qpsmtpd \n"; |
138 |
|
|
@@ -839,17 +857,20 @@ |
139 |
|
|
# get enable/disable subsections |
140 |
|
|
my $enableqpsmtpdcodes; |
141 |
|
|
my $enableSARules; |
142 |
|
|
+ my $enableGeoiptable; |
143 |
|
|
my $enablejunkMailList; |
144 |
|
|
my $savedata; |
145 |
|
|
if ($cdb->get('mailstats')){ |
146 |
|
|
$enableqpsmtpdcodes = ($cdb->get('mailstats')->prop("QpsmtpdCodes") || "enabled") eq "enabled" || $false; |
147 |
|
|
$enableSARules = ($cdb->get('mailstats')->prop("SARules") || "enabled") eq "enabled" || $false; |
148 |
|
|
$enablejunkMailList = ($cdb->get('mailstats')->prop("JunkMailList") || "enabled") eq "enabled" || $false; |
149 |
|
|
+ $enableGeoiptable = ($cdb->get('mailstats')->prop("Geoiptable") || "enabled") eq "enabled" || $false; |
150 |
|
|
$savedata = ($cdb->get('mailstats')->prop("SaveDataToMySQL") || "no") eq "yes" || $false; |
151 |
|
|
} else { |
152 |
|
|
$enableqpsmtpdcodes = $true; |
153 |
|
|
$enableSARules = $true; |
154 |
|
|
$enablejunkMailList = $true; |
155 |
|
|
+ $enableGeoiptable = $true; |
156 |
|
|
$savedata = $false; |
157 |
|
|
} |
158 |
|
|
|
159 |
|
|
@@ -857,6 +878,8 @@ |
160 |
|
|
|
161 |
|
|
if ($enableSARules) {show_SARules_codes();} |
162 |
|
|
|
163 |
|
|
+ if ($enableGeoiptable && $finaldisplay[$BadCountryCateg]){show_Geoip_results();} |
164 |
|
|
+ |
165 |
|
|
if ($enablejunkMailList) {List_Junkmail();} |
166 |
|
|
|
167 |
|
|
print "\nDone. Report generated in $telapsed sec.\n\n"; |
168 |
|
|
@@ -1020,6 +1043,55 @@ |
169 |
|
|
print("---------------------------------------------\n\n"); |
170 |
|
|
} |
171 |
|
|
|
172 |
|
|
+sub show_Geoip_results |
173 |
|
|
+# |
174 |
|
|
+# Show league table of GEoip results |
175 |
|
|
+# |
176 |
|
|
+{ |
177 |
|
|
+ |
178 |
|
|
+ my ($percentthreshold); |
179 |
|
|
+ my ($reject); |
180 |
|
|
+ my ($percent); |
181 |
|
|
+ my ($totalpercent)=0; |
182 |
|
|
+ if ($cdb->get('mailstats')){ |
183 |
|
|
+ $percentthreshold = $cdb->get('mailstats')->prop("GeoipCutoffPercent") || 0.5; |
184 |
|
|
+ } else { |
185 |
|
|
+ $percentthreshold = 0.5; |
186 |
|
|
+ } |
187 |
|
|
+ print("Geoip results: (cutoff at $percentthreshold%) \n"); |
188 |
|
|
+ print("---------------------------------\n"); |
189 |
|
|
+ print("Country\tPercent\tCount\tRejected?\n"); |
190 |
|
|
+ print("---------------------------------\n"); |
191 |
|
|
+ foreach my $country (sort { $found_countries{$b} <=> $found_countries{$a} } |
192 |
|
|
+ keys %found_countries) |
193 |
|
|
+ { |
194 |
|
|
+ $percent = $found_countries{$country} * 100 / $total_countries |
195 |
|
|
+ if $total_countries; |
196 |
|
|
+ $totalpercent = $totalpercent + $percent; |
197 |
|
|
+ if (index($BadCountries, $country) != -1) {$reject = "*";} else { $reject = " ";} |
198 |
|
|
+ if ( $percent >= $percentthreshold ) { |
199 |
|
|
+ print "$country\t" |
200 |
|
|
+ . sprintf( '%4.1f', $percent ) |
201 |
|
|
+ . "%\t$found_countries{$country}","\t$reject\n" |
202 |
|
|
+ if $total_countries; |
203 |
|
|
+ } |
204 |
|
|
+ |
205 |
|
|
+ } |
206 |
|
|
+ print("---------------------------------\n"); |
207 |
|
|
+ my ($showtotals); |
208 |
|
|
+ if ($cdb->get('mailstats')){ |
209 |
|
|
+ $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); |
210 |
|
|
+ } else { |
211 |
|
|
+ $showtotals = $true; |
212 |
|
|
+ } |
213 |
|
|
+ |
214 |
|
|
+ if ($showtotals){ |
215 |
|
|
+ print "TOTALS\t$totalpercent%\t$total_countries\n"; |
216 |
|
|
+ print("---------------------------------\n\n"); |
217 |
|
|
+ } |
218 |
|
|
+ print "\n"; |
219 |
|
|
+} |
220 |
|
|
+ |
221 |
|
|
sub show_SARules_codes |
222 |
|
|
|
223 |
|
|
# |
224 |
|
|
@@ -1028,9 +1100,9 @@ |
225 |
|
|
# |
226 |
|
|
|
227 |
|
|
{ |
228 |
|
|
- |
229 |
|
|
my ($percentthreshold); |
230 |
|
|
my ($defaultpercentthreshold); |
231 |
|
|
+ my ($totalpercent) = 0; |
232 |
|
|
|
233 |
|
|
if ($totalexamined >0 && $sum_SARules*100/$totalexamined > $SARulethresholdPercent) { |
234 |
|
|
$defaultpercentthreshold = $maxcutoff |
235 |
|
|
@@ -1042,27 +1114,41 @@ |
236 |
|
|
} else { |
237 |
|
|
$percentthreshold = $defaultpercentthreshold |
238 |
|
|
} |
239 |
|
|
- print("Spamassassin Rules:\n"); |
240 |
|
|
+ |
241 |
|
|
+ print("Spamassassin Rules:(cutoff at ".sprintf('%4.1f',$percentthreshold)."%)\n"); |
242 |
|
|
print("---------------------------------------------\n"); |
243 |
|
|
- print("Count\tPercent\tRule\t\n"); |
244 |
|
|
+ print("Count\tPercent\tScore\t\t\n"); |
245 |
|
|
print("---------------------------------------------\n"); |
246 |
|
|
foreach my $SARule (sort { $found_SARules{$b}{'count'} <=> $found_SARules{$a}{'count'} } |
247 |
|
|
keys %found_SARules) |
248 |
|
|
{ |
249 |
|
|
- my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined |
250 |
|
|
- if $totalexamined; |
251 |
|
|
- my $avehits = $found_SARules{$SARule}{'totalhits'} / |
252 |
|
|
- $found_SARules{$SARule}{'count'} |
253 |
|
|
- if $found_SARules{$SARule}{'count'}; |
254 |
|
|
- if ( $percent > $percentthreshold ) { |
255 |
|
|
- print "$found_SARules{$SARule}{'count'}\t" |
256 |
|
|
- . sprintf( '%4.1f', $percent ) . "%\t" |
257 |
|
|
- . sprintf( '%4.1f', $avehits ) |
258 |
|
|
- . "\t$SARule\n" |
259 |
|
|
- if $totalexamined; |
260 |
|
|
- } |
261 |
|
|
+ my $percent = $found_SARules{$SARule}{'count'} * 100 / $totalexamined |
262 |
|
|
+ if $totalexamined; |
263 |
|
|
+ #$totalpercent = $totalpercent + $percent; |
264 |
|
|
+ my $avehits = $found_SARules{$SARule}{'totalhits'} / |
265 |
|
|
+ $found_SARules{$SARule}{'count'} |
266 |
|
|
+ if $found_SARules{$SARule}{'count'}; |
267 |
|
|
+ if ( $percent >= $percentthreshold ) { |
268 |
|
|
+ print "$found_SARules{$SARule}{'count'}\t" |
269 |
|
|
+ . sprintf( '%4.1f', $percent ) . "%\t" |
270 |
|
|
+ . sprintf( '%4.1f', $avehits ) |
271 |
|
|
+ . "\t$SARule\n" |
272 |
|
|
+ if $totalexamined; |
273 |
|
|
+} |
274 |
|
|
} |
275 |
|
|
- print("---------------------------------------------\n\n"); |
276 |
|
|
+ print("---------------------------------------------\n"); |
277 |
|
|
+ my ($showtotals); |
278 |
|
|
+ if ($cdb->get('mailstats')){ |
279 |
|
|
+ $showtotals = ((($cdb->get('mailstats')->prop("ShowLeagueTotals")|| 'yes')) eq "yes"); |
280 |
|
|
+ } else { |
281 |
|
|
+ $showtotals = $true; |
282 |
|
|
+ } |
283 |
|
|
+ |
284 |
|
|
+ if ($showtotals){ |
285 |
|
|
+ print "$totalexamined\t(TOTALS)\n"; |
286 |
|
|
+ print("---------------------------------------------\n"); |
287 |
|
|
+ } |
288 |
|
|
+ print "\n"; |
289 |
|
|
|
290 |
|
|
|
291 |
|
|
} |