diff -urN xtables-addons-1.47.1-v6/geoip/Makefile.in xtables-addons-1.47.1-g2/geoip/Makefile.in --- xtables-addons-1.47.1-v6/geoip/Makefile.in 2012-10-15 23:29:29.000000000 +0400 +++ xtables-addons-1.47.1-g2/geoip/Makefile.in 2018-07-02 23:03:12.214541572 +0400 @@ -234,7 +234,7 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ xtlibdir = @xtlibdir@ -pkglibexec_SCRIPTS = xt_geoip_build xt_geoip_dl +pkglibexec_SCRIPTS = xt_geoip_build xt_geoip_dl xt_geoip_fetch man1_MANS = xt_geoip_build.1 xt_geoip_dl.1 all: all-am diff -urN xtables-addons-1.47.1-v6/geoip/xt_geoip_build xtables-addons-1.47.1-g2/geoip/xt_geoip_build --- xtables-addons-1.47.1-v6/geoip/xt_geoip_build 2012-10-15 23:29:22.000000000 +0400 +++ xtables-addons-1.47.1-g2/geoip/xt_geoip_build 2018-06-28 22:45:21.000000000 +0400 @@ -1,10 +1,14 @@ #!/usr/bin/perl # # Converter for MaxMind CSV database to binary, for xt_geoip -# Copyright © Jan Engelhardt , 2008-2011 +# Copyright Jan Engelhardt, 2008-2011 +# Copyright Philip Prindeville, 2018 +# Added Twice output (BE, LE) for v1.x for SME9 - Mab974, 2018 # use Getopt::Long; -use IO::Handle; +use Net::CIDR::Lite; +use Socket qw(AF_INET AF_INET6 inet_pton); +use warnings; use Text::CSV_XS; # or trade for Text::CSV use strict; @@ -32,33 +36,208 @@ } } +my %countryId; +my %countryName; + +my $dir = findVersion(); + +&loadCountries(); + &dump(&collect()); -sub collect +sub findVersion { - my %country; - - while (my $row = $csv->getline(*ARGV)) { - if (!defined($country{$row->[4]})) { - $country{$row->[4]} = { - name => $row->[5], - pool_v4 => [], - pool_v6 => [], - }; + my @dirs = (); + my $filename; + + opendir(my $dh, '.') || die "Can't open .: $!\n"; + + while (($filename = readdir($dh))) { + if ($filename =~ m/^GeoLite2-Country-CSV_\d{8}$/) { + push(@dirs, $filename); } - my $c = $country{$row->[4]}; - if ($row->[0] =~ /:/) { - push(@{$c->{pool_v6}}, - [&ip6_pack($row->[0]), &ip6_pack($row->[1])]); + } + closedir $dh; + + @dirs = sort @dirs; + return pop(@dirs); +} + +sub loadCountries +{ + my $file = "$dir/GeoLite2-Country-Locations-en.csv"; + + sub id; sub cc; sub long; sub ct; sub cn; + + %countryId = (); + %countryName = (); + + open(my $fh, '<', $file) || die "Couldn't open list country names\n"; + + # first line is headers + my $row = $csv->getline($fh); + + my %header = map { ($row->[$_], $_); } (0..$#{$row}); + + my %pairs = ( + country_iso_code => 'ISO Country Code', + geoname_id => 'ID', + country_name => 'Country Name', + continent_code => 'Continent Code', + continent_name => 'Continent Name', + ); + + # verify that the columns we need are present + map { die "Table has no $pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs; + + my %remapping = ( + id => 'geoname_id', + cc => 'country_iso_code', + long => 'country_name', + ct => 'continent_code', + cn => 'continent_name', + ); + + # now create a function which returns the value of that column # + map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping; + + while (my $row = $csv->getline($fh)) { + if ($row->[cc] eq '' && $row->[long] eq '') { + $countryId{$row->[id]} = $row->[ct]; + $countryName{$row->[ct]} = $row->[cn]; } else { - push(@{$c->{pool_v4}}, [$row->[2], $row->[3]]); + $countryId{$row->[id]} = $row->[cc]; + $countryName{$row->[cc]} = $row->[long]; + } + } + + $countryName{A1} = 'Anonymous Proxy'; + $countryName{A2} = 'Satellite Provider'; + $countryName{O1} = 'Other Country'; + + close($fh); + + # clean up the namespace + undef &id; undef &cc; undef &long; undef &ct; undef &cn; +} + +sub lookupCountry +{ + my ($id, $rid, $proxy, $sat) = @_; + + if ($proxy) { + return 'A1'; + } elsif ($sat) { + return 'A2'; + } + $id ||= $rid; + if ($id eq '') { + return 'O1'; + } + die "Unknown id: $id line $.\n" unless (exists $countryId{$id}); + return $countryId{$id}; +} + +sub collect +{ + my ($file, $fh, $row); + my (%country, %header); + + sub net; sub id; sub rid; sub proxy; sub sat; + + my %pairs = ( + network => 'Network', + registered_country_geoname_id => 'Registered Country ID', + geoname_id => 'Country ID', + is_anonymous_proxy => 'Anonymous Proxy', + is_satellite_provider => 'Satellite', + ); + + foreach (sort keys %countryName) { + $country{$_} = { + name => $countryName{$_}, + pool_v4 => Net::CIDR::Lite->new(), + pool_v6 => Net::CIDR::Lite->new(), + }; + } + + $file = "$dir/GeoLite2-Country-Blocks-IPv4.csv"; + + open($fh, '<', $file) || die "Can't open IPv4 database\n"; + + # first line is headers + $row = $csv->getline($fh); + + %header = map { ($row->[$_], $_); } (0..$#{$row}); + + # verify that the columns we need are present + map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs; + + my %remapping = ( + net => 'network', + id => 'geoname_id', + rid => 'registered_country_geoname_id', + proxy => 'is_anonymous_proxy', + sat => 'is_satellite_provider', + ); + + # now create a function which returns the value of that column # + map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping; + + while ($row = $csv->getline($fh)) { + my ($cc, $cidr); + + $cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]); + $cidr = $row->[net]; + $country{$cc}->{pool_v4}->add($cidr); + + if ($. % 4096 == 0) { + print STDERR "\r\e[2K$. entries"; } + } + + print STDERR "\r\e[2K$. entries total\n"; + + close($fh); + + # clean up the namespace + undef &net; undef &id; undef &rid; undef &proxy; undef &sat; + + $file = "$dir/GeoLite2-Country-Blocks-IPv6.csv"; + + open($fh, '<', $file) || die "Can't open IPv6 database\n"; + + # first line is headers + $row = $csv->getline($fh); + + %header = map { ($row->[$_], $_); } (0..$#{$row}); + + # verify that the columns we need are present + map { die "Table has no %pairs{$_} column\n" unless (exists $header{$_}); } keys %pairs; + + # unlikely the IPv6 table has different columns, but just to be sure + # create a function which returns the value of that column # + map { eval "sub $_ () { \$header{\$remapping{$_}}; }" ; } keys %remapping; + + while ($row = $csv->getline($fh)) { + my ($cc, $cidr); + + $cc = lookupCountry($row->[id], $row->[rid], $row->[proxy], $row->[sat]); + $cidr = $row->[net]; + $country{$cc}->{pool_v6}->add($cidr); + if ($. % 4096 == 0) { print STDERR "\r\e[2K$. entries"; } } print STDERR "\r\e[2K$. entries total\n"; + + close($fh); + + # clean up the namespace + undef &net; undef &id; undef &rid; undef &proxy; undef &sat; + return \%country; } @@ -66,18 +245,23 @@ { my $country = shift @_; - foreach my $iso_code (sort keys %$country) { + foreach my $iso_code (sort keys %{$country}) { &dump_one($iso_code, $country->{$iso_code}); } } sub dump_one { +# 2 sub-directories added pour big-endian and little-endian + my($iso_code, $country) = @_; my($file, $fh_le, $fh_be); - + my ($start, $end); + my @ranges; + + @ranges = $country->{pool_v6}->list_range(); printf "%5u IPv6 ranges for %s %s\n", - scalar(@{$country->{pool_v6}}), + scalar(@ranges), $iso_code, $country->{name}; $file = "$target_dir/LE/".uc($iso_code).".iv6"; @@ -90,15 +274,23 @@ print STDERR "Error opening $file: $!\n"; exit 1; } - foreach my $range (@{$country->{pool_v6}}) { - print $fh_be $range->[0], $range->[1]; - print $fh_le &ip6_swap($range->[0]), &ip6_swap($range->[1]); + binmode($fh_be); + binmode($fh_le); + + foreach my $range (@ranges) { + ($start, $end) = split('-', $range); + $start = inet_pton(AF_INET6, $start); + $end = inet_pton(AF_INET6, $end); + print $fh_be $start, $end; + + print $fh_le &ip6_swap($start), &ip6_swap($end); } close $fh_le; close $fh_be; + @ranges = $country->{pool_v4}->list_range(); printf "%5u IPv4 ranges for %s %s\n", - scalar(@{$country->{pool_v4}}), + scalar(@ranges), $iso_code, $country->{name}; $file = "$target_dir/LE/".uc($iso_code).".iv4"; @@ -111,31 +303,29 @@ print STDERR "Error opening $file: $!\n"; exit 1; } - foreach my $range (@{$country->{pool_v4}}) { - print $fh_le pack("VV", $range->[0], $range->[1]); - print $fh_be pack("NN", $range->[0], $range->[1]); + binmode($fh_be); + binmode($fh_le); + + foreach my $range (@ranges) { + ($start, $end) = split('-', $range); + my $start = inet_pton(AF_INET, $start); + my $end = inet_pton(AF_INET, $end); + print $fh_be $start, $end; + print $fh_le ip4_swap($start), ip4_swap($end); } close $fh_le; close $fh_be; } -sub ip6_pack +sub ip6_swap { - my $addr = shift @_; - $addr =~ s{::}{:!:}; - my @addr = split(/:/, $addr); - my @e = (0) x 8; - foreach (@addr) { - if ($_ eq "!") { - $_ = join(':', @e[0..(8-scalar(@addr))]); - } - } - @addr = split(/:/, join(':', @addr)); - $_ = hex($_) foreach @addr; - return pack("n*", @addr); + my ($p1, $p2, $p3, $p4) = unpack 'a4 a4 a4 a4', shift @_; + return pack "a4 a4 a4 a4", ip4_swap($p1), ip4_swap($p2), + ip4_swap($p3), ip4_swap($p4); } -sub ip6_swap +sub ip4_swap { - return pack("V*", unpack("N*", shift @_)); + my ($b1, $b2, $b3, $b4) = unpack 'a a a a', shift @_; + return pack "a a a a", $b4, $b3, $b2, $b1; } diff -urN xtables-addons-1.47.1-v6/geoip/xt_geoip_build.1 xtables-addons-1.47.1-g2/geoip/xt_geoip_build.1 --- xtables-addons-1.47.1-v6/geoip/xt_geoip_build.1 2012-10-15 23:29:22.000000000 +0400 +++ xtables-addons-1.47.1-g2/geoip/xt_geoip_build.1 2018-06-21 21:43:45.000000000 +0400 @@ -5,7 +5,7 @@ .SH Syntax .PP \fI/usr/libexec/xt_geoip/\fP\fBxt_geoip_build\fP [\fB\-D\fP -\fItarget_dir\fP] [\fIfile\fP...] +\fItarget_dir\fP] .SH Description .PP xt_geoip_build is used to build packed raw representations of the range @@ -16,7 +16,12 @@ also ordered, as xt_geoip relies on this property for its bisection approach to work. .PP -Input is processed from the listed files, or if none is given, from stdin. +It expects to find a directory named +.IR GeoLite2-Country-CSV_YYYYMMDD +in the current directory, and will select the most recent if multiple +instances are found. The +.IR xt_geoip_dl +script can be used to populate this directory. .PP Since the script is usually installed to the libexec directory of the xtables-addons package and this is outside $PATH (on purpose), invoking the @@ -29,7 +34,7 @@ .PP Shell commands to build the databases and put them to where they are expected: .PP -xt_geoip_build -D /usr/share/xt_geoip +xt_geoip_build \-D /usr/share/xt_geoip .SH See also .PP xt_geoip_dl(1) diff -urN xtables-addons-1.47.1-v6/geoip/xt_geoip_dl xtables-addons-1.47.1-g2/geoip/xt_geoip_dl --- xtables-addons-1.47.1-v6/geoip/xt_geoip_dl 2012-10-15 23:29:22.000000000 +0400 +++ xtables-addons-1.47.1-g2/geoip/xt_geoip_dl 2018-06-25 15:38:51.043475041 +0400 @@ -1,8 +1,7 @@ #!/bin/sh -rm -f GeoIPv6.csv{,.gz} GeoIPCountryCSV.zip GeoIPCountryWhois.csv; -wget \ - http://geolite.maxmind.com/download/geoip/database/GeoIPv6.csv.gz \ - http://geolite.maxmind.com/download/geoip/database/GeoIPCountryCSV.zip; -gzip -d GeoIPv6.csv.gz; -unzip GeoIPCountryCSV.zip; +rm -rf GeoLite2-Country-CSV_* + +wget -q http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip +unzip -q GeoLite2-Country-CSV.zip +rm -f GeoLite2-Country-CSV.zip diff -urN xtables-addons-1.47.1-v6/geoip/xt_geoip_fetch xtables-addons-1.47.1-g2/geoip/xt_geoip_fetch --- xtables-addons-1.47.1-v6/geoip/xt_geoip_fetch 1970-01-01 04:00:00.000000000 +0400 +++ xtables-addons-1.47.1-g2/geoip/xt_geoip_fetch 2018-06-22 19:15:47.000000000 +0400 @@ -0,0 +1,93 @@ +#!/usr/bin/perl +# +# Utility to query GeoIP database +# Copyright Philip Prindeville, 2018 +# +use Getopt::Long; +use Socket qw(AF_INET AF_INET6 inet_ntop); +use warnings; +use strict; + +sub AF_INET_SIZE() { 4 } +sub AF_INET6_SIZE() { 16 } + +my $target_dir = "."; +my $ipv4 = 0; +my $ipv6 = 0; + +&Getopt::Long::Configure(qw(bundling)); +&GetOptions( + "D=s" => \$target_dir, + "4" => \$ipv4, + "6" => \$ipv6, +); + +if (!-d $target_dir) { + print STDERR "Target directory $target_dir does not exit.\n"; + exit 1; +} + +# if neither specified, assume both +if (! $ipv4 && ! $ipv6) { + $ipv4 = $ipv6 = 1; +} + +foreach my $cc (@ARGV) { + if ($cc !~ m/^([a-z]{2}|a[12]|o1)$/i) { + print STDERR "Invalid country code '$cc'\n"; + exit 1; + } + + my $file = $target_dir . '/LE/' . uc($cc) . '.iv4'; + + if (! -f $file) { + printf STDERR "Can't find data for country '$cc'\n"; + exit 1; + } + + my ($contents, $buffer, $bytes, $fh); + + if ($ipv4) { + open($fh, '<', $file) || die "Couldn't open file for '$cc'\n"; + + binmode($fh); + + while (($bytes = read($fh, $buffer, AF_INET_SIZE * 2)) == AF_INET_SIZE * 2) { + my $start = inet_ntop(AF_INET, substr($buffer, 0, AF_INET_SIZE)); + my $end = inet_ntop(AF_INET, substr($buffer, AF_INET_SIZE)); + print $start, '-', $end, "\n"; + } + close($fh); + if (! defined $bytes) { + printf STDERR "Error reading file for '$cc'\n"; + exit 1; + } elsif ($bytes != 0) { + printf STDERR "Short read on file for '$cc'\n"; + exit 1; + } + } + + substr($file, -1) = '6'; + + if ($ipv6) { + open($fh, '<', $file) || die "Couldn't open file for '$cc'\n"; + + binmode($fh); + + while (($bytes = read($fh, $buffer, AF_INET6_SIZE * 2)) == AF_INET6_SIZE * 2) { + my $start = inet_ntop(AF_INET6, substr($buffer, 0, AF_INET6_SIZE)); + my $end = inet_ntop(AF_INET6, substr($buffer, AF_INET6_SIZE)); + print $start, '-', $end, "\n"; + } + close($fh); + if (! defined $bytes) { + printf STDERR "Error reading file for '$cc'\n"; + exit 1; + } elsif ($bytes != 0) { + printf STDERR "Short read on file for '$cc'\n"; + exit 1; + } + } +} + +exit 0;