This commit is contained in:
parent
64b95749cf
commit
f76452b80e
242
xtr.pl
242
xtr.pl
|
|
@ -7,30 +7,41 @@ use Net::DNS::Resolver;
|
|||
use Net::Whois::IP;
|
||||
use Dancer2;
|
||||
use Socket;
|
||||
use DBI;
|
||||
use File::Fetch;
|
||||
use PerlIO::gzip;
|
||||
use HTTP::Tiny;
|
||||
use HTTP::Request;
|
||||
use Data::Validate::IP qw(is_ipv4);
|
||||
use Data::Validate::Domain qw(is_domain);
|
||||
use Data::Validate::IP qw(is_public_ipv4 is_public_ipv6);
|
||||
use MaxMind::DB::Reader;
|
||||
use Archive::Tar;
|
||||
use Data::Dumper;
|
||||
|
||||
|
||||
|
||||
### README #########################################################################################################################################
|
||||
# This is the client for the XTR (XiCoN Trace Route). It opens a port and listening to HTTP requests. It answers with a traceroute
|
||||
# in a json hash. The hash contains for each hop the IP, the PTR, the request time and the AS (Autonomous System).
|
||||
# in a json hash. The hash contains for each hop the IP, the PTR, the request time and the AS (Autonomous System) including the AS name.
|
||||
#
|
||||
# This client also tries to connect to a "master server" which helps keeping track of all the traceroute clients and their availability.
|
||||
#
|
||||
# This software uses the iptoasn database (https://iptoasn.com/) to determine the AS for each hop. To keep track of all ip to asn relations,
|
||||
# this software creates a local database file "ip2asn.db", which is a simple SQLite3 database with an index for faster searches.
|
||||
# This software uses the MaxMind GeoLite2 database (https://www.maxmind.com) to determine the AS for each hop. To lookup the AS information the script
|
||||
# uses the MaxMind perl module which uses the MaxMind DB library.
|
||||
#
|
||||
# For now, the traceroutes are only IPv4. This will change in the future.
|
||||
#
|
||||
# This product includes GeoLite2 data created by MaxMind, available from
|
||||
# <a href="https://www.maxmind.com">https://www.maxmind.com</a>
|
||||
######################################################################################################################################################
|
||||
|
||||
|
||||
### CHANGELOG ######################################################################################################################################
|
||||
# v0.7 (2019-10-18)
|
||||
# * ADD: IPv6 requests are now possible
|
||||
# * ADD: Domain request are now possible (will be resolved to ip address)
|
||||
# * DEL: removed ip2asn database as source for AS lookup (no more SQLite for client)
|
||||
# * ADD: added MaxMind's GeoLite2-ASN database for AS lookup
|
||||
# * FIX: unknown domains or invalid IPs will be reported as a valid result including error message as object
|
||||
# * FIX: cleaned up source code by moving more code into funtions
|
||||
#
|
||||
# v0.6 (2019-08-22)
|
||||
# * FIX: replaced bug-driven LWP::* with HTTP::Tiny
|
||||
# * FIX: client_protocol_version moved from "3" to "4"
|
||||
|
|
@ -52,23 +63,30 @@ use Data::Dumper;
|
|||
#
|
||||
# Either install all listed modules with "cpan -i <module>" or use your system's package manager (apt, yum, yast).
|
||||
#
|
||||
# On Debian just install there packages:
|
||||
# libnet-whois-ip-perl
|
||||
# libperlio-gzip-perl
|
||||
# libdancer2-perl
|
||||
# libnet-dns-perl
|
||||
# libnet-traceroute-perl
|
||||
# libdbi-perl
|
||||
# libdbd-sqlite3-perl
|
||||
# libdata-validate-ip-perl
|
||||
# On Debian just install these packages:
|
||||
# wget
|
||||
# screen
|
||||
# libnet-whois-ip-perl
|
||||
# libdancer2-perl
|
||||
# libnet-dns-perl
|
||||
# libnet-traceroute-perl
|
||||
# libdata-validate-ip-perl
|
||||
# libdata-validate-domain-perl
|
||||
# build-essential
|
||||
# cpanminus
|
||||
# libmaxminddb0
|
||||
# libmaxminddb-dev
|
||||
#
|
||||
# After that, execute this command to install maxmind database reader module:
|
||||
# cpanm -q -n MaxMind::DB::Reader::XS
|
||||
#
|
||||
######################################################################################################################################################
|
||||
|
||||
|
||||
### VARS ###########################################################################################################################################
|
||||
my $VERSION = "0.6";
|
||||
my $dbfile = 'ip2asn.db';
|
||||
my $ip2asn_csv_url = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz';
|
||||
my $VERSION = "0.7";
|
||||
my $dbfile = 'GeoLite2-ASN.mmdb';
|
||||
my $db_source = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz';
|
||||
my $master_server = 'xtr-master.xicon.eu';
|
||||
my $get_my_ip_service = 'http://ipv4.xicon.eu/'; # some service, which returns just the ip of the requesting host (in this case, us.)
|
||||
my $client_protocol_version = 4;
|
||||
|
|
@ -90,12 +108,10 @@ if($my_ext_ip eq "") { $my_ext_ip = $my_ip; }
|
|||
if($my_ext_ip eq "0.0.0.0") { die "Couldn't determine my own IP"; }
|
||||
if($my_ext_port eq 0 || $my_ext_port eq "") { $my_ext_port = $my_port; }
|
||||
|
||||
### connect to database
|
||||
my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","","");
|
||||
$dbh->do("PRAGMA cache_size = 800000");
|
||||
get_maxmind_db($db_source,$dbfile);
|
||||
|
||||
### create database & table if needed and fill with ip information
|
||||
if(create_db_table($dbh)) { get_ipas_data($dbh,$ip2asn_csv_url); }
|
||||
### connect to database
|
||||
my $maxmind_reader = MaxMind::DB::Reader->new( file => $dbfile );
|
||||
|
||||
### hook for HTTP "security"
|
||||
hook 'before' => sub {
|
||||
|
|
@ -103,10 +119,18 @@ hook 'before' => sub {
|
|||
};
|
||||
|
||||
### main get route
|
||||
get '/v3/client/request/:ip' => sub {
|
||||
my $ip = route_parameters->get('ip') || 8.8.8.8;
|
||||
if(!is_ipv4($ip)) { status('bad_request'); return "invalid IP address" }
|
||||
my $trace = traceit($dbh,$ip);
|
||||
get '/v3/client/request/:host' => sub {
|
||||
my $host = route_parameters->get('host') || 8.8.8.8;
|
||||
|
||||
if(!is_public_ipv4($host) and !is_public_ipv6($host) and !is_domain($host))
|
||||
{
|
||||
#status('bad_request');
|
||||
my @array;
|
||||
push(@array, ["NO","ANSWER","FOUND",":(",""]);
|
||||
return encode_json \@array
|
||||
}
|
||||
|
||||
my $trace = traceit($maxmind_reader,$host);
|
||||
return encode_json $trace;
|
||||
};
|
||||
|
||||
|
|
@ -150,7 +174,7 @@ sub get_my_ipv4
|
|||
my ($ip_service) = @_;
|
||||
my $httptiny = HTTP::Tiny->new("timeout" => 10);
|
||||
my $response = $httptiny->request("GET",$ip_service);
|
||||
if(is_ipv4($response->{'content'}))
|
||||
if(is_public_ipv4($response->{'content'}))
|
||||
{
|
||||
return $response->{'content'};
|
||||
}
|
||||
|
|
@ -163,131 +187,125 @@ sub get_my_ipv4
|
|||
|
||||
sub traceit
|
||||
{
|
||||
my ($dbh,$host) = @_;
|
||||
my ($maxmind_reader,$host) = @_;
|
||||
my @array = ();
|
||||
|
||||
if(is_domain($host))
|
||||
{
|
||||
if(resolve_dns($host,"4") eq "" and resolve_dns($host,"6") eq "")
|
||||
{
|
||||
push(@array, ["NO","ANSWER","FOUND",":(",""]);
|
||||
return \@array;
|
||||
}
|
||||
}
|
||||
|
||||
my $tr = Net::Traceroute->new(host => $host, use_icmp => 1);
|
||||
|
||||
my @array = ();
|
||||
|
||||
my $hops = $tr->hops;
|
||||
if($hops > 1)
|
||||
if($hops > 1)
|
||||
{
|
||||
foreach my $hop (@{$tr->{'hops'}})
|
||||
{
|
||||
my $sum = sprintf("%.1f",($hop->[0]->[2] + $hop->[1]->[2] + $hop->[2]->[2])/3);
|
||||
my $res = Net::DNS::Resolver->new;
|
||||
my $query = $res->query($hop->[0]->[1], 'PTR');
|
||||
my $answer = " ";
|
||||
if($query)
|
||||
{
|
||||
foreach my $rr ($query->answer)
|
||||
{
|
||||
if($rr->type eq "PTR")
|
||||
{
|
||||
$answer = $rr->ptrdname;
|
||||
}
|
||||
}
|
||||
}
|
||||
my $answer = resolve_dns($hop->[0]->[1],"PTR");
|
||||
|
||||
### get AS to IP
|
||||
my $as = 0;
|
||||
my $as_name = " ";
|
||||
|
||||
if($hop->[0]->[1] ne "255.255.255.255")
|
||||
{
|
||||
$as = find_as($dbh, $hop->[0]->[1]);
|
||||
my $as_data = find_as($maxmind_reader, $hop->[0]->[1]);
|
||||
if(defined($as_data->{'autonomous_system_number'})) { $as = $as_data->{'autonomous_system_number'}; }
|
||||
if(defined($as_data->{'autonomous_system_organization'})) { $as_name = $as_data->{'autonomous_system_organization'}; }
|
||||
}
|
||||
push(@array, [$hop->[0]->[1],$answer,$sum,$as]);
|
||||
else
|
||||
{
|
||||
$hop->[0]->[1] = "???";
|
||||
}
|
||||
push(@array, [$hop->[0]->[1],$answer,$sum,$as,$as_name]);
|
||||
}
|
||||
return \@array;
|
||||
}
|
||||
else
|
||||
{
|
||||
push(@array, "NO","ANSWER","FOUND");
|
||||
push(@array, ["NO","ANSWER","FOUND",":(",""]);
|
||||
return \@array;
|
||||
}
|
||||
}
|
||||
|
||||
sub get_ipas_data
|
||||
sub resolve_dns
|
||||
{
|
||||
my ($dbh,$ip2asn_csv_url) = @_;
|
||||
my ($host,$type) = @_;
|
||||
if($type eq 4) { $type = "A"; }
|
||||
elsif($type eq 6) { $type = "AAAA"; }
|
||||
elsif($type eq "PTR") { $type = "PTR"; }
|
||||
else { return ""; }
|
||||
|
||||
my $ff = File::Fetch->new(uri => $ip2asn_csv_url);
|
||||
my $temp_file = $ff->fetch( to => '/tmp' );
|
||||
|
||||
open(FILE,"<:gzip",$temp_file) or die "Can't open file: $!";
|
||||
my @data = <FILE>;
|
||||
close(FILE);
|
||||
|
||||
$dbh->do('begin');
|
||||
|
||||
my $max_commit = 10000;
|
||||
my $inserted = 0;
|
||||
my $array_size = @data;
|
||||
my $last_p = -1;
|
||||
print "loading data...\n";
|
||||
foreach my $line (@data)
|
||||
my $res = Net::DNS::Resolver->new;
|
||||
my $query = $res->query($host, $type);
|
||||
my $answer = "";
|
||||
if($query)
|
||||
{
|
||||
chomp($line);
|
||||
if($line=~/^(\d+)\s+(\d+)\s+(\d+)/)
|
||||
foreach my $rr ($query->answer)
|
||||
{
|
||||
if($3 eq "0") { next; }
|
||||
#my %temp_hash = ('from' => $1, 'to' => $2, 'as' => $3);
|
||||
#push(@ipaslist,\%temp_hash);
|
||||
my $insert_cmd .= "INSERT INTO ip2asn VALUES (".$1.",".$2.",".$3.");";
|
||||
my $insert_sth = $dbh->prepare($insert_cmd);
|
||||
$inserted += $insert_sth->execute();
|
||||
if($rr->type eq "A") { $answer = $rr->address; }
|
||||
if($rr->type eq "AAAA") { $answer = $rr->address_short; }
|
||||
if($rr->type eq "PTR") { $answer = $rr->ptrdname; }
|
||||
}
|
||||
}
|
||||
|
||||
# print status for console users
|
||||
my $p = int($inserted * 100 / $array_size);
|
||||
if($p != $last_p) { print $p % 10 ? "" : "$p%\n"; }
|
||||
$last_p = $p;
|
||||
return $answer;
|
||||
}
|
||||
|
||||
# only commit every $max_commit statements (it's faster)
|
||||
unless ($inserted % $max_commit)
|
||||
sub resolve_dns_reverse
|
||||
{
|
||||
my ($host) = @_;
|
||||
my $res = Net::DNS::Resolver->new;
|
||||
my $query = $res->query($host, 'PTR');
|
||||
my $answer = " ";
|
||||
if($query)
|
||||
{
|
||||
foreach my $rr ($query->answer)
|
||||
{
|
||||
if($rr->type eq "PTR")
|
||||
{
|
||||
$dbh->do('commit');
|
||||
$dbh->do('begin');
|
||||
$answer = $rr->ptrdname;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$dbh->do('commit');
|
||||
unlink($temp_file);
|
||||
print "data prepared for searches!\n";
|
||||
return $answer;
|
||||
}
|
||||
|
||||
sub find_as
|
||||
{
|
||||
my ($dbh, $ip) = @_;
|
||||
my $ip_id = unpack("N", inet_aton($ip));
|
||||
|
||||
my $get_as_cmd = "SELECT ip_as FROM ip2asn WHERE ip_from < ".$ip_id." AND ip_to > ".$ip_id.";";
|
||||
my $get_as_sth = $dbh->prepare($get_as_cmd);
|
||||
$get_as_sth->execute();
|
||||
my $result = $get_as_sth->fetch;
|
||||
return $result->[0] || 0;
|
||||
my ($maxmind_reader, $ip) = @_;
|
||||
my $record = $maxmind_reader->record_for_address($ip);
|
||||
return $record;
|
||||
}
|
||||
|
||||
sub create_db_table
|
||||
sub get_maxmind_db
|
||||
{
|
||||
my ($dbh) = @_;
|
||||
my ($db_source,$dbfile) = @_;
|
||||
|
||||
my $check_table_cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='ip2asn';";
|
||||
my $check_table_sth = $dbh->prepare($check_table_cmd);
|
||||
$check_table_sth->execute();
|
||||
my $ff = File::Fetch->new(uri => $db_source);
|
||||
my $temp_file = $ff->fetch( to => '/tmp' );
|
||||
|
||||
if(!$check_table_sth->fetch)
|
||||
my $tar = Archive::Tar->new;
|
||||
$tar->read($temp_file);
|
||||
|
||||
my @files = $tar->get_files();
|
||||
foreach my $file (@files)
|
||||
{
|
||||
print "creating main table...\n";
|
||||
my $create_table_cmd = "CREATE TABLE ip2asn( ip_from INTEGER, ip_to INTEGER, ip_as INTEGER)";
|
||||
my $create_table_sth = $dbh->prepare($create_table_cmd);
|
||||
$create_table_sth->execute();
|
||||
|
||||
print "creating index table...\n";
|
||||
my $create_index_cmd = "CREATE INDEX ip2asnIndex ON ip2asn(ip_from,ip_to);";
|
||||
my $create_index_sth = $dbh->prepare($create_index_cmd);
|
||||
$create_index_sth->execute();
|
||||
return 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0;
|
||||
if($file->{'name'}=~/${dbfile}$/)
|
||||
{
|
||||
open(FILE,">",$dbfile) or die "Can't open file for writing: $!";
|
||||
print FILE $file->{'data'};
|
||||
close(FILE);
|
||||
}
|
||||
}
|
||||
$tar->clear;
|
||||
unlink($temp_file);
|
||||
}
|
||||
Loading…
Reference in New Issue