xicon
/
xtr
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

v7

This commit is contained in:
XiCoN-FJS- 2019-10-19 23:26:11 +02:00
parent 64b95749cf
commit f76452b80e
1 changed files with 132 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

246
xtr.pl
View File

@ -7,30 +7,41 @@ use Net::DNS::Resolver;
use Net::Whois::IP; use Net::Whois::IP;
use Dancer2; use Dancer2;
use Socket; use Socket;
use DBI;
use File::Fetch; use File::Fetch;
use PerlIO::gzip;
use HTTP::Tiny; use HTTP::Tiny;
use HTTP::Request; use HTTP::Request;
use Data::Validate::IP qw(is_ipv4); use Data::Validate::Domain qw(is_domain);
use Data::Validate::IP qw(is_public_ipv4 is_public_ipv6);
use MaxMind::DB::Reader;
use Archive::Tar;
use Data::Dumper; use Data::Dumper;
### README ######################################################################################################################################### ### README #########################################################################################################################################
# This is the client for the XTR (XiCoN Trace Route). It opens a port and listening to HTTP requests. It answers with a traceroute # This is the client for the XTR (XiCoN Trace Route). It opens a port and listening to HTTP requests. It answers with a traceroute
# in a json hash. The hash contains for each hop the IP, the PTR, the request time and the AS (Autonomous System). # in a json hash. The hash contains for each hop the IP, the PTR, the request time and the AS (Autonomous System) including the AS name.
# #
# This client also tries to connect to a "master server" which helps keeping track of all the traceroute clients and their availability. # This client also tries to connect to a "master server" which helps keeping track of all the traceroute clients and their availability.
# #
# This software uses the iptoasn database (https://iptoasn.com/) to determine the AS for each hop. To keep track of all ip to asn relations, # This software uses the MaxMind GeoLite2 database (https://www.maxmind.com) to determine the AS for each hop. To lookup the AS information the script
# this software creates a local database file "ip2asn.db", which is a simple SQLite3 database with an index for faster searches. # uses the MaxMind perl module which uses the MaxMind DB library.
# #
# For now, the traceroutes are only IPv4. This will change in the future. #
# This product includes GeoLite2 data created by MaxMind, available from
# <a href="https://www.maxmind.com">https://www.maxmind.com</a>
###################################################################################################################################################### ######################################################################################################################################################
### CHANGELOG ###################################################################################################################################### ### CHANGELOG ######################################################################################################################################
# v0.7 (2019-10-18)
# * ADD: IPv6 requests are now possible
# * ADD: Domain request are now possible (will be resolved to ip address)
# * DEL: removed ip2asn database as source for AS lookup (no more SQLite for client)
# * ADD: added MaxMind's GeoLite2-ASN database for AS lookup
# * FIX: unknown domains or invalid IPs will be reported as a valid result including error message as object
# * FIX: cleaned up source code by moving more code into funtions
#
# v0.6 (2019-08-22) # v0.6 (2019-08-22)
# * FIX: replaced bug-driven LWP::* with HTTP::Tiny # * FIX: replaced bug-driven LWP::* with HTTP::Tiny
# * FIX: client_protocol_version moved from "3" to "4" # * FIX: client_protocol_version moved from "3" to "4"
@ -52,23 +63,30 @@ use Data::Dumper;
# #
# Either install all listed modules with "cpan -i <module>" or use your system's package manager (apt, yum, yast). # Either install all listed modules with "cpan -i <module>" or use your system's package manager (apt, yum, yast).
# #
# On Debian just install there packages: # On Debian just install these packages:
# libnet-whois-ip-perl # wget
# libperlio-gzip-perl # screen
# libdancer2-perl # libnet-whois-ip-perl
# libnet-dns-perl # libdancer2-perl
# libnet-traceroute-perl # libnet-dns-perl
# libdbi-perl # libnet-traceroute-perl
# libdbd-sqlite3-perl # libdata-validate-ip-perl
# libdata-validate-ip-perl # libdata-validate-domain-perl
# build-essential
# cpanminus
# libmaxminddb0
# libmaxminddb-dev
#
# After that, execute this command to install maxmind database reader module:
# cpanm -q -n MaxMind::DB::Reader::XS
# #
###################################################################################################################################################### ######################################################################################################################################################
### VARS ########################################################################################################################################### ### VARS ###########################################################################################################################################
my $VERSION = "0.6"; my $VERSION = "0.7";
my $dbfile = 'ip2asn.db'; my $dbfile = 'GeoLite2-ASN.mmdb';
my $ip2asn_csv_url = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz'; my $db_source = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz';
my $master_server = 'xtr-master.xicon.eu'; my $master_server = 'xtr-master.xicon.eu';
my $get_my_ip_service = 'http://ipv4.xicon.eu/'; # some service, which returns just the ip of the requesting host (in this case, us.) my $get_my_ip_service = 'http://ipv4.xicon.eu/'; # some service, which returns just the ip of the requesting host (in this case, us.)
my $client_protocol_version = 4; my $client_protocol_version = 4;
@ -90,12 +108,10 @@ if($my_ext_ip eq "") { $my_ext_ip = $my_ip; }
if($my_ext_ip eq "0.0.0.0") { die "Couldn't determine my own IP"; } if($my_ext_ip eq "0.0.0.0") { die "Couldn't determine my own IP"; }
if($my_ext_port eq 0 || $my_ext_port eq "") { $my_ext_port = $my_port; } if($my_ext_port eq 0 || $my_ext_port eq "") { $my_ext_port = $my_port; }
### connect to database get_maxmind_db($db_source,$dbfile);
my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","","");
$dbh->do("PRAGMA cache_size = 800000");
### create database & table if needed and fill with ip information ### connect to database
if(create_db_table($dbh)) { get_ipas_data($dbh,$ip2asn_csv_url); } my $maxmind_reader = MaxMind::DB::Reader->new( file => $dbfile );
### hook for HTTP "security" ### hook for HTTP "security"
hook 'before' => sub { hook 'before' => sub {
@ -103,10 +119,18 @@ hook 'before' => sub {
}; };
### main get route ### main get route
get '/v3/client/request/:ip' => sub { get '/v3/client/request/:host' => sub {
my $ip = route_parameters->get('ip') || 8.8.8.8; my $host = route_parameters->get('host') || 8.8.8.8;
if(!is_ipv4($ip)) { status('bad_request'); return "invalid IP address" }
my $trace = traceit($dbh,$ip); if(!is_public_ipv4($host) and !is_public_ipv6($host) and !is_domain($host))
{
#status('bad_request');
my @array;
push(@array, ["NO","ANSWER","FOUND",":(",""]);
return encode_json \@array
}
my $trace = traceit($maxmind_reader,$host);
return encode_json $trace; return encode_json $trace;
}; };
@ -150,7 +174,7 @@ sub get_my_ipv4
my ($ip_service) = @_; my ($ip_service) = @_;
my $httptiny = HTTP::Tiny->new("timeout" => 10); my $httptiny = HTTP::Tiny->new("timeout" => 10);
my $response = $httptiny->request("GET",$ip_service); my $response = $httptiny->request("GET",$ip_service);
if(is_ipv4($response->{'content'})) if(is_public_ipv4($response->{'content'}))
{ {
return $response->{'content'}; return $response->{'content'};
} }
@ -163,131 +187,125 @@ sub get_my_ipv4
sub traceit sub traceit
{ {
my ($dbh,$host) = @_; my ($maxmind_reader,$host) = @_;
my @array = ();
if(is_domain($host))
{
if(resolve_dns($host,"4") eq "" and resolve_dns($host,"6") eq "")
{
push(@array, ["NO","ANSWER","FOUND",":(",""]);
return \@array;
}
}
my $tr = Net::Traceroute->new(host => $host, use_icmp => 1); my $tr = Net::Traceroute->new(host => $host, use_icmp => 1);
my @array = ();
my $hops = $tr->hops; my $hops = $tr->hops;
if($hops > 1) if($hops > 1)
{ {
foreach my $hop (@{$tr->{'hops'}}) foreach my $hop (@{$tr->{'hops'}})
{ {
my $sum = sprintf("%.1f",($hop->[0]->[2] + $hop->[1]->[2] + $hop->[2]->[2])/3); my $sum = sprintf("%.1f",($hop->[0]->[2] + $hop->[1]->[2] + $hop->[2]->[2])/3);
my $res = Net::DNS::Resolver->new; my $answer = resolve_dns($hop->[0]->[1],"PTR");
my $query = $res->query($hop->[0]->[1], 'PTR');
my $answer = " ";
if($query)
{
foreach my $rr ($query->answer)
{
if($rr->type eq "PTR")
{
$answer = $rr->ptrdname;
}
}
}
### get AS to IP ### get AS to IP
my $as = 0; my $as = 0;
my $as_name = " ";
if($hop->[0]->[1] ne "255.255.255.255") if($hop->[0]->[1] ne "255.255.255.255")
{ {
$as = find_as($dbh, $hop->[0]->[1]); my $as_data = find_as($maxmind_reader, $hop->[0]->[1]);
if(defined($as_data->{'autonomous_system_number'})) { $as = $as_data->{'autonomous_system_number'}; }
if(defined($as_data->{'autonomous_system_organization'})) { $as_name = $as_data->{'autonomous_system_organization'}; }
} }
push(@array, [$hop->[0]->[1],$answer,$sum,$as]); else
{
$hop->[0]->[1] = "???";
}
push(@array, [$hop->[0]->[1],$answer,$sum,$as,$as_name]);
} }
return \@array; return \@array;
} }
else else
{ {
push(@array, "NO","ANSWER","FOUND"); push(@array, ["NO","ANSWER","FOUND",":(",""]);
return \@array; return \@array;
} }
} }
sub get_ipas_data sub resolve_dns
{ {
my ($dbh,$ip2asn_csv_url) = @_; my ($host,$type) = @_;
if($type eq 4) { $type = "A"; }
elsif($type eq 6) { $type = "AAAA"; }
elsif($type eq "PTR") { $type = "PTR"; }
else { return ""; }
my $ff = File::Fetch->new(uri => $ip2asn_csv_url); my $res = Net::DNS::Resolver->new;
my $temp_file = $ff->fetch( to => '/tmp' ); my $query = $res->query($host, $type);
my $answer = "";
open(FILE,"<:gzip",$temp_file) or die "Can't open file: $!"; if($query)
my @data = <FILE>;
close(FILE);
$dbh->do('begin');
my $max_commit = 10000;
my $inserted = 0;
my $array_size = @data;
my $last_p = -1;
print "loading data...\n";
foreach my $line (@data)
{ {
chomp($line); foreach my $rr ($query->answer)
if($line=~/^(\d+)\s+(\d+)\s+(\d+)/)
{ {
if($3 eq "0") { next; } if($rr->type eq "A") { $answer = $rr->address; }
#my %temp_hash = ('from' => $1, 'to' => $2, 'as' => $3); if($rr->type eq "AAAA") { $answer = $rr->address_short; }
#push(@ipaslist,\%temp_hash); if($rr->type eq "PTR") { $answer = $rr->ptrdname; }
my $insert_cmd .= "INSERT INTO ip2asn VALUES (".$1.",".$2.",".$3.");"; }
my $insert_sth = $dbh->prepare($insert_cmd); }
$inserted += $insert_sth->execute();
# print status for console users
my $p = int($inserted * 100 / $array_size);
if($p != $last_p) { print $p % 10 ? "" : "$p%\n"; }
$last_p = $p;
# only commit every $max_commit statements (it's faster) return $answer;
unless ($inserted % $max_commit) }
sub resolve_dns_reverse
{
my ($host) = @_;
my $res = Net::DNS::Resolver->new;
my $query = $res->query($host, 'PTR');
my $answer = " ";
if($query)
{
foreach my $rr ($query->answer)
{
if($rr->type eq "PTR")
{ {
$dbh->do('commit'); $answer = $rr->ptrdname;
$dbh->do('begin');
} }
} }
} }
$dbh->do('commit'); return $answer;
unlink($temp_file);
print "data prepared for searches!\n";
} }
sub find_as sub find_as
{ {
my ($dbh, $ip) = @_; my ($maxmind_reader, $ip) = @_;
my $ip_id = unpack("N", inet_aton($ip)); my $record = $maxmind_reader->record_for_address($ip);
return $record;
my $get_as_cmd = "SELECT ip_as FROM ip2asn WHERE ip_from < ".$ip_id." AND ip_to > ".$ip_id.";";
my $get_as_sth = $dbh->prepare($get_as_cmd);
$get_as_sth->execute();
my $result = $get_as_sth->fetch;
return $result->[0] || 0;
} }
sub create_db_table sub get_maxmind_db
{ {
my ($dbh) = @_; my ($db_source,$dbfile) = @_;
my $check_table_cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='ip2asn';"; my $ff = File::Fetch->new(uri => $db_source);
my $check_table_sth = $dbh->prepare($check_table_cmd); my $temp_file = $ff->fetch( to => '/tmp' );
$check_table_sth->execute();
if(!$check_table_sth->fetch) my $tar = Archive::Tar->new;
$tar->read($temp_file);
my @files = $tar->get_files();
foreach my $file (@files)
{ {
print "creating main table...\n"; if($file->{'name'}=~/${dbfile}$/)
my $create_table_cmd = "CREATE TABLE ip2asn( ip_from INTEGER, ip_to INTEGER, ip_as INTEGER)"; {
my $create_table_sth = $dbh->prepare($create_table_cmd); open(FILE,">",$dbfile) or die "Can't open file for writing: $!";
$create_table_sth->execute(); print FILE $file->{'data'};
close(FILE);
print "creating index table...\n"; }
my $create_index_cmd = "CREATE INDEX ip2asnIndex ON ip2asn(ip_from,ip_to);";
my $create_index_sth = $dbh->prepare($create_index_cmd);
$create_index_sth->execute();
return 1;
} }
else $tar->clear;
{ unlink($temp_file);
return 0; }
}
}