diff --git a/xtr.pl b/xtr.pl index 2e81b4d..6bc130b 100644 --- a/xtr.pl +++ b/xtr.pl @@ -7,30 +7,41 @@ use Net::DNS::Resolver; use Net::Whois::IP; use Dancer2; use Socket; -use DBI; use File::Fetch; -use PerlIO::gzip; use HTTP::Tiny; use HTTP::Request; -use Data::Validate::IP qw(is_ipv4); +use Data::Validate::Domain qw(is_domain); +use Data::Validate::IP qw(is_public_ipv4 is_public_ipv6); +use MaxMind::DB::Reader; +use Archive::Tar; use Data::Dumper; ### README ######################################################################################################################################### # This is the client for the XTR (XiCoN Trace Route). It opens a port and listening to HTTP requests. It answers with a traceroute -# in a json hash. The hash contains for each hop the IP, the PTR, the request time and the AS (Autonomous System). +# in a json hash. The hash contains for each hop the IP, the PTR, the request time and the AS (Autonomous System) including the AS name. # # This client also tries to connect to a "master server" which helps keeping track of all the traceroute clients and their availability. # -# This software uses the iptoasn database (https://iptoasn.com/) to determine the AS for each hop. To keep track of all ip to asn relations, -# this software creates a local database file "ip2asn.db", which is a simple SQLite3 database with an index for faster searches. +# This software uses the MaxMind GeoLite2 database (https://www.maxmind.com) to determine the AS for each hop. To lookup the AS information the script +# uses the MaxMind perl module which uses the MaxMind DB library. # -# For now, the traceroutes are only IPv4. This will change in the future. +# +# This product includes GeoLite2 data created by MaxMind, available from +# https://www.maxmind.com ###################################################################################################################################################### ### CHANGELOG ###################################################################################################################################### +# v0.7 (2019-10-18) +# * ADD: IPv6 requests are now possible +# * ADD: Domain request are now possible (will be resolved to ip address) +# * DEL: removed ip2asn database as source for AS lookup (no more SQLite for client) +# * ADD: added MaxMind's GeoLite2-ASN database for AS lookup +# * FIX: unknown domains or invalid IPs will be reported as a valid result including error message as object +# * FIX: cleaned up source code by moving more code into funtions +# # v0.6 (2019-08-22) # * FIX: replaced bug-driven LWP::* with HTTP::Tiny # * FIX: client_protocol_version moved from "3" to "4" @@ -52,23 +63,30 @@ use Data::Dumper; # # Either install all listed modules with "cpan -i " or use your system's package manager (apt, yum, yast). # -# On Debian just install there packages: -# libnet-whois-ip-perl -# libperlio-gzip-perl -# libdancer2-perl -# libnet-dns-perl -# libnet-traceroute-perl -# libdbi-perl -# libdbd-sqlite3-perl -# libdata-validate-ip-perl +# On Debian just install these packages: +# wget +# screen +# libnet-whois-ip-perl +# libdancer2-perl +# libnet-dns-perl +# libnet-traceroute-perl +# libdata-validate-ip-perl +# libdata-validate-domain-perl +# build-essential +# cpanminus +# libmaxminddb0 +# libmaxminddb-dev +# +# After that, execute this command to install maxmind database reader module: +# cpanm -q -n MaxMind::DB::Reader::XS # ###################################################################################################################################################### ### VARS ########################################################################################################################################### -my $VERSION = "0.6"; -my $dbfile = 'ip2asn.db'; -my $ip2asn_csv_url = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz'; +my $VERSION = "0.7"; +my $dbfile = 'GeoLite2-ASN.mmdb'; +my $db_source = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz'; my $master_server = 'xtr-master.xicon.eu'; my $get_my_ip_service = 'http://ipv4.xicon.eu/'; # some service, which returns just the ip of the requesting host (in this case, us.) my $client_protocol_version = 4; @@ -90,12 +108,10 @@ if($my_ext_ip eq "") { $my_ext_ip = $my_ip; } if($my_ext_ip eq "0.0.0.0") { die "Couldn't determine my own IP"; } if($my_ext_port eq 0 || $my_ext_port eq "") { $my_ext_port = $my_port; } -### connect to database -my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","",""); -$dbh->do("PRAGMA cache_size = 800000"); +get_maxmind_db($db_source,$dbfile); -### create database & table if needed and fill with ip information -if(create_db_table($dbh)) { get_ipas_data($dbh,$ip2asn_csv_url); } +### connect to database +my $maxmind_reader = MaxMind::DB::Reader->new( file => $dbfile ); ### hook for HTTP "security" hook 'before' => sub { @@ -103,10 +119,18 @@ hook 'before' => sub { }; ### main get route -get '/v3/client/request/:ip' => sub { - my $ip = route_parameters->get('ip') || 8.8.8.8; - if(!is_ipv4($ip)) { status('bad_request'); return "invalid IP address" } - my $trace = traceit($dbh,$ip); +get '/v3/client/request/:host' => sub { + my $host = route_parameters->get('host') || 8.8.8.8; + + if(!is_public_ipv4($host) and !is_public_ipv6($host) and !is_domain($host)) + { + #status('bad_request'); + my @array; + push(@array, ["NO","ANSWER","FOUND",":(",""]); + return encode_json \@array + } + + my $trace = traceit($maxmind_reader,$host); return encode_json $trace; }; @@ -150,7 +174,7 @@ sub get_my_ipv4 my ($ip_service) = @_; my $httptiny = HTTP::Tiny->new("timeout" => 10); my $response = $httptiny->request("GET",$ip_service); - if(is_ipv4($response->{'content'})) + if(is_public_ipv4($response->{'content'})) { return $response->{'content'}; } @@ -163,131 +187,125 @@ sub get_my_ipv4 sub traceit { - my ($dbh,$host) = @_; + my ($maxmind_reader,$host) = @_; + my @array = (); + + if(is_domain($host)) + { + if(resolve_dns($host,"4") eq "" and resolve_dns($host,"6") eq "") + { + push(@array, ["NO","ANSWER","FOUND",":(",""]); + return \@array; + } + } + my $tr = Net::Traceroute->new(host => $host, use_icmp => 1); - my @array = (); + my $hops = $tr->hops; - if($hops > 1) + if($hops > 1) { foreach my $hop (@{$tr->{'hops'}}) { my $sum = sprintf("%.1f",($hop->[0]->[2] + $hop->[1]->[2] + $hop->[2]->[2])/3); - my $res = Net::DNS::Resolver->new; - my $query = $res->query($hop->[0]->[1], 'PTR'); - my $answer = " "; - if($query) - { - foreach my $rr ($query->answer) - { - if($rr->type eq "PTR") - { - $answer = $rr->ptrdname; - } - } - } + my $answer = resolve_dns($hop->[0]->[1],"PTR"); ### get AS to IP my $as = 0; + my $as_name = " "; + if($hop->[0]->[1] ne "255.255.255.255") { - $as = find_as($dbh, $hop->[0]->[1]); + my $as_data = find_as($maxmind_reader, $hop->[0]->[1]); + if(defined($as_data->{'autonomous_system_number'})) { $as = $as_data->{'autonomous_system_number'}; } + if(defined($as_data->{'autonomous_system_organization'})) { $as_name = $as_data->{'autonomous_system_organization'}; } } - push(@array, [$hop->[0]->[1],$answer,$sum,$as]); + else + { + $hop->[0]->[1] = "???"; + } + push(@array, [$hop->[0]->[1],$answer,$sum,$as,$as_name]); } return \@array; } else { - push(@array, "NO","ANSWER","FOUND"); + push(@array, ["NO","ANSWER","FOUND",":(",""]); return \@array; } } -sub get_ipas_data +sub resolve_dns { - my ($dbh,$ip2asn_csv_url) = @_; + my ($host,$type) = @_; + if($type eq 4) { $type = "A"; } + elsif($type eq 6) { $type = "AAAA"; } + elsif($type eq "PTR") { $type = "PTR"; } + else { return ""; } - my $ff = File::Fetch->new(uri => $ip2asn_csv_url); - my $temp_file = $ff->fetch( to => '/tmp' ); - - open(FILE,"<:gzip",$temp_file) or die "Can't open file: $!"; - my @data = ; - close(FILE); - - $dbh->do('begin'); - - my $max_commit = 10000; - my $inserted = 0; - my $array_size = @data; - my $last_p = -1; - print "loading data...\n"; - foreach my $line (@data) + my $res = Net::DNS::Resolver->new; + my $query = $res->query($host, $type); + my $answer = ""; + if($query) { - chomp($line); - if($line=~/^(\d+)\s+(\d+)\s+(\d+)/) + foreach my $rr ($query->answer) { - if($3 eq "0") { next; } - #my %temp_hash = ('from' => $1, 'to' => $2, 'as' => $3); - #push(@ipaslist,\%temp_hash); - my $insert_cmd .= "INSERT INTO ip2asn VALUES (".$1.",".$2.",".$3.");"; - my $insert_sth = $dbh->prepare($insert_cmd); - $inserted += $insert_sth->execute(); - - # print status for console users - my $p = int($inserted * 100 / $array_size); - if($p != $last_p) { print $p % 10 ? "" : "$p%\n"; } - $last_p = $p; + if($rr->type eq "A") { $answer = $rr->address; } + if($rr->type eq "AAAA") { $answer = $rr->address_short; } + if($rr->type eq "PTR") { $answer = $rr->ptrdname; } + } + } - # only commit every $max_commit statements (it's faster) - unless ($inserted % $max_commit) + return $answer; +} + +sub resolve_dns_reverse +{ + my ($host) = @_; + my $res = Net::DNS::Resolver->new; + my $query = $res->query($host, 'PTR'); + my $answer = " "; + if($query) + { + foreach my $rr ($query->answer) + { + if($rr->type eq "PTR") { - $dbh->do('commit'); - $dbh->do('begin'); + $answer = $rr->ptrdname; } } } - $dbh->do('commit'); - unlink($temp_file); - print "data prepared for searches!\n"; + return $answer; } sub find_as { - my ($dbh, $ip) = @_; - my $ip_id = unpack("N", inet_aton($ip)); - - my $get_as_cmd = "SELECT ip_as FROM ip2asn WHERE ip_from < ".$ip_id." AND ip_to > ".$ip_id.";"; - my $get_as_sth = $dbh->prepare($get_as_cmd); - $get_as_sth->execute(); - my $result = $get_as_sth->fetch; - return $result->[0] || 0; + my ($maxmind_reader, $ip) = @_; + my $record = $maxmind_reader->record_for_address($ip); + return $record; } -sub create_db_table +sub get_maxmind_db { - my ($dbh) = @_; + my ($db_source,$dbfile) = @_; - my $check_table_cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='ip2asn';"; - my $check_table_sth = $dbh->prepare($check_table_cmd); - $check_table_sth->execute(); + my $ff = File::Fetch->new(uri => $db_source); + my $temp_file = $ff->fetch( to => '/tmp' ); - if(!$check_table_sth->fetch) + my $tar = Archive::Tar->new; + $tar->read($temp_file); + + my @files = $tar->get_files(); + foreach my $file (@files) { - print "creating main table...\n"; - my $create_table_cmd = "CREATE TABLE ip2asn( ip_from INTEGER, ip_to INTEGER, ip_as INTEGER)"; - my $create_table_sth = $dbh->prepare($create_table_cmd); - $create_table_sth->execute(); - - print "creating index table...\n"; - my $create_index_cmd = "CREATE INDEX ip2asnIndex ON ip2asn(ip_from,ip_to);"; - my $create_index_sth = $dbh->prepare($create_index_cmd); - $create_index_sth->execute(); - return 1; + if($file->{'name'}=~/${dbfile}$/) + { + open(FILE,">",$dbfile) or die "Can't open file for writing: $!"; + print FILE $file->{'data'}; + close(FILE); + } } - else - { - return 0; - } -} + $tar->clear; + unlink($temp_file); +} \ No newline at end of file