xtr/xtr.pl

#!/usr/bin/perl

use strict;
use warnings;
use Net::Traceroute;
use Net::DNS::Resolver;
use Net::Whois::IP;
use Dancer2;
use Socket;
use DBI;
use File::Fetch;
use PerlIO::gzip;
use HTTP::Tiny;
use HTTP::Request;
use Data::Validate::IP qw(is_ipv4);
use Data::Dumper;



###  README  #########################################################################################################################################
# This is the client for the XTR (XiCoN Trace Route). It opens a port and listening to HTTP requests. It answers with a traceroute
# in a json hash. The hash contains for each hop the IP, the PTR, the request time and the AS (Autonomous System).
#
# This client also tries to connect to a "master server" which helps keeping track of all the traceroute clients and their availability.
#
# This software uses the iptoasn database (https://iptoasn.com/) to determine the AS for each hop. To keep track of all ip to asn relations,
# this software creates a local database file "ip2asn.db", which is a simple SQLite3 database with an index for faster searches.
#
# For now, the traceroutes are only IPv4. This will change in the future.
######################################################################################################################################################


###  CHANGELOG  ######################################################################################################################################
# v0.6 (2019-08-22)
#      * FIX: replaced bug-driven LWP::* with HTTP::Tiny
#      * FIX: client_protocol_version moved from "3" to "4"
#
# v0.5 (2019-08-18)
#      * FIX: dropped "timeout" option of Net::Traceroute (details -> https://rt.cpan.org/Ticket/Display.html?id=107066)
#      * FIX: altered dependency on which an output will be send ("$tr->found" -> "$tr->hops > 1") 
#
# v0.4 (2019-08-13)
#      * ADD: added better documentation
#      * ADD: updated API to v3
#      * FIX: limied inputs to ipv4 addresses only (no more hostnames for now in requests)
######################################################################################################################################################


###  INSTALL  ########################################################################################################################################
# This is a standalone software which usually runs in the "foreground". Starting it in a screen session or via init.d/systemd/rc.d is
# highly recommended. For testing, starting this script in a screen session is also fine.
#
# Either install all listed modules with "cpan -i <module>" or use your system's package manager (apt, yum, yast).
# 
# On Debian just install there packages:
#  libnet-whois-ip-perl
#  libperlio-gzip-perl
#  libdancer2-perl
#  libnet-dns-perl
#  libnet-traceroute-perl
#  libdbi-perl
#  libdbd-sqlite3-perl
#  libdata-validate-ip-perl
#
######################################################################################################################################################


###  VARS  ###########################################################################################################################################
my $VERSION = "0.6";
my $dbfile = 'ip2asn.db';
my $ip2asn_csv_url = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz';
my $master_server = 'xtr-master.xicon.eu';
my $get_my_ip_service = 'http://ipv4.xicon.eu/'; # some service, which returns just the ip of the requesting host (in this case, us.)
my $client_protocol_version = 4;
my $my_ip = get_my_ipv4($get_my_ip_service);
my $my_ext_ip = ""; # set different ip, if access differs from default public ip
my $my_port = 12111;
my $my_ext_port = 0; # set different port, if public access port differs from app port
######################################################################################################################################################


######################################################################################################################################################
###  NOTHING TO CHANGE BELOW HERE  ###################################################################################################################
######################################################################################################################################################


### set vars
set port => $my_port;
if($my_ext_ip eq "") { $my_ext_ip = $my_ip; }
if($my_ext_ip eq "0.0.0.0") { die "Couldn't determine my own IP"; }
if($my_ext_port eq 0 || $my_ext_port eq "") { $my_ext_port = $my_port; }

### connect to database
my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","","");
$dbh->do("PRAGMA cache_size = 800000");

### create database & table if needed and fill with ip information
if(create_db_table($dbh)) { get_ipas_data($dbh,$ip2asn_csv_url); }

### hook for HTTP "security"
hook 'before' => sub {
    header 'Access-Control-Allow-Origin' => '*';
};

### main get route
get '/v3/client/request/:ip' => sub {
	my $ip = route_parameters->get('ip') || 8.8.8.8;
	if(!is_ipv4($ip)) { status('bad_request'); return "invalid IP address" }
	my $trace = traceit($dbh,$ip);
	return encode_json $trace;
};

### return client protocol verison
get '/v3/client/info/version' => sub {
	return $client_protocol_version;
};

### give the master server the info, that we are available
if(send_server_status($master_server,$my_ext_ip,$my_ext_port,"1"))
{
	print "send the master server ".$master_server." the info, that we are online.\n";
}
else
{
	print "Failed to send our status to the master server ".$master_server." - nobody knows we are online :(\n";
}

### catch "INT" signal to send the status to the master server
$SIG{'INT'} = sub { send_server_status($master_server,$my_ext_ip,$my_ext_port,"0"); exit; };

### start the main loop
start;

### if you reach this, you're done
exit;



sub send_server_status
{
	my ($master_server,$hostname,$port,$status) = @_;
	my $url = "http://".$master_server."/v3/server/add/".$hostname."/".$port."/".$status;
	my $httptiny = HTTP::Tiny->new("timeout" => 10);
	my $response = $httptiny->request("PUT",$url);
	return $response->{'success'};
}

sub get_my_ipv4
{
	my ($ip_service) = @_;
	my $httptiny = HTTP::Tiny->new("timeout" => 10);
	my $response = $httptiny->request("GET",$ip_service);
	if(is_ipv4($response->{'content'}))
	{
		return $response->{'content'};
	}
	else
	{
		# no valid ip returned from ip service
		return "0.0.0.0";
	}
}

sub traceit
{
	my ($dbh,$host) = @_;
	my $tr = Net::Traceroute->new(host => $host, use_icmp => 1);

	my @array = ();
	my $hops = $tr->hops;
    	if($hops > 1)
	{
		foreach my $hop (@{$tr->{'hops'}})
		{
			my $sum = sprintf("%.1f",($hop->[0]->[2] + $hop->[1]->[2] + $hop->[2]->[2])/3);
			my $res = Net::DNS::Resolver->new;
			my $query = $res->query($hop->[0]->[1], 'PTR');
			my $answer = " ";
			if($query)
			{
				foreach my $rr ($query->answer)
				{
					if($rr->type eq "PTR")
					{
						$answer = $rr->ptrdname;
					}
				}
			}

			### get AS to IP
			my $as = 0;
			if($hop->[0]->[1] ne "255.255.255.255")
			{
				$as = find_as($dbh, $hop->[0]->[1]);
			}
			push(@array, [$hop->[0]->[1],$answer,$sum,$as]);
		}
		return \@array;
	}
	else
	{
		push(@array, "NO","ANSWER","FOUND");
		return \@array;
	}
}

sub get_ipas_data
{
	my ($dbh,$ip2asn_csv_url) = @_;

	my $ff = File::Fetch->new(uri => $ip2asn_csv_url);
	my $temp_file = $ff->fetch( to => '/tmp' );

	open(FILE,"<:gzip",$temp_file) or die "Can't open file: $!";
	my @data = <FILE>;
	close(FILE);

	$dbh->do('begin');

	my $max_commit  = 10000;
	my $inserted    = 0;
	my $array_size  = @data;
	my $last_p      = -1;
	print "loading data...\n";
	foreach my $line (@data)
	{
		chomp($line);
		if($line=~/^(\d+)\s+(\d+)\s+(\d+)/)
		{
			if($3 eq "0") { next; }
			#my %temp_hash = ('from' => $1, 'to' => $2, 'as' => $3);
			#push(@ipaslist,\%temp_hash);
			my $insert_cmd .= "INSERT INTO ip2asn VALUES (".$1.",".$2.",".$3.");";
			my $insert_sth = $dbh->prepare($insert_cmd);
			$inserted += $insert_sth->execute();
			
			# print status for console users
			my $p = int($inserted * 100 / $array_size);
			if($p != $last_p) { print $p % 10 ? "" : "$p%\n"; }
			$last_p = $p;

			# only commit every $max_commit statements (it's faster)	
			unless ($inserted % $max_commit)
			{
				$dbh->do('commit');
				$dbh->do('begin');
			}
		}
	}

	$dbh->do('commit');
	unlink($temp_file);
	print "data prepared for searches!\n";
}

sub find_as
{
	my ($dbh, $ip) = @_;
	my $ip_id = unpack("N", inet_aton($ip));

	my $get_as_cmd = "SELECT ip_as FROM ip2asn WHERE ip_from < ".$ip_id." AND ip_to > ".$ip_id.";";
	my $get_as_sth = $dbh->prepare($get_as_cmd);
	$get_as_sth->execute();
	my $result = $get_as_sth->fetch;
	return $result->[0] || 0;
}

sub create_db_table
{
	my ($dbh) = @_;

	my $check_table_cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='ip2asn';";
	my $check_table_sth = $dbh->prepare($check_table_cmd);
	$check_table_sth->execute();

	if(!$check_table_sth->fetch)
	{
		print "creating main table...\n";
		my $create_table_cmd = "CREATE TABLE ip2asn( ip_from INTEGER, ip_to INTEGER, ip_as INTEGER)";
		my $create_table_sth = $dbh->prepare($create_table_cmd);
		$create_table_sth->execute();

		print "creating index table...\n";
		my $create_index_cmd = "CREATE INDEX ip2asnIndex ON ip2asn(ip_from,ip_to);";
		my $create_index_sth = $dbh->prepare($create_index_cmd);
		$create_index_sth->execute();
		return 1;
	}
	else
	{
		return 0;
	}
}