xicon/xtrd
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

* switched from ip2asn to maxmind-db

Browse Source 
* added check if as for xtr-client ip changed to update it in the database
This commit is contained in:
XiCoN-FJS- 2019-12-18 01:00:50 +01:00
parent e2283081cd
commit 8ea4311894
1 changed files with 78 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
xtrd.pl
View File

@ -9,7 +9,10 @@ use File::Fetch;
use PerlIO::gzip; use PerlIO::gzip;
use HTTP::Tiny; use HTTP::Tiny;
use HTTP::Request; use HTTP::Request;
use Data::Validate::IP qw(is_ipv4); use Data::Validate::Domain qw(is_domain);
use Data::Validate::IP qw(is_public_ipv4 is_public_ipv6);
use MaxMind::DB::Reader;
use Archive::Tar;
use Data::Dumper; use Data::Dumper;
@ -55,6 +58,8 @@ use Data::Dumper;
### VARS ############################################################################################################################################# ### VARS #############################################################################################################################################
my $dbfile = 'xtrd.db'; my $dbfile = 'xtrd.db';
my $ip2asn_csv_url = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz'; my $ip2asn_csv_url = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz';
my $max_dbfile = 'GeoLite2-ASN.mmdb';
my $max_db_source = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz';
my $server_protocol_version = 3; my $server_protocol_version = 3;
my $min_client_version = 4; my $min_client_version = 4;
set port => 12110; set port => 12110;
@ -63,8 +68,10 @@ set port => 12110;
get_maxmind_db($max_db_source, $max_dbfile);
### connect to database
my $maxmind_reader = MaxMind::DB::Reader->new( file => $max_dbfile );
@ -79,13 +86,7 @@ set port => 12110;
### connect to database ### connect to database
my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","",""); my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","","");
$dbh->do("PRAGMA cache_size = 800000");
### create ip2asn database & table if needed and fill with ip information
if(create_db_table($dbh,"ip2asn","ip2asn( ip_from INTEGER, ip_to INTEGER, ip_as INTEGER)","ip2asnIndex ON ip2asn(ip_from,ip_to);"))
{
get_ipas_data($dbh,$ip2asn_csv_url);
}
### create server database & table if needed ### create server database & table if needed
create_db_table($dbh,"server","server( server_ip TEXT, server_port INTEGER, server_as INTEGER, server_status INTEGER)",""); create_db_table($dbh,"server","server( server_ip TEXT, server_port INTEGER, server_as INTEGER, server_status INTEGER)","");
@ -137,14 +138,14 @@ put '/v3/server/add/:ip/:port/:state' => sub {
### get client ip ### get client ip
my $client_ip = request->address; my $client_ip = request->address;
my $request_ffa = request->forwarded_for_address; my $request_ffa = request->forwarded_for_address;
if(is_ipv4($request_ffa)) { $client_ip = $request_ffa; } if(is_public_ipv4($request_ffa)) { $client_ip = $request_ffa; }
### check input ### check input
if(!is_ipv4($ip) || $client_ip ne $ip) { status('bad_request'); return "invalid IP address"; } if(!is_public_ipv4($ip) || $client_ip ne $ip) { status('bad_request'); return "invalid IP address"; }
if(!($port > 0 and $port < 65536)) { status('bad_request'); return "invalid port"; } if(!($port > 0 and $port < 65536)) { status('bad_request'); return "invalid port"; }
if(!($state >= 0 and $state < 256)) { status('bad_request'); return "invalid status"; } if(!($state >= 0 and $state < 256)) { status('bad_request'); return "invalid status"; }
my $result = add_server($dbh,$ip,$port,$state); my $result = add_server($dbh,$maxmind_reader,$ip,$port,$state);
return "OK"; return "OK";
}; };
@ -166,19 +167,34 @@ sub check_server_status
sub add_server sub add_server
{ {
my ($dbh,$ip,$port,$status) = @_; my ($dbh,$maxmind_reader,$ip,$port,$status) = @_;
# prepare data
my $resolve = gethostbyname($ip);
my $a_record = inet_ntoa($resolve);
my $as = 0;
my $as_name = " ";
my $as_data = find_as($maxmind_reader, $a_record);
if(defined($as_data->{'autonomous_system_number'})) { $as = $as_data->{'autonomous_system_number'}; }
if(defined($as_data->{'autonomous_system_organization'})) { $as_name = $as_data->{'autonomous_system_organization'}; }
my $id = get_server_id($dbh,$ip,$port); my $id = get_server_id($dbh,$ip,$port);
if($id ne "0") if($id ne "0")
{ {
# get server as from database
my $db_as = get_server_as($dbh,$id);
# update server status # update server status
set_server_status($dbh,$id,$status); set_server_status($dbh,$id,$status);
if($as ne $db_as)
{
set_server_as($dbh,$id,$as);
}
} }
else else
{ {
# create
my $resolve = gethostbyname($ip);
my $a_record = inet_ntoa($resolve);
my $as = find_as($dbh,$a_record);
my $add_server_cmd = "INSERT INTO server VALUES ('".$ip."','".$port."','".$as."','".$status."');"; my $add_server_cmd = "INSERT INTO server VALUES ('".$ip."','".$port."','".$as."','".$status."');";
my $add_server_sth = $dbh->prepare($add_server_cmd); my $add_server_sth = $dbh->prepare($add_server_cmd);
$add_server_sth->execute(); $add_server_sth->execute();
@ -205,6 +221,26 @@ sub get_server_id
return $result->[0] || 0; return $result->[0] || 0;
} }
sub set_server_as
{
my ($dbh,$id,$as) = @_;
my $set_as_cmd = "UPDATE server set server_as = '".$as."' WHERE _rowid_ = '".$id."';";
my $set_as_sth = $dbh->prepare($set_as_cmd);
$set_as_sth->execute();
}
sub get_server_as
{
my ($dbh,$id) = @_;
my $get_as_cmd = "SELECT server_as FROM server WHERE _rowid_ = '".$id."';";
my $get_as_sth = $dbh->prepare($get_as_cmd);
$get_as_sth->execute();
my $result = $get_as_sth->fetch;
return $result->[0] || 0;
}
sub refresh_server_status sub refresh_server_status
{ {
my ($dbh,$status) = @_; my ($dbh,$status) = @_;
@ -257,65 +293,35 @@ sub get_server
return $result; return $result;
} }
sub get_ipas_data
{
my ($dbh,$ip2asn_csv_url) = @_;
my $ff = File::Fetch->new(uri => $ip2asn_csv_url);
my $temp_file = $ff->fetch( to => '/tmp' );
open(FILE,"<:gzip",$temp_file) or die "Can't open file: $!";
my @data = <FILE>;
close(FILE);
$dbh->do('begin');
my $max_commit = 10000;
my $inserted = 0;
my $array_size = @data;
my $last_p = -1;
print "loading data...\n";
foreach my $line (@data)
{
chomp($line);
if($line=~/^(\d+)\s+(\d+)\s+(\d+)/)
{
if($3 eq "0") { next; }
#my %temp_hash = ('from' => $1, 'to' => $2, 'as' => $3);
#push(@ipaslist,\%temp_hash);
my $insert_cmd .= "INSERT INTO ip2asn VALUES ('".$1."','".$2."','".$3."');";
my $insert_sth = $dbh->prepare($insert_cmd);
$inserted += $insert_sth->execute();
# print status for console users
my $p = int($inserted * 100 / $array_size);
if($p != $last_p) { print $p % 10 ? "" : "$p%\n"; }
$last_p = $p;
# only commit every $max_commit statements (it's faster)
unless ($inserted % $max_commit)
{
$dbh->do('commit');
$dbh->do('begin');
}
}
}
$dbh->do('commit');
unlink($temp_file);
print "data prepared for searches!\n";
}
sub find_as sub find_as
{ {
my ($dbh, $ip) = @_; my ($maxmind_reader, $ip) = @_;
my $ip_id = unpack("N", inet_aton($ip)); my $record = $maxmind_reader->record_for_address($ip);
return $record;
}
my $get_as_cmd = "SELECT ip_as FROM ip2asn WHERE ip_from < '".$ip_id."' AND ip_to > '".$ip_id."';"; sub get_maxmind_db
my $get_as_sth = $dbh->prepare($get_as_cmd); {
$get_as_sth->execute(); my ($max_db_source,$max_dbfile) = @_;
my $result = $get_as_sth->fetch;
return $result->[0] || 0; my $ff = File::Fetch->new(uri => $max_db_source);
my $temp_file = $ff->fetch( to => '/tmp' );
my $tar = Archive::Tar->new;
$tar->read($temp_file);
my @files = $tar->get_files();
foreach my $file (@files)
{
if($file->{'name'}=~/${max_dbfile}$/)
{
open(FILE,">",$max_dbfile) or die "Can't open file for writing: $!";
print FILE $file->{'data'};
close(FILE);
}
}
$tar->clear;
unlink($temp_file);
} }
sub create_db_table sub create_db_table