From 8ea4311894cebe8a25d8a5768831437a4702ff48 Mon Sep 17 00:00:00 2001 From: XiCoN-FJS- Date: Wed, 18 Dec 2019 01:00:50 +0100 Subject: [PATCH] * switched from ip2asn to maxmind-db * added check if as for xtr-client ip changed to update it in the database --- xtrd.pl | 150 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 78 insertions(+), 72 deletions(-) diff --git a/xtrd.pl b/xtrd.pl index a851e51..b2f995a 100644 --- a/xtrd.pl +++ b/xtrd.pl @@ -9,7 +9,10 @@ use File::Fetch; use PerlIO::gzip; use HTTP::Tiny; use HTTP::Request; -use Data::Validate::IP qw(is_ipv4); +use Data::Validate::Domain qw(is_domain); +use Data::Validate::IP qw(is_public_ipv4 is_public_ipv6); +use MaxMind::DB::Reader; +use Archive::Tar; use Data::Dumper; @@ -55,6 +58,8 @@ use Data::Dumper; ### VARS ############################################################################################################################################# my $dbfile = 'xtrd.db'; my $ip2asn_csv_url = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz'; +my $max_dbfile = 'GeoLite2-ASN.mmdb'; +my $max_db_source = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz'; my $server_protocol_version = 3; my $min_client_version = 4; set port => 12110; @@ -63,8 +68,10 @@ set port => 12110; +get_maxmind_db($max_db_source, $max_dbfile); - +### connect to database +my $maxmind_reader = MaxMind::DB::Reader->new( file => $max_dbfile ); @@ -79,13 +86,7 @@ set port => 12110; ### connect to database my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","",""); -$dbh->do("PRAGMA cache_size = 800000"); -### create ip2asn database & table if needed and fill with ip information -if(create_db_table($dbh,"ip2asn","ip2asn( ip_from INTEGER, ip_to INTEGER, ip_as INTEGER)","ip2asnIndex ON ip2asn(ip_from,ip_to);")) -{ - get_ipas_data($dbh,$ip2asn_csv_url); -} ### create server database & table if needed create_db_table($dbh,"server","server( server_ip TEXT, server_port INTEGER, server_as INTEGER, server_status INTEGER)",""); @@ -137,14 +138,14 @@ put '/v3/server/add/:ip/:port/:state' => sub { ### get client ip my $client_ip = request->address; my $request_ffa = request->forwarded_for_address; - if(is_ipv4($request_ffa)) { $client_ip = $request_ffa; } + if(is_public_ipv4($request_ffa)) { $client_ip = $request_ffa; } ### check input - if(!is_ipv4($ip) || $client_ip ne $ip) { status('bad_request'); return "invalid IP address"; } + if(!is_public_ipv4($ip) || $client_ip ne $ip) { status('bad_request'); return "invalid IP address"; } if(!($port > 0 and $port < 65536)) { status('bad_request'); return "invalid port"; } if(!($state >= 0 and $state < 256)) { status('bad_request'); return "invalid status"; } - my $result = add_server($dbh,$ip,$port,$state); + my $result = add_server($dbh,$maxmind_reader,$ip,$port,$state); return "OK"; }; @@ -166,19 +167,34 @@ sub check_server_status sub add_server { - my ($dbh,$ip,$port,$status) = @_; + my ($dbh,$maxmind_reader,$ip,$port,$status) = @_; + + + # prepare data + my $resolve = gethostbyname($ip); + my $a_record = inet_ntoa($resolve); + my $as = 0; + my $as_name = " "; + my $as_data = find_as($maxmind_reader, $a_record); + if(defined($as_data->{'autonomous_system_number'})) { $as = $as_data->{'autonomous_system_number'}; } + if(defined($as_data->{'autonomous_system_organization'})) { $as_name = $as_data->{'autonomous_system_organization'}; } + my $id = get_server_id($dbh,$ip,$port); if($id ne "0") { + # get server as from database + my $db_as = get_server_as($dbh,$id); + # update server status set_server_status($dbh,$id,$status); + + if($as ne $db_as) + { + set_server_as($dbh,$id,$as); + } } else { - # create - my $resolve = gethostbyname($ip); - my $a_record = inet_ntoa($resolve); - my $as = find_as($dbh,$a_record); my $add_server_cmd = "INSERT INTO server VALUES ('".$ip."','".$port."','".$as."','".$status."');"; my $add_server_sth = $dbh->prepare($add_server_cmd); $add_server_sth->execute(); @@ -205,6 +221,26 @@ sub get_server_id return $result->[0] || 0; } +sub set_server_as +{ + my ($dbh,$id,$as) = @_; + + my $set_as_cmd = "UPDATE server set server_as = '".$as."' WHERE _rowid_ = '".$id."';"; + my $set_as_sth = $dbh->prepare($set_as_cmd); + $set_as_sth->execute(); +} + +sub get_server_as +{ + my ($dbh,$id) = @_; + + my $get_as_cmd = "SELECT server_as FROM server WHERE _rowid_ = '".$id."';"; + my $get_as_sth = $dbh->prepare($get_as_cmd); + $get_as_sth->execute(); + my $result = $get_as_sth->fetch; + return $result->[0] || 0; +} + sub refresh_server_status { my ($dbh,$status) = @_; @@ -257,65 +293,35 @@ sub get_server return $result; } -sub get_ipas_data -{ - my ($dbh,$ip2asn_csv_url) = @_; - - my $ff = File::Fetch->new(uri => $ip2asn_csv_url); - my $temp_file = $ff->fetch( to => '/tmp' ); - - open(FILE,"<:gzip",$temp_file) or die "Can't open file: $!"; - my @data = ; - close(FILE); - - $dbh->do('begin'); - - my $max_commit = 10000; - my $inserted = 0; - my $array_size = @data; - my $last_p = -1; - print "loading data...\n"; - foreach my $line (@data) - { - chomp($line); - if($line=~/^(\d+)\s+(\d+)\s+(\d+)/) - { - if($3 eq "0") { next; } - #my %temp_hash = ('from' => $1, 'to' => $2, 'as' => $3); - #push(@ipaslist,\%temp_hash); - my $insert_cmd .= "INSERT INTO ip2asn VALUES ('".$1."','".$2."','".$3."');"; - my $insert_sth = $dbh->prepare($insert_cmd); - $inserted += $insert_sth->execute(); - - # print status for console users - my $p = int($inserted * 100 / $array_size); - if($p != $last_p) { print $p % 10 ? "" : "$p%\n"; } - $last_p = $p; - - # only commit every $max_commit statements (it's faster) - unless ($inserted % $max_commit) - { - $dbh->do('commit'); - $dbh->do('begin'); - } - } - } - - $dbh->do('commit'); - unlink($temp_file); - print "data prepared for searches!\n"; -} - sub find_as { - my ($dbh, $ip) = @_; - my $ip_id = unpack("N", inet_aton($ip)); + my ($maxmind_reader, $ip) = @_; + my $record = $maxmind_reader->record_for_address($ip); + return $record; +} - my $get_as_cmd = "SELECT ip_as FROM ip2asn WHERE ip_from < '".$ip_id."' AND ip_to > '".$ip_id."';"; - my $get_as_sth = $dbh->prepare($get_as_cmd); - $get_as_sth->execute(); - my $result = $get_as_sth->fetch; - return $result->[0] || 0; +sub get_maxmind_db +{ + my ($max_db_source,$max_dbfile) = @_; + + my $ff = File::Fetch->new(uri => $max_db_source); + my $temp_file = $ff->fetch( to => '/tmp' ); + + my $tar = Archive::Tar->new; + $tar->read($temp_file); + + my @files = $tar->get_files(); + foreach my $file (@files) + { + if($file->{'name'}=~/${max_dbfile}$/) + { + open(FILE,">",$max_dbfile) or die "Can't open file for writing: $!"; + print FILE $file->{'data'}; + close(FILE); + } + } + $tar->clear; + unlink($temp_file); } sub create_db_table