Merge branch 'change/switch_to_maxmind' of xicon/xtrd into master

xtrd.pl

@@ -9,7 +9,10 @@ use File::Fetch;
 use PerlIO::gzip;
 use HTTP::Tiny;
 use HTTP::Request;
-use Data::Validate::IP qw(is_ipv4);
+use Data::Validate::Domain qw(is_domain);
+use Data::Validate::IP qw(is_public_ipv4 is_public_ipv6);
+use MaxMind::DB::Reader;
+use Archive::Tar;
 use Data::Dumper;
 
 
@@ -55,6 +58,8 @@ use Data::Dumper;
 ### VARS #############################################################################################################################################
 my $dbfile                      = 'xtrd.db';
 my $ip2asn_csv_url              = 'http://iptoasn.com/data/ip2asn-v4-u32.tsv.gz';
+my $max_dbfile                  = 'GeoLite2-ASN.mmdb';
+my $max_db_source               = 'http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz';
 my $server_protocol_version     = 3;
 my $min_client_version          = 4;
 set port                        => 12110;
@@ -63,8 +68,10 @@ set port                        => 12110;
 
 
 
+get_maxmind_db($max_db_source, $max_dbfile);
 
-
+### connect to database
+my $maxmind_reader = MaxMind::DB::Reader->new( file => $max_dbfile );
 
 
 
@@ -79,13 +86,7 @@ set port                        => 12110;
 
 ### connect to database
 my $dbh = DBI->connect("dbi:SQLite:dbname=$dbfile","","");
-$dbh->do("PRAGMA cache_size = 800000");
 
-### create ip2asn database & table if needed and fill with ip information
-if(create_db_table($dbh,"ip2asn","ip2asn( ip_from INTEGER, ip_to INTEGER, ip_as INTEGER)","ip2asnIndex ON ip2asn(ip_from,ip_to);"))
-{
-	get_ipas_data($dbh,$ip2asn_csv_url);
-}
 
 ### create server database & table if needed
 create_db_table($dbh,"server","server( server_ip TEXT, server_port INTEGER, server_as INTEGER, server_status INTEGER)","");
@@ -137,14 +138,14 @@ put '/v3/server/add/:ip/:port/:state' => sub {
 	### get client ip
 	my $client_ip = request->address;
 	my $request_ffa = request->forwarded_for_address;
-	if(is_ipv4($request_ffa)) { $client_ip = $request_ffa; }
+	if(is_public_ipv4($request_ffa)) { $client_ip = $request_ffa; }
 	
 	### check input
-	if(!is_ipv4($ip) || $client_ip ne $ip) { status('bad_request'); return "invalid IP address"; }
+	if(!is_public_ipv4($ip) || $client_ip ne $ip) { status('bad_request'); return "invalid IP address"; }
 	if(!($port > 0 and $port < 65536)) { status('bad_request'); return "invalid port"; }
 	if(!($state >= 0 and $state < 256)) { status('bad_request'); return "invalid status"; }
 
-	my $result = add_server($dbh,$ip,$port,$state);
+	my $result = add_server($dbh,$maxmind_reader,$ip,$port,$state);
 	return "OK";
 };
 
@@ -166,19 +167,34 @@ sub check_server_status
 
 sub add_server
 {
-	my ($dbh,$ip,$port,$status) = @_;
+	my ($dbh,$maxmind_reader,$ip,$port,$status) = @_;
+	
+
+	# prepare data
+	my $resolve = gethostbyname($ip);
+	my $a_record = inet_ntoa($resolve);
+	my $as = 0;
+	my $as_name = " ";
+	my $as_data = find_as($maxmind_reader, $a_record);
+	if(defined($as_data->{'autonomous_system_number'})) { $as = $as_data->{'autonomous_system_number'}; }
+	if(defined($as_data->{'autonomous_system_organization'})) { $as_name = $as_data->{'autonomous_system_organization'}; }
+	
 	my $id = get_server_id($dbh,$ip,$port);
 	if($id ne "0")
 	{
+		# get server as from database
+		my $db_as = get_server_as($dbh,$id);
+
 		# update server status
 		set_server_status($dbh,$id,$status);
+
+		if($as ne $db_as)
+		{
+			set_server_as($dbh,$id,$as);
+		}
 	}
 	else
 	{
-		# create
-		my $resolve = gethostbyname($ip);
-		my $a_record = inet_ntoa($resolve);
-		my $as = find_as($dbh,$a_record);
 		my $add_server_cmd = "INSERT INTO server VALUES ('".$ip."','".$port."','".$as."','".$status."');";
 		my $add_server_sth = $dbh->prepare($add_server_cmd);
 		$add_server_sth->execute();
@@ -205,6 +221,26 @@ sub get_server_id
 	return $result->[0] || 0;
 }
 
+sub set_server_as
+{
+	my ($dbh,$id,$as) = @_;
+	
+	my $set_as_cmd = "UPDATE server set server_as = '".$as."' WHERE _rowid_ = '".$id."';";
+	my $set_as_sth = $dbh->prepare($set_as_cmd);
+	$set_as_sth->execute();
+}
+
+sub get_server_as
+{
+	my ($dbh,$id) = @_;
+
+	my $get_as_cmd = "SELECT server_as FROM server WHERE _rowid_ = '".$id."';";
+	my $get_as_sth = $dbh->prepare($get_as_cmd);
+	$get_as_sth->execute();
+	my $result = $get_as_sth->fetch;
+	return $result->[0] || 0;
+}
+
 sub refresh_server_status
 {
 	my ($dbh,$status) = @_;
@@ -257,65 +293,35 @@ sub get_server
 	return $result;
 }
 
-sub get_ipas_data
-{
-	my ($dbh,$ip2asn_csv_url) = @_;
-
-	my $ff = File::Fetch->new(uri => $ip2asn_csv_url);
-	my $temp_file = $ff->fetch( to => '/tmp' );
-
-	open(FILE,"<:gzip",$temp_file) or die "Can't open file: $!";
-	my @data = <FILE>;
-	close(FILE);
-
-	$dbh->do('begin');
-
-	my $max_commit  = 10000;
-	my $inserted    = 0;
-	my $array_size  = @data;
-	my $last_p      = -1;
-	print "loading data...\n";
-	foreach my $line (@data)
-	{
-		chomp($line);
-		if($line=~/^(\d+)\s+(\d+)\s+(\d+)/)
-		{
-			if($3 eq "0") { next; }
-			#my %temp_hash = ('from' => $1, 'to' => $2, 'as' => $3);
-			#push(@ipaslist,\%temp_hash);
-			my $insert_cmd .= "INSERT INTO ip2asn VALUES ('".$1."','".$2."','".$3."');";
-			my $insert_sth = $dbh->prepare($insert_cmd);
-			$inserted += $insert_sth->execute();
-			
-			# print status for console users
-			my $p = int($inserted * 100 / $array_size);
-			if($p != $last_p) { print $p % 10 ? "" : "$p%\n"; }
-			$last_p = $p;
-
-			# only commit every $max_commit statements (it's faster)	
-			unless ($inserted % $max_commit)
-			{
-				$dbh->do('commit');
-				$dbh->do('begin');
-			}
-		}
-	}
-
-	$dbh->do('commit');
-	unlink($temp_file);
-	print "data prepared for searches!\n";
-}
-
 sub find_as
 {
-	my ($dbh, $ip) = @_;
+	my ($maxmind_reader, $ip) = @_;
-	my $ip_id = unpack("N", inet_aton($ip));
+	my $record = $maxmind_reader->record_for_address($ip);
+	return $record;
+}
 
-	my $get_as_cmd = "SELECT ip_as FROM ip2asn WHERE ip_from < '".$ip_id."' AND ip_to > '".$ip_id."';";
+sub get_maxmind_db
-	my $get_as_sth = $dbh->prepare($get_as_cmd);
+{
-	$get_as_sth->execute();
+	my ($max_db_source,$max_dbfile) = @_;
-	my $result = $get_as_sth->fetch;
+
-	return $result->[0] || 0;
+	my $ff = File::Fetch->new(uri => $max_db_source);
+	my $temp_file = $ff->fetch( to => '/tmp' );
+
+	my $tar = Archive::Tar->new;
+	$tar->read($temp_file);
+	
+	my @files = $tar->get_files();
+	foreach my $file (@files)
+	{
+		if($file->{'name'}=~/${max_dbfile}$/)
+		{
+			open(FILE,">",$max_dbfile) or die "Can't open file for writing: $!";
+			print FILE $file->{'data'};
+			close(FILE);
+		}
+	}
+	$tar->clear;
+	unlink($temp_file);
 }
 
 sub create_db_table