diff --git a/lib/MogileFS/Network.pm b/lib/MogileFS/Network.pm index ec94089..0b01348 100644 --- a/lib/MogileFS/Network.pm +++ b/lib/MogileFS/Network.pm @@ -25,6 +25,7 @@ use warnings; use Net::Netmask; use Net::Patricia; use MogileFS::Config; +use IO::Socket::INET; our $VERSION = "0.06"; @@ -34,6 +35,10 @@ my $trie = Net::Patricia->new(); # Net::Patricia object used for cache and looku my $next_reload = 0; # Epoch time at or after which the trie expires and must be regenerated. my $has_cached = MogileFS::Config->can('server_setting_cached'); +my $self_ip; # the external-facing IP of this host +my $local_trie = Net::Patricia->new(); +$local_trie->add_string("127.0.0.0/8", 1); + sub zone_for_ip { my $class = shift; my $ip = shift; @@ -42,7 +47,11 @@ sub zone_for_ip { check_cache(); - return $trie->match_string($ip); + my $rv = $trie->match_string($ip); + if (!$rv && $local_trie->match_string($ip)) { + $rv = $trie->match_string($self_ip); + } + return $rv; } sub check_cache { @@ -88,6 +97,16 @@ sub check_cache { $trie->add_string("$netmask", $zone); } + + # grab the our external address, no getifaddrs(3) support in Perl, yet. + my $sock = IO::Socket::INET->new(Proto => 'udp', + PeerAddr => '10.0.0.1', + PeerPort => 1); + if ($sock) { + $self_ip = $sock->sockhost; + $sock->close; + } + my $interval = get_setting("network_reload_interval") || DEFAULT_RELOAD_INTERVAL; $next_reload = time() + $interval; diff --git a/lib/MogileFS/ReplicationPolicy/MultipleNetworks.pm b/lib/MogileFS/ReplicationPolicy/MultipleNetworks.pm index b684153..dc754c0 100644 --- a/lib/MogileFS/ReplicationPolicy/MultipleNetworks.pm +++ b/lib/MogileFS/ReplicationPolicy/MultipleNetworks.pm @@ -136,6 +136,12 @@ sub replicate_to { @host_desp = weighted_list(map { [$_, 100 * $_->percent_free] } splice(@host_desp, 0, 20)); + # avoid ending up on potentially every single possible local network + # device due to a network split + if (@ideal == 0 && $already_on >= $min && @network_desp == 0) { + return TEMP_NO_ANSWER; + } + my @desp = (@network_desp, @host_desp); return MogileFS::ReplicationRequest->new( diff --git a/t/multiple-networks-replpol.t b/t/multiple-networks-replpol.t index c17f958..3279c52 100644 --- a/t/multiple-networks-replpol.t +++ b/t/multiple-networks-replpol.t @@ -11,7 +11,7 @@ use MogileFS::Util qw(error_code); use MogileFS::ReplicationPolicy::MultipleNetworks; use MogileFS::Test; -plan tests => 25; +plan tests => 27; # need just the one, so we only have to stuff the cache once my $polclass = "MogileFS::ReplicationPolicy::MultipleNetworks"; @@ -128,6 +128,18 @@ is(rr("min=2 h1[d1=_ d2=X]$ad1 h2[d3=X d4=X]$ad2"), is(rr("min=2 h1[d1=X d2=X]$ad1 h2[d3=X d4=X]$ad2 h3[d5=_ d6=_]$ad3 h4[d7=_ d8=_]$ad4"), "ideal(5,6,7,8)", "more than min hosts, but all on one network"); +is(rr("min=4 h1[d1=X d5=_]$ad1 h2[d2=X d6=_]$ad2 " . + "h3=alive:unreachable[d3=_]$ad3 " . + "h4=alive:unreachable[d4=_]$ad4"), + 'desperate(5,6)', + 'copy locally if only one network is reachable'); + +is(rr("min=4 h1[d1=X d5=X d7=_]$ad1 h2[d2=X d6=X d8=_]$ad2 " . + "h3=alive:unreachable[d3=_]$ad3 " . + "h4=alive:unreachable[d4=_]$ad4"), + 'temp_fail', + 'avoid overreplicating if we meet min during network failure'); + # mess with netmasks $pol->stuff_cache('146.101.246.2' , Net::Netmask->new('146.101.246.0/24')); $pol->stuff_cache('146.101.142.130', Net::Netmask->new('146.101.142.0/24')); @@ -160,6 +172,7 @@ sub rr { while ($state =~ s/\bh(\d+)(?:=(.+?))?\[(.+?)\](#\d+\.\d+\.\d+\.\d+\.?#)?//) { my ($n, $opts, $devstr, $ip) = ($1, $2, $3, $4); $opts ||= ""; + my $obstate = "reachable"; die "dup host $n" if $hosts->{$n}; # print "1 2 3 4 : <<$1>> <<$2>> <<$3>> <<$4>>\n"; @@ -171,8 +184,12 @@ sub rr { $extras{hostip} = $ip; } + if ($opts =~ s/:(\w+)\z//) { + $obstate = $1; + } + my $h = $hosts->{$n} = $hfac->set({ hostid => $n, - status => ($opts || "alive"), observed_state => "reachable", + status => ($opts || "alive"), observed_state => $obstate, hostname => $n, %extras }); diff --git a/t/network_zones.t b/t/network_zones.t index ae4e6f0..4692764 100644 --- a/t/network_zones.t +++ b/t/network_zones.t @@ -2,10 +2,11 @@ use strict; use warnings; -use Test::More tests => 5; +use Test::More tests => 9; use FindBin qw($Bin); use MogileFS::Network; +use IO::Socket::INET; MogileFS::Network->test_config( zone_one => '127.0.0.0/16', @@ -21,6 +22,30 @@ is(lookup('10.1.0.1'), 'three', "Inner netblock match"); is(lookup('172.16.0.1'), 'two', "Zone with multiple netblocks"); is(lookup('192.168.0.1'), undef, "Unknown zone"); + +my $sock = IO::Socket::INET->new(Proto => 'udp', + PeerAddr => '10.0.0.1', + PeerPort => 1); +my $self_ip = $sock->sockhost; +$sock->close; +my $self_net = $self_ip; +$self_net =~ s!\.\d+\z!.0/24!; + +MogileFS::Network->test_config( + zone_self => $self_net, + zone_one => '10.1.0.0/16', + zone_two => '10.2.0.0/16', + zone_big => '10.0.0.0/8', + network_zones => 'one,two,big,self', +); + +is(lookup('127.0.0.1'), 'self', "local connection goes to self"); + +# ensure existing configs work +is(lookup('10.1.1.1'), 'one', "one match"); +is(lookup('10.2.1.1'), 'two', "two match"); +is(lookup('10.9.1.1'), 'big', "big match"); + sub lookup { return MogileFS::Network->zone_for_ip(@_); }