Skip to content

Commit 4c9d956

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request #114 from rancher-sandbox/all-interfaces
Allow forwarding of specified port ranges to all interfaces
2 parents 5e4688e + 24f9bdb commit 4c9d956

File tree

11 files changed

+551
-48
lines changed

11 files changed

+551
-48
lines changed

‎.cirrus.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ task:
2929
- cat /proc/cpuinfo
3030
install_deps_script:
3131
- apt-get update
32-
- apt-get install -y --no-install-recommends ca-certificates curl git golang openssh-client make ovmf sudo qemu-system-x86 qemu-utils
32+
- apt-get install -y --no-install-recommends ca-certificates curl git golang openssh-client make netcat ovmf sudo qemu-system-x86 qemu-utils
3333
go_cache:
3434
fingerprint_script: uname -s ; cat go.sum
3535
folder: $GOPATH/pkg/mod

‎hack/test-example.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/usr/bin/env bash
22
set -eu -o pipefail
33

4+
scriptdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5+
46
function INFO() {
57
echo "TEST| [INFO] $*"
68
}
@@ -35,6 +37,7 @@ declare -A CHECKS=(
3537
["mount-home"]="1"
3638
["containerd-user"]="1"
3739
["restart"]="1"
40+
["port-forwards"]="1"
3841
)
3942

4043
case "$NAME" in
@@ -60,6 +63,18 @@ if limactl ls -q | grep -q "$NAME"; then
6063
exit 1
6164
fi
6265

66+
if [[ -n ${CHECKS["port-forwards"]} ]]; then
67+
tmpconfig="$HOME/lime-config-tmp"
68+
mkdir -p "${tmpconfig}"
69+
trap 'rm -rf $tmpconfig' EXIT
70+
tmpfile="${tmpconfig}/${NAME}.yaml"
71+
cp "$FILE" "${tmpfile}"
72+
FILE="${tmpfile}"
73+
INFO "Setup port forwarding rules for testing in \"${FILE}\""
74+
"${scriptdir}/test-port-forwarding.pl" "${FILE}"
75+
limactl validate "$FILE"
76+
fi
77+
6378
function diagnose() {
6479
NAME="$1"
6580
set -x +e
@@ -142,13 +157,33 @@ if [[ -n ${CHECKS["containerd-user"]} ]]; then
142157
set +x
143158
fi
144159

160+
if [[ -n ${CHECKS["port-forwards"]} ]]; then
161+
INFO "Testing port forwarding rules using netcat"
162+
set -x
163+
if [ "${NAME}" = "archlinux" ]; then
164+
limactl shell "$NAME" sudo pacman -Syu --noconfirm openbsd-netcat
165+
fi
166+
if [ "${NAME}" = "debian" ]; then
167+
limactl shell "$NAME" sudo apt-get install -y netcat
168+
fi
169+
if [ "${NAME}" = "fedora" ]; then
170+
limactl shell "$NAME" sudo dnf install -y nc
171+
fi
172+
if [ "${NAME}" = "opensuse" ]; then
173+
limactl shell "$NAME" sudo zypper in -y netcat-openbsd
174+
fi
175+
"${scriptdir}/test-port-forwarding.pl" "${NAME}"
176+
set +x
177+
fi
178+
145179
if [[ -n ${CHECKS["restart"]} ]]; then
146180
INFO "Create file in the guest home directory and verify that it still exists after a restart"
147181
# shellcheck disable=SC2016
148182
limactl shell "$NAME" sh -c 'touch $HOME/sweet-home'
149183

150184
INFO "Stopping \"$NAME\""
151185
limactl stop "$NAME"
186+
sleep 3
152187

153188
INFO "Restarting \"$NAME\""
154189
limactl start "$NAME"
@@ -162,6 +197,7 @@ fi
162197

163198
INFO "Stopping \"$NAME\""
164199
limactl stop "$NAME"
200+
sleep 3
165201

166202
INFO "Deleting \"$NAME\""
167203
limactl delete "$NAME"

‎hack/test-port-forwarding.pl

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,296 @@
1+
#!/usr/bin/env perl
2+
3+
# This script tests the port forwarding settings of lima. It has to be run
4+
# twice: once to update the instance yaml file with the port forwarding
5+
# rules (before the instance is started). And once when the instance is
6+
# running to perform the tests:
7+
#
8+
# ./hack/test-port-forwarding.pl examples/default.yaml
9+
# limactl start --tty=false examples/default.yaml
10+
# git restore pkg/limayaml/default.yaml
11+
# ./hack/test-port-forwarding.pl default
12+
#
13+
# TODO: support for ipv6 host addresses
14+
15+
use strict;
16+
use warnings;
17+
18+
use Config qw(%Config);
19+
use IO::Handle qw();
20+
use Socket qw(inet_ntoa);
21+
use Sys::Hostname qw(hostname);
22+
23+
my $instance = shift;
24+
25+
my $ipv4 = inet_ntoa(scalar gethostbyname(hostname())) or die;
26+
my $ipv6 = ""; # todo
27+
28+
# If $instance is a filename, add our portForwards to it to enable testing
29+
if (-f $instance) {
30+
open(my $fh, "+< $instance") or die "Can't open $instance for read/write: $!";
31+
my @yaml;
32+
while (<$fh>) {
33+
# Remove existing "portForwards:" section from the config file
34+
my $seq = /^portForwards:/ ... /^[a-z]/;
35+
next if $seq && $seq !~ /E0$/;
36+
push @yaml, $_;
37+
}
38+
seek($fh, 0, 0);
39+
truncate($fh, 0);
40+
print $fh $_ for @yaml;
41+
while (<DATA>) {
42+
s/ipv4/$ipv4/g;
43+
s/ipv6/$ipv6/g;
44+
print $fh $_;
45+
}
46+
exit;
47+
}
48+
49+
# Otherwise $instance must be the name of an already running instance that has been
50+
# configured with our portForwards settings.
51+
52+
# Get sshLocalPort for lima instance
53+
my $sshLocalPort;
54+
open(my $ls, "limactl ls --json |") or die;
55+
while (<$ls>) {
56+
next unless /"name":"$instance"/;
57+
($sshLocalPort) = /"sshLocalPort":(\d+)/ or die;
58+
last;
59+
}
60+
die "Cannot determine sshLocalPort" unless $sshLocalPort;
61+
62+
# Extract forwarding tests from the "portForwards" section
63+
my @test;
64+
while (<DATA>) {
65+
chomp;
66+
s/^\s+#\s*//;
67+
next unless /^(forward|ignore)/;
68+
if (/ipv6/ && !$ipv6) {
69+
printf "🚧 Not yet: # $_\n";
70+
next;
71+
}
72+
s/sshLocalPort/$sshLocalPort/g;
73+
s/ipv4/$ipv4/g;
74+
s/ipv6/$ipv6/g;
75+
# forward: 127.0.0.1 899 → 127.0.0.1 799
76+
# ignore: 127.0.0.2 8888
77+
/^(forward|ignore):\s+([0-9.:]+)\s+(\d+)(?:\s+→)?(?:\s+([0-9.:]+)(?:\s+(\d+))?)?/;
78+
die "Cannot parse test '$_'" unless $1;
79+
my %test; @test{qw(mode guest_ip guest_port host_ip host_port)} = ($1, $2, $3, $4, $5);
80+
if ($test{mode} eq "forward" && $test{host_port} < 1024 && $Config{osname} ne "darwin") {
81+
printf "🚧 Not supported on $Config{osname}: # $_\n";
82+
next;
83+
}
84+
$test{host_ip} ||= "127.0.0.1";
85+
$test{host_port} ||= $test{guest_port};
86+
87+
my $remote = JoinHostPort($test{guest_ip},$test{guest_port});
88+
my $local = JoinHostPort($test{host_ip},$test{host_port});
89+
if ($test{mode} eq "ignore") {
90+
$test{log_msg} = "Not forwarding TCP $remote";
91+
}
92+
else {
93+
$test{log_msg} = "Forwarding TCP from $remote to $local";
94+
}
95+
push @test, \%test;
96+
}
97+
98+
open(my $lima, "| limactl shell --workdir / $instance")
99+
or die "Can't run lima shell on $instance: $!";
100+
$lima->autoflush;
101+
102+
print $lima <<'EOF';
103+
set -e
104+
cd $HOME
105+
sudo pkill -x nc || true
106+
rm -f nc.*
107+
EOF
108+
109+
# Give the hostagent some time to remove any port forwards from a previous (crashed?) test run
110+
sleep 5;
111+
112+
# Record current log size, so we can skip prior output
113+
$ENV{LIMA_HOME} ||= "$ENV{HOME}/.lima";
114+
my $ha_log = "$ENV{LIMA_HOME}/$instance/ha.stderr.log";
115+
my $ha_log_size = -s $ha_log or die;
116+
117+
# Setup a netcat listener on the guest for each test
118+
foreach my $id (0..@test-1) {
119+
my $test = $test[$id];
120+
my $nc = "nc -l $test->{guest_ip} $test->{guest_port}";
121+
if ($instance eq "alpine") {
122+
$nc = "nc -l -s $test->{guest_ip} -p $test->{guest_port}";
123+
}
124+
125+
my $sudo = $test->{guest_port} < 1024 ? "sudo " : "";
126+
print $lima "${sudo}${nc} >nc.${id} 2>/dev/null &\n";
127+
}
128+
129+
# Make sure the guest- and hostagents had enough time to set up the forwards
130+
sleep 5;
131+
132+
# Try to reach each listener from the host
133+
foreach my $test (@test) {
134+
next if $test->{host_port} == $sshLocalPort;
135+
my $nc = "nc -w 1 $test->{host_ip} $test->{host_port}";
136+
open(my $netcat, "| $nc") or die "Can't run '$nc': $!";
137+
print $netcat "$test->{log_msg}\n";
138+
# Don't check for errors on close; macOS nc seems to return non-zero exit code even on success
139+
close($netcat);
140+
}
141+
142+
# Extract forwarding log messages from hostagent log
143+
open(my $log, "< $ha_log") or die "Can't read $ha_log: $!";
144+
seek($log, $ha_log_size, 0) or die "Can't seek $ha_log to $ha_log_size: $!";
145+
my %seen;
146+
while (<$log>) {
147+
$seen{$1}++ if /(Forwarding TCP from .*? to (\d.*?|\[.*?\]):\d+)/;
148+
$seen{$1}++ if /(Not forwarding TCP .*?:\d+)/;
149+
}
150+
close $log or die;
151+
152+
my $rc = 0;
153+
my %expected;
154+
foreach my $id (0..@test-1) {
155+
my $test = $test[$id];
156+
my $err = "";
157+
$expected{$test->{log_msg}}++;
158+
unless ($seen{$test->{log_msg}}) {
159+
$err .= "\n Message missing from ha.stderr.log";
160+
}
161+
my $log = qx(limactl shell --workdir / $instance sh -c "cd; cat nc.$id");
162+
chomp $log;
163+
if ($test->{mode} eq "forward" && $test->{log_msg} ne $log) {
164+
$err .= "\n Guest received: '$log'";
165+
}
166+
if ($test->{mode} eq "ignore" && $log) {
167+
$err .= "\n Guest received: '$log' (instead of nothing)";
168+
}
169+
printf "%s %s%s\n", ($err ? "❌" : "✅"), $test->{log_msg}, $err;
170+
$rc = 1 if $err;
171+
}
172+
173+
foreach (keys %seen) {
174+
next if $expected{$_};
175+
# Should this be an error? Really should only happen if something else failed as well.
176+
print "😕 Unexpected log message: $_\n";
177+
}
178+
179+
# Cleanup remaining netcat instances (and port forwards)
180+
print $lima "sudo pkill -x nc";
181+
182+
exit $rc;
183+
184+
sub JoinHostPort {
185+
my($host,$port) = @_;
186+
$host = "[$host]" if $host =~ /:/;
187+
return "$host:$port";
188+
}
189+
190+
# This YAML section includes port forwarding `rules` for the guest- and hostagents,
191+
# with interleaved `tests` (in comments) that are executed by this script. The strings
192+
# "ipv4" and "ipv6" will be replaced by the actual host ipv4 and ipv6 addresses.
193+
__DATA__
194+
portForwards:
195+
# We can't test that port 22 will be blocked because the guestagent has
196+
# been ignoring it since startup, so the log message is in the part of
197+
# the log we skipped.
198+
# skip: 127.0.0.1 22 → 127.0.0.1 2222
199+
# ignore: 127.0.0.1 sshLocalPort
200+
201+
- guestIP: 127.0.0.2
202+
guestPortRange: [3000, 3009]
203+
hostPortRange: [2000, 2009]
204+
ignore: true
205+
206+
- guestIP: 0.0.0.0
207+
guestPortRange: [3010, 3019]
208+
hostPortRange: [2010, 2019]
209+
ignore: true
210+
211+
- guestIP: 0.0.0.0
212+
guestPortRange: [3000, 3029]
213+
hostPortRange: [2000, 2029]
214+
215+
# The following rule is completely shadowed by the previous one and has no effect
216+
- guestIP: 0.0.0.0
217+
guestPortRange: [3020, 3029]
218+
hostPortRange: [2020, 2029]
219+
ignore: true
220+
221+
# ignore: 127.0.0.2 3000
222+
# forward: 127.0.0.3 3001 → 127.0.0.1 2001
223+
224+
# Blocking 127.0.0.2 cannot block forwarding from 0.0.0.0
225+
# forward: 0.0.0.0 3002 → 127.0.0.1 2002
226+
227+
# Blocking 0.0.0.0 will block forwarding from any interface
228+
# ignore: 0.0.0.0 3010
229+
# ignore: 127.0.0.1 3011
230+
231+
# Forwarding from 0.0.0.0 works for any interface (including IPv6)
232+
# The "ignore" rule above has no effect because the previous rule already matched.
233+
# forward: 127.0.0.2 3020 → 127.0.0.1 2020
234+
# forward: 127.0.0.1 3021 → 127.0.0.1 2021
235+
# forward: 0.0.0.0 3022 → 127.0.0.1 2022
236+
# forward: :: 3023 → 127.0.0.1 2023
237+
# forward: ::1 3024 → 127.0.0.1 2024
238+
239+
- guestPortRange: [3030, 3039]
240+
hostPortRange: [2030, 2039]
241+
hostIP: ipv4
242+
243+
# forward: 127.0.0.1 3030 → ipv4 2030
244+
# forward: 0.0.0.0 3031 → ipv4 2031
245+
# forward: :: 3032 → ipv4 2032
246+
# forward: ::1 3033 → ipv4 2033
247+
248+
# Forwarding privileged ports to 127.0.0.1 does not work yet
249+
- guestPortRange: [300, 309]
250+
251+
# Hostagent will try to setup forward, but ssh will fail to bind to port
252+
# skip: 127.0.0.1 300 → 127.0.0.1 300
253+
254+
- guestPortRange: [310, 319]
255+
hostIP: 0.0.0.0
256+
257+
# forward: 127.0.0.1 310 → 0.0.0.0 310
258+
259+
# Things we can't test:
260+
# - Accessing a forward from a different interface (e.g. connect to ipv4 to connect to 0.0.0.0)
261+
# - failed forward to privileged port
262+
263+
264+
- guestIP: "192.168.5.15"
265+
guestPortRange: [4000, 4009]
266+
hostIP: "ipv4"
267+
268+
# forward: 192.168.5.15 4000 → ipv4 4000
269+
270+
- guestIP: "::1"
271+
guestPortRange: [4010, 4019]
272+
hostIP: "::"
273+
274+
# forward: ::1 4010 → :: 4010
275+
276+
- guestIP: "::"
277+
guestPortRange: [4020, 4029]
278+
hostIP: "ipv4"
279+
280+
# forward: 127.0.0.1 4020 → ipv4 4020
281+
# forward: 127.0.0.2 4021 → ipv4 4021
282+
# forward: 192.168.5.15 4022 → ipv4 4022
283+
# forward: 0.0.0.0 4023 → ipv4 4023
284+
# forward: :: 4024 → ipv4 4024
285+
# forward: ::1 4025 → ipv4 4025
286+
287+
- guestIP: "0.0.0.0"
288+
guestPortRange: [4030, 4039]
289+
hostIP: "ipv4"
290+
291+
# forward: 127.0.0.1 4030 → ipv4 4030
292+
# forward: 127.0.0.2 4031 → ipv4 4031
293+
# forward: 192.168.5.15 4032 → ipv4 4032
294+
# forward: 0.0.0.0 4033 → ipv4 4033
295+
# forward: :: 4034 → ipv4 4034
296+
# forward: ::1 4035 → ipv4 4035

‎pkg/guestagent/api/api.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ type ErrorJSON struct {
1111
Message string `json:"message"`
1212
}
1313

14+
var (
15+
IPv4loopback1 = net.IPv4(127,0,0,1)
16+
)
17+
1418
type IPPort struct {
1519
IP net.IP `json:"ip"`
1620
Port int `json:"port"`

0 commit comments

Comments
 (0)