Skip to content

Commit ad45119

Browse files
janduboisjandubois
committed
Add test harness for port forwarding rules
Signed-off-by: Jan Dubois <[email protected]>
1 parent 5241aea commit ad45119

File tree

2 files changed

+328
-0
lines changed

2 files changed

+328
-0
lines changed

‎hack/test-example.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#!/usr/bin/env bash
22
set -eu -o pipefail
33

4+
scriptdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5+
46
function INFO() {
57
echo "TEST| [INFO] $*"
68
}
@@ -35,6 +37,7 @@ declare -A CHECKS=(
3537
["mount-home"]="1"
3638
["containerd-user"]="1"
3739
["restart"]="1"
40+
["port-forwards"]="1"
3841
)
3942

4043
case "$NAME" in
@@ -60,6 +63,18 @@ if limactl ls -q | grep -q "$NAME"; then
6063
exit 1
6164
fi
6265

66+
if [[ -n ${CHECKS["port-forwards"]} ]]; then
67+
tmpconfig="$HOME/lime-config-tmp"
68+
mkdir -p "${tmpconfig}"
69+
trap 'rm -rf $tmpconfig' EXIT
70+
tmpfile="${tmpconfig}/${NAME}.yaml"
71+
cp "$FILE" "${tmpfile}"
72+
FILE="${tmpfile}"
73+
INFO "Setup port forwarding rules for testing in \"${FILE}\""
74+
"${scriptdir}/test-port-forwarding.pl" "${FILE}"
75+
limactl validate "$FILE"
76+
fi
77+
6378
function diagnose() {
6479
NAME="$1"
6580
set -x +e
@@ -142,13 +157,33 @@ if [[ -n ${CHECKS["containerd-user"]} ]]; then
142157
set +x
143158
fi
144159

160+
if [[ -n ${CHECKS["port-forwards"]} ]]; then
161+
INFO "Testing port forwarding rules using netcat"
162+
set -x
163+
if [ "${NAME}" = "archlinux" ]; then
164+
limactl shell "$NAME" sudo pacman -Syu --noconfirm openbsd-netcat
165+
fi
166+
if [ "${NAME}" = "debian" ]; then
167+
limactl shell "$NAME" sudo apt-get install -y netcat
168+
fi
169+
if [ "${NAME}" = "fedora" ]; then
170+
limactl shell "$NAME" sudo dnf install -y nc
171+
fi
172+
if [ "${NAME}" = "opensuse" ]; then
173+
limactl shell "$NAME" sudo zypper in -y netcat-openbsd
174+
fi
175+
"${scriptdir}/test-port-forwarding.pl" "${NAME}"
176+
set +x
177+
fi
178+
145179
if [[ -n ${CHECKS["restart"]} ]]; then
146180
INFO "Create file in the guest home directory and verify that it still exists after a restart"
147181
# shellcheck disable=SC2016
148182
limactl shell "$NAME" sh -c 'touch $HOME/sweet-home'
149183

150184
INFO "Stopping \"$NAME\""
151185
limactl stop "$NAME"
186+
sleep 3
152187

153188
INFO "Restarting \"$NAME\""
154189
limactl start "$NAME"
@@ -162,6 +197,7 @@ fi
162197

163198
INFO "Stopping \"$NAME\""
164199
limactl stop "$NAME"
200+
sleep 3
165201

166202
INFO "Deleting \"$NAME\""
167203
limactl delete "$NAME"

‎hack/test-port-forwarding.pl

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
#!/usr/bin/env perl
2+
3+
# This script tests the port forwarding settings of lima. It has to be run
4+
# twice: once to update the instance yaml file with the port forwarding
5+
# rules (before the instance is started). And once when the instance is
6+
# running to perform the tests:
7+
#
8+
# ./hack/test-port-forwarding.pl examples/default.yaml
9+
# limactl start --tty=false examples/default.yaml
10+
# git restore pkg/limayaml/default.yaml
11+
# ./hack/test-port-forwarding.pl default
12+
#
13+
# TODO: support for ipv6 host addresses
14+
15+
use strict;
16+
use warnings;
17+
18+
use IO::Handle qw();
19+
use Socket qw(inet_ntoa);
20+
use Sys::Hostname qw(hostname);
21+
22+
my $instance = shift;
23+
24+
my $ipv4 = inet_ntoa(scalar gethostbyname(hostname())) or die;
25+
my $ipv6 = ""; # todo
26+
27+
# If $instance is a filename, add our portForwards to it to enable testing
28+
if (-f $instance) {
29+
open(my $fh, "+< $instance") or die "Can't open $instance for read/write: $!";
30+
my @yaml;
31+
while (<$fh>) {
32+
# Remove existing "portForwards:" section from the config file
33+
my $seq = /^portForwards:/ ... /^[a-z]/;
34+
next if $seq && $seq !~ /E0$/;
35+
push @yaml, $_;
36+
}
37+
seek($fh, 0, 0);
38+
truncate($fh, 0);
39+
print $fh $_ for @yaml;
40+
while (<DATA>) {
41+
s/ipv4/$ipv4/g;
42+
s/ipv6/$ipv6/g;
43+
print $fh $_;
44+
}
45+
exit;
46+
}
47+
48+
# Otherwise $instance must be the name of an already running instance that has been
49+
# configured with our portForwards settings.
50+
51+
# Get sshLocalPort for lima instance
52+
my $sshLocalPort;
53+
open(my $ls, "limactl ls --json |") or die;
54+
while (<$ls>) {
55+
next unless /"name":"$instance"/;
56+
($sshLocalPort) = /"sshLocalPort":(\d+)/ or die;
57+
last;
58+
}
59+
die "Cannot determine sshLocalPort" unless $sshLocalPort;
60+
61+
# Extract forwarding tests from the "portForwards" section
62+
my @test;
63+
while (<DATA>) {
64+
chomp;
65+
s/^\s+#\s*//;
66+
next unless /^(forward|ignore)/;
67+
if (/ipv6/ && !$ipv6) {
68+
printf "🚧 Not yet: # $_\n";
69+
next;
70+
}
71+
s/sshLocalPort/$sshLocalPort/g;
72+
s/ipv4/$ipv4/g;
73+
s/ipv6/$ipv6/g;
74+
# forward: 127.0.0.1 899 → 127.0.0.1 799
75+
# ignore: 127.0.0.2 8888
76+
/^(forward|ignore):\s+([0-9.:]+)\s+(\d+)(?:\s+→)?(?:\s+([0-9.:]+)(?:\s+(\d+))?)?/;
77+
die "Cannot parse test '$_'" unless $1;
78+
my %test; @test{qw(mode guest_ip guest_port host_ip host_port)} = ($1, $2, $3, $4, $5);
79+
80+
$test{host_ip} ||= "127.0.0.1";
81+
$test{host_port} ||= $test{guest_port};
82+
83+
my $remote = JoinHostPort($test{guest_ip},$test{guest_port});
84+
my $local = JoinHostPort($test{host_ip},$test{host_port});
85+
if ($test{mode} eq "ignore") {
86+
$test{log_msg} = "Not forwarding TCP $remote";
87+
}
88+
else {
89+
$test{log_msg} = "Forwarding TCP from $remote to $local";
90+
}
91+
push @test, \%test;
92+
}
93+
94+
open(my $lima, "| limactl shell --workdir / $instance")
95+
or die "Can't run lima shell on $instance: $!";
96+
$lima->autoflush;
97+
98+
print $lima <<'EOF';
99+
set -e
100+
cd $HOME
101+
sudo pkill -x nc || true
102+
rm -f nc.*
103+
EOF
104+
105+
# Give the hostagent some time to remove any port forwards from a previous (crashed?) test run
106+
sleep 5;
107+
108+
# Record current log size, so we can skip prior output
109+
$ENV{LIMA_HOME} ||= "$ENV{HOME}/.lima";
110+
my $ha_log = "$ENV{LIMA_HOME}/$instance/ha.stderr.log";
111+
my $ha_log_size = -s $ha_log or die;
112+
113+
# Setup a netcat listener on the guest for each test
114+
foreach my $id (0..@test-1) {
115+
my $test = $test[$id];
116+
my $nc = "nc -l $test->{guest_ip} $test->{guest_port}";
117+
if ($instance eq "alpine") {
118+
$nc = "nc -l -s $test->{guest_ip} -p $test->{guest_port}";
119+
}
120+
121+
my $sudo = $test->{guest_port} < 1024 ? "sudo " : "";
122+
print $lima "${sudo}${nc} >nc.${id} 2>/dev/null &\n";
123+
}
124+
125+
# Make sure the guest- and hostagents had enough time to set up the forwards
126+
sleep 5;
127+
128+
# Try to reach each listener from the host
129+
foreach my $test (@test) {
130+
next if $test->{host_port} == $sshLocalPort;
131+
my $nc = "nc -w 1 $test->{host_ip} $test->{host_port}";
132+
open(my $netcat, "| $nc") or die "Can't run '$nc': $!";
133+
print $netcat "$test->{log_msg}\n";
134+
# Don't check for errors on close; macOS nc seems to return non-zero exit code even on success
135+
close($netcat);
136+
}
137+
138+
# Extract forwarding log messages from hostagent log
139+
open(my $log, "< $ha_log") or die "Can't read $ha_log: $!";
140+
seek($log, $ha_log_size, 0) or die "Can't seek $ha_log to $ha_log_size: $!";
141+
my %seen;
142+
while (<$log>) {
143+
$seen{$1}++ if /(Forwarding TCP from .*? to (\d.*?|\[.*?\]):\d+)/;
144+
$seen{$1}++ if /(Not forwarding TCP .*?:\d+)/;
145+
}
146+
close $log or die;
147+
148+
my $rc = 0;
149+
my %expected;
150+
foreach my $id (0..@test-1) {
151+
my $test = $test[$id];
152+
my $err = "";
153+
$expected{$test->{log_msg}}++;
154+
unless ($seen{$test->{log_msg}}) {
155+
$err .= "\n Message missing from ha.stderr.log";
156+
}
157+
my $log = qx(limactl shell --workdir / $instance sh -c "cd; cat nc.$id");
158+
chomp $log;
159+
if ($test->{mode} eq "forward" && $test->{log_msg} ne $log) {
160+
$err .= "\n Guest received: '$log'";
161+
}
162+
if ($test->{mode} eq "ignore" && $log) {
163+
$err .= "\n Guest received: '$log' (instead of nothing)";
164+
}
165+
printf "%s %s%s\n", ($err ? "❌" : "✅"), $test->{log_msg}, $err;
166+
$rc = 1 if $err;
167+
}
168+
169+
foreach (keys %seen) {
170+
next if $expected{$_};
171+
# Should this be an error? Really should only happen if something else failed as well.
172+
print "😕 Unexpected log message: $_\n";
173+
}
174+
175+
# Cleanup remaining netcat instances (and port forwards)
176+
print $lima "sudo pkill -x nc";
177+
178+
exit $rc;
179+
180+
sub JoinHostPort {
181+
my($host,$port) = @_;
182+
$host = "[$host]" if $host =~ /:/;
183+
return "$host:$port";
184+
}
185+
186+
# This YAML section includes port forwarding `rules` for the guest- and hostagents,
187+
# with interleaved `tests` (in comments) that are executed by this script. The strings
188+
# "ipv4" and "ipv6" will be replaced by the actual host ipv4 and ipv6 addresses.
189+
__DATA__
190+
portForwards:
191+
# We can't test that port 22 will be blocked because the guestagent has
192+
# been ignoring it since startup, so the log message is in the part of
193+
# the log we skipped.
194+
# skip: 127.0.0.1 22 → 127.0.0.1 2222
195+
# ignore: 127.0.0.1 sshLocalPort
196+
197+
- guestIP: 127.0.0.2
198+
guestPortRange: [3000, 3009]
199+
hostPortRange: [2000, 2009]
200+
ignore: true
201+
202+
- guestIP: 0.0.0.0
203+
guestPortRange: [3010, 3019]
204+
hostPortRange: [2010, 2019]
205+
ignore: true
206+
207+
- guestIP: 0.0.0.0
208+
guestPortRange: [3000, 3029]
209+
hostPortRange: [2000, 2029]
210+
211+
# The following rule is completely shadowed by the previous one and has no effect
212+
- guestIP: 0.0.0.0
213+
guestPortRange: [3020, 3029]
214+
hostPortRange: [2020, 2029]
215+
ignore: true
216+
217+
# ignore: 127.0.0.2 3000
218+
# forward: 127.0.0.3 3001 → 127.0.0.1 2001
219+
220+
# Blocking 127.0.0.2 cannot block forwarding from 0.0.0.0
221+
# forward: 0.0.0.0 3002 → 127.0.0.1 2002
222+
223+
# Blocking 0.0.0.0 will block forwarding from any interface
224+
# ignore: 0.0.0.0 3010
225+
# ignore: 127.0.0.1 3011
226+
227+
# Forwarding from 0.0.0.0 works for any interface (including IPv6)
228+
# The "ignore" rule above has no effect because the previous rule already matched.
229+
# forward: 127.0.0.2 3020 → 127.0.0.1 2020
230+
# forward: 127.0.0.1 3021 → 127.0.0.1 2021
231+
# forward: 0.0.0.0 3022 → 127.0.0.1 2022
232+
# forward: :: 3023 → 127.0.0.1 2023
233+
# forward: ::1 3024 → 127.0.0.1 2024
234+
235+
- guestPortRange: [3030, 3039]
236+
hostPortRange: [2030, 2039]
237+
hostIP: ipv4
238+
239+
# forward: 127.0.0.1 3030 → ipv4 2030
240+
# forward: 0.0.0.0 3031 → ipv4 2031
241+
# forward: :: 3032 → ipv4 2032
242+
# forward: ::1 3033 → ipv4 2033
243+
244+
# Forwarding privileged ports to 127.0.0.1 does not work yet
245+
- guestPortRange: [300, 309]
246+
247+
# Hostagent will try to setup forward, but ssh will fail to bind to port
248+
# skip: 127.0.0.1 300 → 127.0.0.1 300
249+
250+
- guestPortRange: [310, 319]
251+
hostIP: 0.0.0.0
252+
253+
# forward: 127.0.0.1 310 → 0.0.0.0 310
254+
255+
# Things we can't test:
256+
# - Accessing a forward from a different interface (e.g. connect to ipv4 to connect to 0.0.0.0)
257+
# - failed forward to privileged port
258+
259+
260+
- guestIP: "192.168.5.15"
261+
guestPortRange: [4000, 4009]
262+
hostIP: "ipv4"
263+
264+
# forward: 192.168.5.15 4000 → ipv4 4000
265+
266+
- guestIP: "::1"
267+
guestPortRange: [4010, 4019]
268+
hostIP: "::"
269+
270+
# forward: ::1 4010 → :: 4010
271+
272+
- guestIP: "::"
273+
guestPortRange: [4020, 4029]
274+
hostIP: "ipv4"
275+
276+
# forward: 127.0.0.1 4020 → ipv4 4020
277+
# forward: 127.0.0.2 4021 → ipv4 4021
278+
# forward: 192.168.5.15 4022 → ipv4 4022
279+
# forward: 0.0.0.0 4023 → ipv4 4023
280+
# forward: :: 4024 → ipv4 4024
281+
# forward: ::1 4025 → ipv4 4025
282+
283+
- guestIP: "0.0.0.0"
284+
guestPortRange: [4030, 4039]
285+
hostIP: "ipv4"
286+
287+
# forward: 127.0.0.1 4030 → ipv4 4030
288+
# forward: 127.0.0.2 4031 → ipv4 4031
289+
# forward: 192.168.5.15 4032 → ipv4 4032
290+
# forward: 0.0.0.0 4033 → ipv4 4033
291+
# forward: :: 4034 → ipv4 4034
292+
# forward: ::1 4035 → ipv4 4035

0 commit comments

Comments
 (0)