Forward arbitrary TCP traffic using aproxy (#580)

* Forward arbitrary TCP traffic using aproxy

* Update integration tests and fix linting tests

* Fix linting issue and update integration issues

* Fix linting issues

* Update e2e test

* Add validator to aproxy_redirect_ports and aproxy_exclude_addresses

* Use partition instead of split

* Simplify _generate_cloud_init

* Fix _generate_cloud_init

* Simplify _generate_cloud_init

* Fix e2e test

* Apply suggestions from review comments

* Small refactor over the charm_state.py

* Fix unit tests

* remove test for TCP proxy

* Update e2e tests

* Always exclude 127.0.0.0/8

* Increase timeout for runner_manager_with_one_runner_fixture

Author: weiiwang01

.github/workflows/e2e_test_run.yaml

@@ -22,7 +22,7 @@ on:
 jobs:
   e2e-test:
     name: End-to-End Test Run
-    runs-on: [self-hosted, linux, "${{ inputs.runner-tag }}"]
+    runs-on: [ self-hosted, linux, "${{ inputs.runner-tag }}" ]
     steps:
       - name: Hostname is set to "github-runner"
         run: sudo hostnamectl hostname | grep github-runner
@@ -47,6 +47,14 @@ jobs:
         run: |
           [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \
             || cat /home/ubuntu/.docker/config.json | grep httpProxy
+      - name: test network connectivity
+        run: |
+          timeout 60 curl --noproxy "*" http://example.com -svS -o /dev/null
+          timeout 60 curl --noproxy "*" https://example.com -svS -o /dev/null
+      - name: test aproxy logs
+        run: |
+          sudo snap logs aproxy.aproxy | grep -Fq "example.com:80"
+          sudo snap logs aproxy.aproxy | grep -Fq "example.com:443"
       - name: Install microk8s
         run: sudo snap install microk8s --classic
       - name: Wait for microk8s
@@ -87,3 +95,9 @@ jobs:
       # ~/.local/bin is added to path runner env through in scripts/env.j2
       - name: test check-jsonschema
         run: check-jsonschema --version
+      - name: show aproxy logs
+        if: always()
+        run: |
+          sudo snap get aproxy
+          sudo snap logs aproxy.aproxy -n=all
+          sudo nft list ruleset

config.yaml

@@ -37,13 +37,26 @@ options:
     default: false
     description: >-
       (Experimental, may be removed) When set to true, aproxy (https://github.com/canonical/aproxy)
-      will be installed within the runners. It will forward all HTTP(S) traffic to standard ports
-      (80, 443, 11371) to a proxy server configured by the juju model config 'juju-http-proxy'
+      will be installed within the runners. It will forward TCP traffic matching the 'aproxy-exclude-addresses'
+      and 'aproxy-redirect-ports' settings to a proxy server configured by the Juju model config 'juju-http-proxy'
       (or, if this is not set, 'juju-https-proxy' will be used).
       This is useful when the charm is deployed in a network that requires a proxy to access the
       internet.
-      Note that you should not specify a proxy server listening on port 80 or 443, as all traffic
-      to these ports is relayed to aproxy, which would cause an infinite loop.
+      Note that you should carefully choose values for the 'aproxy-exclude-addresses' and
+      'aproxy-redirect-ports' so that the network traffic from the runner to the HTTP proxy is not 
+      captured by aproxy. The simplest way to achieve this is to add the IP address of the HTTP proxy 
+      to 'aproxy-exclude-addresses' or exclude the HTTP proxy port from 'aproxy-redirect-ports'.
+  aproxy-exclude-addresses:
+    type: string
+    default: "10.0.0.0/8, 171.16.0.0/12, 192.168.0.0/16"
+    description: >-
+      A comma-separated list of IP addresses that should be excluded from redirection to aproxy.
+      127.0.0.0/8 are always excluded so you can omit if from the configuration.
+  aproxy-redirect-ports:
+    type: string
+    default: "80, 443"
+    description: >-
+      A comma-separated list of ports or port ranges that should be redirected to aproxy.
   group:
     type: string
     default: "default"

docs/changelog.md

@@ -2,12 +2,13 @@
 
 This changelog documents user-relevant changes to the GitHub runner charm.
 
+### 2025-06-30
+- New configuration options aproxy-exclude-addresses and aproxy-redirect-ports for allowing aproxy to redirect arbitrary TCP traffic
+
 ## 2025-06-26
 
 - Fix a process leak internal to the charm.
 
-## 2025-06-24
-
 - Fix a bug where deleted GitHub Actions Job would cause an endless loop of retries.
 
 ### 2025-06-17

github-runner-manager/src/github_runner_manager/configuration/base.py

@@ -81,6 +81,8 @@ class SupportServiceConfig(BaseModel):
         proxy_config: The proxy configuration.
         runner_proxy_config: The proxy configuration for the runner.
         use_aproxy: Whether aproxy should be used for the runners.
+        aproxy_exclude_addresses: A list of addresses to exclude from the aproxy proxy.
+        aproxy_redirect_ports: A list of ports to redirect to the aproxy proxy.
         dockerhub_mirror: The dockerhub mirror to use for runners.
         ssh_debug_connections: The information on the ssh debug services.
         repo_policy_compliance: The configuration of the repo policy compliance service.
@@ -91,6 +93,8 @@ class SupportServiceConfig(BaseModel):
     proxy_config: "ProxyConfig | None"
     runner_proxy_config: "ProxyConfig | None"
     use_aproxy: bool
+    aproxy_exclude_addresses: list[str] = []
+    aproxy_redirect_ports: list[str] = []
     dockerhub_mirror: str | None
     ssh_debug_connections: "list[SSHDebugConnection]"
     repo_policy_compliance: "RepoPolicyComplianceConfig | None"

github-runner-manager/src/github_runner_manager/openstack_cloud/openstack_runner_manager.py

@@ -282,15 +282,24 @@ def _generate_cloud_init(self, runner_context: RunnerContext) -> str:
 
         pre_job_contents = jinja.get_template("pre-job.j2").render(pre_job_contents_dict)
 
-        aproxy_address = (
-            service_config.runner_proxy_config.proxy_address if service_config.use_aproxy else None
-        )
+        use_aproxy = service_config.use_aproxy
+        if not service_config.runner_proxy_config.proxy_address:
+            use_aproxy = False
+        aproxy_redirect_ports = service_config.aproxy_redirect_ports
+        if not aproxy_redirect_ports:
+            use_aproxy = False
+        aproxy_exclude_ipv4_addresses = [
+            address for address in service_config.aproxy_exclude_addresses if ":" not in address
+        ]
         return jinja.get_template("openstack-userdata.sh.j2").render(
             run_script=runner_context.shell_run_script,
             env_contents=env_contents,
             pre_job_contents=pre_job_contents,
             metrics_exchange_path=str(METRICS_EXCHANGE_PATH),
-            aproxy_address=aproxy_address,
+            use_aproxy=use_aproxy,
+            aproxy_address=service_config.runner_proxy_config.proxy_address,
+            aproxy_exclude_ipv4_addresses=", ".join(aproxy_exclude_ipv4_addresses),
+            aproxy_redirect_ports=", ".join(aproxy_redirect_ports),
             dockerhub_mirror=service_config.dockerhub_mirror,
             ssh_debug_info=ssh_debug_info,
             runner_proxy_config=service_config.runner_proxy_config,

github-runner-manager/src/github_runner_manager/templates/openstack-userdata.sh.j2

@@ -10,23 +10,26 @@ su - ubuntu -c 'cd ~/actions-runner && echo "{{ env_contents }}" > .env'
 snap refresh --hold=48h
 snap watch --last=auto-refresh?
 
-{% if aproxy_address %}
+{% if use_aproxy %}
 snap install aproxy --edge
 snap set aproxy proxy={{ aproxy_address }} listen=:54969
 cat << EOF > /etc/nftables.conf
-define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+')
-define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 }
+define default-ipv4 = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+')
 table ip aproxy
 flush table ip aproxy
 table ip aproxy {
+      set exclude {
+          type ipv4_addr;
+          flags interval; auto-merge;
+          elements = { 127.0.0.0/8, {{ aproxy_exclude_ipv4_addresses }} }
+      }
       chain prerouting {
               type nat hook prerouting priority dstnat; policy accept;
-              ip daddr != \$private-ips tcp dport { 80, 443, 11371 } counter dnat to \$default-ip:54969
+              ip daddr != @exclude tcp dport { {{ aproxy_redirect_ports }} } counter dnat to \$default-ipv4:54969
       }
-
       chain output {
               type nat hook output priority -100; policy accept;
-              ip daddr != \$private-ips tcp dport { 80, 443, 11371 } counter dnat to \$default-ip:54969
+              ip daddr != @exclude tcp dport { {{ aproxy_redirect_ports }} } counter dnat to \$default-ipv4:54969
       }
 }
 EOF

github-runner-manager/tests/unit/openstack_cloud/test_openstack_runner_manager.py

@@ -3,6 +3,7 @@
 
 """Module for unit-testing OpenStack runner manager."""
 import logging
+import textwrap
 from datetime import datetime, timezone
 from typing import Iterable
 from unittest.mock import MagicMock
@@ -77,8 +78,89 @@ def runner_metrics_mock_fixture(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
     return runner_metrics_mock
 
 
+@pytest.mark.parametrize(
+    "aproxy_redirect_ports, aproxy_exclude_addresses, aproxy_used, except_aproxy_script",
+    [
+        pytest.param(
+            [],
+            ["10.0.0.0/8"],
+            False,
+            "",
+            id="empty aproxy_redirect_ports disables aproxy",
+        ),
+        pytest.param(
+            ["80", "443"],
+            ["10.0.0.0/8", "192.168.0.0/16"],
+            True,
+            "10.0.0.0/8, 192.168.0.0/16",
+            id="aproxy with custom aproxy_exclude_addresses",
+        ),
+        pytest.param(
+            ["0-3127", "3129-65535"],
+            ["10.0.0.0/8", "192.168.0.0/16"],
+            True,
+            "0-3127, 3129-65535",
+            id="aproxy with custom aproxy_redirect_ports",
+        ),
+        pytest.param(
+            ["80", "443"],
+            ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"],
+            True,
+            textwrap.dedent(
+                """\
+    table ip aproxy {
+          set exclude {
+              type ipv4_addr;
+              flags interval; auto-merge;
+              elements = { 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 }
+          }
+          chain prerouting {
+                  type nat hook prerouting priority dstnat; policy accept;
+                  ip daddr != @exclude tcp dport { 80, 443 } counter dnat to \\$default-ipv4:54969
+          }
+          chain output {
+                  type nat hook output priority -100; policy accept;
+                  ip daddr != @exclude tcp dport { 80, 443 } counter dnat to \\$default-ipv4:54969
+          }
+    }
+            """
+            ),
+            id="aproxy default config",
+        ),
+        pytest.param(
+            ["80", "443"],
+            [],
+            True,
+            textwrap.dedent(
+                """\
+    table ip aproxy {
+          set exclude {
+              type ipv4_addr;
+              flags interval; auto-merge;
+              elements = { 127.0.0.0/8,  }
+          }
+          chain prerouting {
+                  type nat hook prerouting priority dstnat; policy accept;
+                  ip daddr != @exclude tcp dport { 80, 443 } counter dnat to \\$default-ipv4:54969
+          }
+          chain output {
+                  type nat hook output priority -100; policy accept;
+                  ip daddr != @exclude tcp dport { 80, 443 } counter dnat to \\$default-ipv4:54969
+          }
+    }
+            """
+            ),
+            id="aproxy with no aproxy_exclude_addresses",
+        ),
+    ],
+)
 def test_create_runner_with_aproxy(
-    runner_manager: OpenStackRunnerManager, monkeypatch: pytest.MonkeyPatch
+    aproxy_redirect_ports: list[str],
+    aproxy_exclude_addresses: list[str],
+    aproxy_used: str,
+    except_aproxy_script: str,
+    runner_manager: OpenStackRunnerManager,
+    monkeypatch: pytest.MonkeyPatch,
 ):
     """
     arrange: Prepare service config with aproxy enabled and a runner proxy config.
@@ -88,6 +170,8 @@ def test_create_runner_with_aproxy(
     # Pending to pass service_config as a dependency instead of mocking it this way.
     service_config = runner_manager._config.service_config
     service_config.use_aproxy = True
+    service_config.aproxy_redirect_ports = aproxy_redirect_ports
+    service_config.aproxy_exclude_addresses = aproxy_exclude_addresses
     service_config.runner_proxy_config = ProxyConfig(http="http://proxy.example.com:3128")
 
     prefix = "test"
@@ -101,10 +185,11 @@ def test_create_runner_with_aproxy(
 
     runner_manager.create_runner(identity, runner_context)
     openstack_cloud.launch_instance.assert_called_once()
-    assert (
-        "snap set aproxy proxy=proxy.example.com:3128"
-        in openstack_cloud.launch_instance.call_args.kwargs["cloud_init"]
-    )
+
+    cloud_init = openstack_cloud.launch_instance.call_args.kwargs["cloud_init"]
+    assert ("snap set aproxy proxy=proxy.example.com:3128" in cloud_init) == aproxy_used
+    if aproxy_used:
+        assert except_aproxy_script in cloud_init
 
 
 def test_create_runner_without_aproxy(