feat: update to use the localnet specific conduit algod importer (#700)

* feat: updating to use new localnet conduit importer
* chore: removing nginx proxy as it is no longer needed
* chore: removing version from the generated algod_config.json so we always use the latest version.
---------

Co-authored-by: Aidan Morgan <[email protected]>

Author: neilcampbell

src/algokit/cli/localnet.py

@@ -273,7 +273,7 @@ def reset_localnet(*, update: bool, config_path: Path | None, check: bool) -> No
     sandbox.up()
 
 
-SERVICE_NAMES = ("algod", "conduit", "indexer-db", "indexer", "proxy")
+SERVICE_NAMES = ("algod", "conduit", "indexer-db", "indexer")
 
 
 @localnet_group.command("status", short_help="Check the status of the AlgoKit LocalNet.")
@@ -303,10 +303,10 @@ def localnet_status(*, check: bool) -> None:
     }
     # fill out remaining output_by_name["algod"] values
     if output_by_name["algod"]["Status"] == "Running":
-        output_by_name["algod"].update(fetch_algod_status_data(ps_by_name["proxy"]))
+        output_by_name["algod"].update(fetch_algod_status_data(ps_by_name["algod"]))
     # fill out remaining output_by_name["indexer"] values
     if output_by_name["indexer"]["Status"] == "Running":
-        output_by_name["indexer"].update(fetch_indexer_status_data(ps_by_name["proxy"]))
+        output_by_name["indexer"].update(fetch_indexer_status_data(ps_by_name["indexer"]))
 
     # Print the status details
     for service_name, service_info in output_by_name.items():

src/algokit/core/sandbox.py

@@ -58,7 +58,6 @@ def __init__(self, name: str = SANDBOX_BASE_NAME, config_path: Path | None = Non
         self._latest_yaml = get_docker_compose_yml(name=f"algokit_{self.name}")
         self._latest_config_json = get_config_json()
         self._latest_algod_network_template = get_algod_network_template()
-        self._latest_proxy_config = get_proxy_config()
 
     @property
     def compose_file_path(self) -> Path:
@@ -76,10 +75,6 @@ def algod_config_file_path(self) -> Path:
     def algod_network_template_file_path(self) -> Path:
         return self.directory / "algod_network_template.json"
 
-    @property
-    def proxy_config_file_path(self) -> Path:
-        return self.directory / "nginx.conf"
-
     @classmethod
     def from_environment(cls) -> ComposeSandbox | None:
         try:
@@ -148,7 +143,6 @@ def compose_file_status(self) -> ComposeFileStatus:
             compose_content = self.compose_file_path.read_text()
             config_content = self.algod_config_file_path.read_text()
             algod_network_template_content = self.algod_network_template_file_path.read_text()
-            proxy_config_content = self.proxy_config_file_path.read_text()
 
         except FileNotFoundError:
             # treat as out of date if compose file exists but algod config doesn't
@@ -157,32 +151,38 @@ def compose_file_status(self) -> ComposeFileStatus:
                 return ComposeFileStatus.OUT_OF_DATE
             return ComposeFileStatus.MISSING
         else:
-            algod_network_json_content = json.loads(
-                algod_network_template_content.replace("NUM_ROUNDS", '"NUM_ROUNDS"')
-            )
-            latest_algod_network_json_content = json.loads(
-                self._latest_algod_network_template.replace("NUM_ROUNDS", '"NUM_ROUNDS"')
-            )
-
-            del algod_network_json_content["Genesis"]["DevMode"]
-            del latest_algod_network_json_content["Genesis"]["DevMode"]
-
-            if (
-                compose_content == self._latest_yaml
-                and config_content == self._latest_config_json
-                and algod_network_json_content == latest_algod_network_json_content
-                and proxy_config_content == self._latest_proxy_config
-            ):
-                return ComposeFileStatus.UP_TO_DATE
-            else:
+            try:
+                # Perform some content normalization to ensure we correctly detect up to date files
+
+                # Ensure the NUM_ROUNDS placeholder is the same in both files
+                algod_network_json_content = json.loads(
+                    algod_network_template_content.replace("NUM_ROUNDS", '"NUM_ROUNDS"')
+                )
+                latest_algod_network_json_content = json.loads(
+                    self._latest_algod_network_template.replace("NUM_ROUNDS", '"NUM_ROUNDS"')
+                )
+
+                # Remove DevMode from comparison as the value is configurable via the `--dev` option
+                del algod_network_json_content["Genesis"]["DevMode"]
+                del latest_algod_network_json_content["Genesis"]["DevMode"]
+
+                if (
+                    compose_content == self._latest_yaml
+                    and config_content == self._latest_config_json
+                    and algod_network_json_content == latest_algod_network_json_content
+                ):
+                    return ComposeFileStatus.UP_TO_DATE
+                else:
+                    return ComposeFileStatus.OUT_OF_DATE
+            except (json.JSONDecodeError, KeyError):
+                # If config files are corrupted or malformed, treat as out of date
                 return ComposeFileStatus.OUT_OF_DATE
 
     def write_compose_file(self) -> None:
         self.conduit_file_path.write_text(self._conduit_yaml)
         self.compose_file_path.write_text(self._latest_yaml)
         self.algod_config_file_path.write_text(self._latest_config_json)
         self.algod_network_template_file_path.write_text(self._latest_algod_network_template)
-        self.proxy_config_file_path.write_text(self._latest_proxy_config)
 
     def _run_compose_command(
         self,
@@ -328,7 +328,7 @@ def check_docker_compose_for_new_image_versions(self, *, force: bool = False) ->
 INDEXER_HEALTH_URL = f"{DEFAULT_INDEXER_SERVER}:{DEFAULT_INDEXER_PORT}/health"
 INDEXER_IMAGE = "algorand/indexer:latest"
 ALGORAND_IMAGE = "algorand/algod:latest"
-CONDUIT_IMAGE = "algorand/conduit:latest"
+CONDUIT_IMAGE = "algorandfoundation/conduit-localnet:latest"
 IMAGE_VERSION_CHECK_INTERVAL = timedelta(weeks=1).total_seconds()
 
 
@@ -440,8 +440,9 @@ def _wait_for_indexer() -> bool:
 
 def get_config_json() -> str:
     return (
-        '{ "Version": 12, "GossipFanout": 1, "EndpointAddress": "0.0.0.0:8080", "DNSBootstrapID": "",'
-        ' "IncomingConnectionsLimit": 0, "Archival":true, "isIndexerActive":false, "EnableDeveloperAPI":true}'
+        '{ "GossipFanout": 1, "EndpointAddress": "0.0.0.0:8080", "DNSBootstrapID": "",'
+        ' "IncomingConnectionsLimit": 0, "Archival":true, "isIndexerActive":false, "EnableDeveloperAPI":true,'
+        ' "EnablePrivateNetworkAccessHeader":true}'
     )
 
 
@@ -512,7 +513,7 @@ def get_conduit_yaml() -> str:
 retry-count: 10
 
 # Time duration to wait between retry attempts.
-retry-delay: "1s"
+retry-delay: "5s"
 
 # Optional filepath to use for pidfile.
 #pid-filepath: /path/to/pidfile
@@ -528,20 +529,10 @@ def get_conduit_yaml() -> str:
 
 # The importer is typically an algod follower node.
 importer:
-  name: algod
+  name: localnet_algod
   config:
-    # The mode of operation, either "archival" or "follower".
-    # * archival mode allows you to start processing on any round but does not
-    # contain the ledger state delta objects required for the postgres writer.
-    # * follower mode allows you to use a lightweight non-archival node as the
-    # data source. In addition, it will provide ledger state delta objects to
-    # the processors and exporter.
-    mode: "follower"
-
-    # Algod API address.
-    netaddr: "http://algod:8081"
-
-    # Algod API token.
+    lead-node-url: "http://algod:8080"
+    follower-node-url: "http://algod:8081"
     token: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 
 # Zero or more processors may be defined to manipulate what data
@@ -563,70 +554,6 @@ def get_conduit_yaml() -> str:
 """
 
 
-def get_proxy_config(algod_port: int = DEFAULT_ALGOD_PORT, kmd_port: int = 4002) -> str:
-    return f"""worker_processes 1;
-
-events {{
-  worker_connections 1024;
-}}
-
-http {{
-  access_log off;
-
-  resolver 127.0.0.11 ipv6=off valid=10s;
-  resolver_timeout 5s;
-  client_max_body_size 0;
-
-  map $request_method$http_access_control_request_private_network $cors_allow_private_network {{
-    "OPTIONStrue" "true";
-    default "";
-  }}
-
-  add_header Access-Control-Allow-Private-Network $cors_allow_private_network;
-
-  server {{
-    listen {algod_port};
-
-    location / {{
-      proxy_http_version 1.1;
-      proxy_read_timeout 120s;
-      proxy_set_header Host $host;
-      proxy_set_header Connection "";
-      proxy_pass_header Server;
-      set $target http://algod:8080;
-      proxy_pass $target;
-    }}
-  }}
-
-  server {{
-    listen {kmd_port};
-
-    location / {{
-      proxy_http_version 1.1;
-      proxy_set_header Host $host;
-      proxy_set_header Connection "";
-      proxy_pass_header Server;
-      set $target http://algod:7833;
-      proxy_pass $target;
-    }}
-  }}
-
-  server {{
-    listen 8980;
-
-    location / {{
-      proxy_http_version 1.1;
-      proxy_set_header Host $host;
-      proxy_set_header Connection "";
-      proxy_pass_header Server;
-      set $target http://indexer:8980;
-      proxy_pass $target;
-    }}
-  }}
-}}
-    """
-
-
 def get_docker_compose_yml(
     name: str = "algokit_sandbox",
     algod_port: int = DEFAULT_ALGOD_PORT,
@@ -640,7 +567,8 @@ def get_docker_compose_yml(
     container_name: "{name}_algod"
     image: {ALGORAND_IMAGE}
     ports:
-      - 7833
+      - {algod_port}:8080
+      - {kmd_port}:7833
       - {tealdbg_port}:9392
     environment:
       START_KMD: 1
@@ -684,28 +612,14 @@ def get_docker_compose_yml(
   indexer:
     container_name: "{name}_indexer"
     image: {INDEXER_IMAGE}
+    ports:
+      - 8980:8980
     restart: unless-stopped
     command: daemon --enable-all-parameters
     environment:
       INDEXER_POSTGRES_CONNECTION_STRING: "host=indexer-db port=5432 user=algorand password=algorand dbname=indexerdb sslmode=disable"
     depends_on:
       - conduit
-
-  proxy:
-    container_name: "{name}_proxy"
-    image: nginx:1.27.0-alpine
-    restart: unless-stopped
-    ports:
-      - {algod_port}:{algod_port}
-      - {kmd_port}:{kmd_port}
-      - 8980:8980
-    volumes:
-      - type: bind
-        source: ./nginx.conf
-        target: /etc/nginx/nginx.conf
-    depends_on:
-      - algod
-      - indexer
 """  # noqa: E501
 
 

tests/goal/test_goal.py

@@ -12,7 +12,6 @@
     get_algod_network_template,
     get_config_json,
     get_docker_compose_yml,
-    get_proxy_config,
 )
 from tests.utils.app_dir_mock import AppDirs
 from tests.utils.approvals import verify
@@ -51,7 +50,6 @@ def _setup_latest_dummy_compose(app_dir_mock: AppDirs) -> None:
     (app_dir_mock.app_config_dir / "sandbox" / "docker-compose.yml").write_text(get_docker_compose_yml())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_config.json").write_text(get_config_json())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_network_template.json").write_text(get_algod_network_template())
-    (app_dir_mock.app_config_dir / "sandbox" / "nginx.conf").write_text(get_proxy_config())
 
 
 @pytest.fixture

tests/localnet/test_localnet_console.py

@@ -4,7 +4,7 @@
 import pytest
 from pytest_mock import MockerFixture
 
-from algokit.core.sandbox import get_algod_network_template, get_config_json, get_docker_compose_yml, get_proxy_config
+from algokit.core.sandbox import get_algod_network_template, get_config_json, get_docker_compose_yml
 from tests.goal.test_goal import _normalize_output
 from tests.utils.app_dir_mock import AppDirs
 from tests.utils.approvals import verify
@@ -26,7 +26,6 @@ def test_goal_console(
     (app_dir_mock.app_config_dir / "sandbox" / "docker-compose.yml").write_text(get_docker_compose_yml())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_config.json").write_text(get_config_json())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_network_template.json").write_text(get_algod_network_template())
-    (app_dir_mock.app_config_dir / "sandbox" / "nginx.conf").write_text(get_proxy_config())
 
     mocker.patch("algokit.core.proc.subprocess_run").return_value = CompletedProcess(
         ["docker", "exec"], 0, "STDOUT+STDERR"

tests/localnet/test_localnet_reset.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from algokit.core.sandbox import get_algod_network_template, get_config_json, get_docker_compose_yml, get_proxy_config
+from algokit.core.sandbox import get_algod_network_template, get_config_json, get_docker_compose_yml
 from tests import get_combined_verify_output
 from tests.utils.app_dir_mock import AppDirs
 from tests.utils.approvals import verify
@@ -56,7 +56,6 @@ def test_localnet_reset_with_existing_sandbox_with_up_to_date_config(app_dir_moc
     (app_dir_mock.app_config_dir / "sandbox" / "docker-compose.yml").write_text(get_docker_compose_yml())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_config.json").write_text(get_config_json())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_network_template.json").write_text(get_algod_network_template())
-    (app_dir_mock.app_config_dir / "sandbox" / "nginx.conf").write_text(get_proxy_config())
 
     result = invoke("localnet reset")
 
@@ -81,7 +80,6 @@ def test_localnet_reset_with_named_sandbox_config(proc_mock: ProcMock, app_dir_m
     (app_dir_mock.app_config_dir / "sandbox_test" / "algod_network_template.json").write_text(
         get_algod_network_template()
     )
-    (app_dir_mock.app_config_dir / "sandbox_test" / "nginx.conf").write_text(get_proxy_config())
 
     result = invoke("localnet reset")
 
@@ -99,7 +97,6 @@ def test_localnet_reset_with_existing_sandbox_with_up_to_date_config_with_pull(a
     (app_dir_mock.app_config_dir / "sandbox" / "docker-compose.yml").write_text(get_docker_compose_yml())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_config.json").write_text(get_config_json())
     (app_dir_mock.app_config_dir / "sandbox" / "algod_network_template.json").write_text(get_algod_network_template())
-    (app_dir_mock.app_config_dir / "sandbox" / "nginx.conf").write_text(get_proxy_config())
 
     result = invoke("localnet reset --update")
 

tests/localnet/test_localnet_reset.test_localnet_reset_with_existing_sandbox_with_out_of_date_config.approved.txt

@@ -35,7 +35,8 @@ services:
     container_name: "algokit_sandbox_algod"
     image: algorand/algod:latest
     ports:
-      - 7833
+      - 4001:8080
+      - 4002:7833
       - 9392:9392
     environment:
       START_KMD: 1
@@ -55,7 +56,7 @@ services:
 
   conduit:
     container_name: "algokit_sandbox_conduit"
-    image: algorand/conduit:latest
+    image: algorandfoundation/conduit-localnet:latest
     restart: unless-stopped
     volumes:
       - type: bind
@@ -79,28 +80,14 @@ services:
   indexer:
     container_name: "algokit_sandbox_indexer"
     image: algorand/indexer:latest
+    ports:
+      - 8980:8980
     restart: unless-stopped
     command: daemon --enable-all-parameters
     environment:
       INDEXER_POSTGRES_CONNECTION_STRING: "host=indexer-db port=5432 user=algorand password=algorand dbname=indexerdb sslmode=disable"
     depends_on:
       - conduit
 
-  proxy:
-    container_name: "algokit_sandbox_proxy"
-    image: nginx:1.27.0-alpine
-    restart: unless-stopped
-    ports:
-      - 4001:4001
-      - 4002:4002
-      - 8980:8980
-    volumes:
-      - type: bind
-        source: ./nginx.conf
-        target: /etc/nginx/nginx.conf
-    depends_on:
-      - algod
-      - indexer
-
 {app_config}/sandbox/algod_config.json
-{ "Version": 12, "GossipFanout": 1, "EndpointAddress": "0.0.0.0:8080", "DNSBootstrapID": "", "IncomingConnectionsLimit": 0, "Archival":true, "isIndexerActive":false, "EnableDeveloperAPI":true}
+{ "GossipFanout": 1, "EndpointAddress": "0.0.0.0:8080", "DNSBootstrapID": "", "IncomingConnectionsLimit": 0, "Archival":true, "isIndexerActive":false, "EnableDeveloperAPI":true, "EnablePrivateNetworkAccessHeader":true}