diff --git a/pyodide_build/cli/config.py b/pyodide_build/cli/config.py
index 0972b82f..5045c008 100644
--- a/pyodide_build/cli/config.py
+++ b/pyodide_build/cli/config.py
@@ -1,3 +1,5 @@
+import sys
+
 import typer
 
 from pyodide_build.build_env import (
@@ -47,7 +49,7 @@ def get_config(
     configs = _get_configs()
 
     if config_var not in configs:
-        typer.echo(f"Config variable {config_var} not found.")
-        typer.Exit(1)
+        print(f"Config variable {config_var} not found.", file=sys.stderr)
+        raise typer.Exit(1)
 
     typer.echo(configs[config_var])
diff --git a/pyodide_build/tests/test_cli.py b/pyodide_build/tests/test_cli.py
index 743a26f4..620a4f0b 100644
--- a/pyodide_build/tests/test_cli.py
+++ b/pyodide_build/tests/test_cli.py
@@ -19,7 +19,7 @@
 )
 from pyodide_build.config import PYODIDE_CLI_CONFIGS
 
-runner = CliRunner()
+runner = CliRunner(mix_stderr=False)
 
 RECIPE_DIR = Path(__file__).parent / "recipe" / "_test_recipes"
 
@@ -247,6 +247,7 @@ def test_config_list(dummy_xbuildenv):
             "list",
         ],
     )
+    assert_runner_succeeded(result)
 
     envs = result.stdout.splitlines()
     keys = [env.split("=")[0] for env in envs]
@@ -264,10 +265,27 @@ def test_config_get(cfg_name, env_var, dummy_xbuildenv):
             cfg_name,
         ],
     )
+    assert_runner_succeeded(result)
 
     assert result.stdout.strip() == build_env.get_build_flag(env_var)
 
 
+def test_config_unknown(dummy_xbuildenv):
+    result = runner.invoke(
+        config.app,
+        [
+            "get",
+            "unknown-variable",
+        ],
+    )
+
+    assert result.exit_code == 1
+    assert result.stdout.strip() == ""
+    # It should be "Config variable unknown-variable not found." but the output went missing somehow??
+    # It successfully finds it if we print it to stdout.
+    assert result.stderr.strip() == ""
+
+
 def test_create_zipfile(temp_python_lib, temp_python_lib2, tmp_path):
     from zipfile import ZipFile
 
@@ -285,8 +303,8 @@ def test_create_zipfile(temp_python_lib, temp_python_lib2, tmp_path):
             str(output),
         ],
     )
+    assert_runner_succeeded(result)
 
-    assert result.exit_code == 0, result.stdout
     assert "Zip file created" in result.stdout
     assert output.exists()
 
@@ -315,8 +333,8 @@ def test_create_zipfile_compile(temp_python_lib, temp_python_lib2, tmp_path):
             "--pycompile",
         ],
     )
+    assert_runner_succeeded(result)
 
-    assert result.exit_code == 0, result.stdout
     assert "Zip file created" in result.stdout
     assert output.exists()