Smoketests can run against remote servers (#3012)

# Description of Changes

This enables smoketests to run against remote servers, such as maincloud
/ maincloud staging.

I also added a `--spacetime-login` param, for servers that require a
"proper" spacetime login (such as both servers above).

Usage:
```bash
python3 -m smoketests \
  --remote-server https://maincloud.staging.spacetimedb.com \
  --spacetime-login \
  -x replication # for some reason this is required, even though I swear it should be disabled by not passing `--docker`
```

# API and ABI breaking changes

None. CI only.

# Expected complexity level and risk

1

# Testing
- [x] Smoketests pass on this PR
- [x] Smoketests pass when run against maincloud staging (using the
instructions above)
- [x] Manual review to check whether I've accidentally de-fanged any
"test for negative case" tests

---------

Co-authored-by: Zeke Foppa <[email protected]>

Author: bfops

smoketests/__init__.py

@@ -37,6 +37,14 @@
 # and a dotnet installation is detected
 HAVE_DOTNET = False
 
+# When we pass --spacetime-login, we are running against a server that requires "real" spacetime logins (rather than `--server-issued-login`).
+# This is used to skip tests that don't work with that.
+USE_SPACETIME_LOGIN = False
+
+# If we pass `--remote-server`, the server address will be something other than the default. This is used to skip tests that rely on use
+# having the default localhost server.
+REMOTE_SERVER = False
+
 # default value can be overridden by `--compose-file` flag
 COMPOSE_FILE = "./docker-compose.yml"
 
@@ -61,6 +69,15 @@ def requires_dotnet(item):
         return item
     return unittest.skip("dotnet 8.0 not available")(item)
 
+def requires_anonymous_login(item):
+    if USE_SPACETIME_LOGIN:
+        return unittest.skip("using `spacetime login`")(item)
+    return item
+
+def requires_local_server(item):
+    if REMOTE_SERVER:
+        return unittest.skip("running against a remote server")(item)
+    return item
 
 def build_template_target():
     if not TEMPLATE_TARGET_DIR.exists():
@@ -260,9 +277,20 @@ def run():
     def api_call(self, method, path, body = None, headers = {}):
         with open(self.config_path, "rb") as f:
             config = tomllib.load(f)
-            host = config['default_server']
             token = config['spacetimedb_token']
-            conn = http.client.HTTPConnection(host)
+            server_name = config['default_server']
+            server_config = next((c for c in config['server_configs'] if c['nickname'] == server_name), None)
+            if server_config is None:
+                raise Exception(f"Unable to find server in config with nickname {server_name}")
+            host = server_config['host']
+            protocol = server_config['protocol']
+            conn = None
+            if protocol == "http":
+                conn = http.client.HTTPConnection(host)
+            elif protocol == "https":
+                conn = http.client.HTTPSConnection(host)
+            else:
+                raise Exception(f"Unknown protocol: {protocol}")
             auth = {"Authorization": f'Bearer {token}'}
             headers.update(auth)
             log_cmd([method, path])

smoketests/__main__.py

@@ -72,6 +72,8 @@ def main():
     parser.add_argument("-x", dest="exclude", nargs="*", default=[])
     parser.add_argument("--no-build-cli", action="store_true", help="don't cargo build the cli")
     parser.add_argument("--list", action="store_true", help="list the tests that would be run, but don't run them")
+    parser.add_argument("--remote-server", action="store", help="Run against a remote server")
+    parser.add_argument("--spacetime-login", action="store_true", help="Use `spacetime login` for these tests (and disable tests that don't work with that)")
     args = parser.parse_args()
 
     if not args.no_build_cli:
@@ -110,7 +112,16 @@ def main():
                 subprocess.Popen(["docker", "logs", "-f", docker_container])
         smoketests.HAVE_DOCKER = True
 
-    smoketests.new_identity(TEST_DIR / 'config.toml')
+    if args.remote_server is not None:
+        smoketests.spacetime("--config-path", TEST_DIR / 'config.toml', "server", "edit", "localhost", "--url", args.remote_server, "--yes")
+        smoketests.REMOTE_SERVER = True
+
+    if args.spacetime_login:
+        smoketests.spacetime("--config-path", TEST_DIR / 'config.toml', "logout")
+        smoketests.spacetime("--config-path", TEST_DIR / 'config.toml', "login")
+        smoketests.USE_SPACETIME_LOGIN = True
+    else:
+        smoketests.new_identity(TEST_DIR / 'config.toml')
 
     if not args.skip_dotnet:
         smoketests.HAVE_DOTNET = check_dotnet()

smoketests/config.toml

@@ -1,4 +1,4 @@
-default_server = "127.0.0.1:3000"
+default_server = "localhost"
 spacetimedb_token = "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJoZXhfaWRlbnRpdHkiOiJjMjAwYzc3NDY1NTE5MDM2MTE4M2JiNjFmMWMxYzY3NDUzMzYzY2MxMTY4MmM1NTUwNWZiNjdlYzI0ZWMyMWViIiwic3ViIjoiOTJlMmNkOGQtNTk5Ny00NjZlLWIwNmYtZDNjOGQ1NzU3ODI4IiwiaXNzIjoibG9jYWxob3N0IiwiYXVkIjpbInNwYWNldGltZWRiIl0sImlhdCI6MTc1MjA0NjgwMCwiZXhwIjpudWxsfQ.dgefoxC7eCOONVUufu2JTVFo9876zQ4Mqwm0ivZ0PQK7Hacm3Ip_xqyav4bilZ0vIEf8IM8AB0_xawk8WcbvMg"
 
 [[server_configs]]

smoketests/tests/energy.py

@@ -1,8 +1,9 @@
-from .. import Smoketest
+from .. import Smoketest, requires_anonymous_login
 import time
 
 class EnergyFlow(Smoketest):
 
+    @requires_anonymous_login
     def test_energy_balance(self):
         """Test getting energy balance."""
 

smoketests/tests/new_user_flow.py

@@ -1,4 +1,4 @@
-from .. import Smoketest
+from .. import Smoketest, requires_anonymous_login
 import time
 
 class NewUserFlow(Smoketest):
@@ -25,6 +25,7 @@ class NewUserFlow(Smoketest):
 }
 """
 
+    @requires_anonymous_login
     def test_new_user_flow(self):
         """Test the entirety of the new user flow."""
 

smoketests/tests/permissions.py

@@ -10,8 +10,6 @@ def setUp(self):
     def test_call(self):
         """Ensure that anyone has the permission to call any standard reducer"""
 
-        self.new_identity()
-
         self.publish_module()
 
         self.call("say_hello", anon=True)
@@ -21,28 +19,22 @@ def test_call(self):
     def test_delete(self):
         """Ensure that you cannot delete a database that you do not own"""
 
-        self.new_identity()
-
         self.publish_module()
 
-        self.reset_config()
+        self.new_identity()
         with self.assertRaises(Exception):
             self.spacetime("delete", self.database_identity)
 
     def test_describe(self):
         """Ensure that anyone can describe any database"""
 
-        self.new_identity()
         self.publish_module()
 
-        self.reset_config()
-        self.new_identity()
-        self.spacetime("describe", "--json", self.database_identity)
+        self.spacetime("describe", "--anonymous", "--json", self.database_identity)
 
     def test_logs(self):
         """Ensure that we are not able to view the logs of a module that we don't have permission to view"""
 
-        self.new_identity()
         self.publish_module()
 
         self.reset_config()
@@ -57,10 +49,9 @@ def test_logs(self):
     def test_publish(self):
         """This test checks to make sure that you cannot publish to an identity that you do not own."""
 
-        self.new_identity()
         self.publish_module()
 
-        self.reset_config()
+        self.new_identity()
 
         with self.assertRaises(Exception):
             # TODO: This raises for the wrong reason - `--clear-database` doesn't exist anymore!
@@ -73,11 +64,10 @@ def test_publish(self):
     def test_replace_names(self):
         """Test that you can't replace names of a database you don't own"""
 
-        self.new_identity()
         name = random_string()
         self.publish_module(name)
 
-        self.reset_config()
+        self.new_identity()
 
         with self.assertRaises(Exception):
             self.api_call(

smoketests/tests/replication.py

@@ -229,6 +229,7 @@ def tearDown(self):
         self.docker.compose("up", "-d")
         super().tearDown()
 
+    # TODO: This function seems to run even when `--docker` is not passed, leading to errors unless `-x replication` is passed, due to the docker-related code below.
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 

smoketests/tests/servers.py

@@ -1,6 +1,8 @@
-from .. import Smoketest, extract_field
+from .. import Smoketest, extract_field, requires_local_server
 import re
 
+# We require a local server because these tests have hardcoded server addresses.
+@requires_local_server
 class Servers(Smoketest):
     AUTOPUBLISH = False