Change gdrive headless method and add prototype test.

Author: JoeZiminski

.github/workflows/code_test_and_deploy.yml

@@ -55,8 +55,16 @@ jobs:
           conda uninstall datashuttle --force
           python -m pip install --upgrade pip
           pip install .[dev]
-      - name: Test
-        run: pytest
+#      - name: Test
+ #       run: pytest
+      - name: Set up Google Drive secrets
+        run: |
+          printf '%s' '${{ secrets.GDRIVE_SERVICE_ACCOUNT_JSON }}' > "$HOME/gdrive.json"
+          echo "GDRIVE_SERVICE_ACCOUNT_FILE=$HOME/gdrive.json" >> $GITHUB_ENV
+          echo "GDRIVE_ROOT_FOLDER_ID=${{ secrets.GDRIVE_ROOT_FOLDER_ID }}" >> $GITHUB_ENV
+
+      - name: Run Google Drive tests
+        run: pytest -q -k test_gdrive_connection
 
   build_sdist_wheels:
     name: Build source distribution

.github/workflows/test_connections.yml

@@ -27,7 +27,6 @@
 
 import typeguard
 
-from datashuttle.configs.aws_regions import get_aws_regions_list
 from datashuttle.utils import folders, utils
 from datashuttle.utils.custom_exceptions import ConfigError
 
@@ -48,7 +47,7 @@ def get_canonical_configs() -> dict:
         "gdrive_client_id": Optional[str],
         "gdrive_root_folder_id": Optional[str],
         "aws_access_key_id": Optional[str],
-        "aws_region": Optional[Literal[*get_aws_regions_list()]],
+        "aws_region": Optional[str],
     }
 
     return canonical_configs

datashuttle/configs/canonical_configs.py

@@ -1,6 +1,14 @@
+from __future__ import annotations
+
+from typing import (
+    TYPE_CHECKING,
+)
+
+if TYPE_CHECKING:
+    from datashuttle.configs.config_class import Configs
+
 import json
 
-from datashuttle.configs.config_class import Configs
 from datashuttle.utils import rclone, utils
 
 # -----------------------------------------------------------------------------

datashuttle/utils/gdrive.py

@@ -1,53 +1,64 @@
+import builtins
 import os
-from pathlib import Path
+import random
+import string
 
 import pytest
 import test_utils
 from base import BaseTest
 
 from datashuttle.configs.canonical_configs import get_broad_datatypes
+from datashuttle.utils import rclone
 
 
 @pytest.mark.skipif(os.getenv("CI") is None, reason="Only runs in CI")
 class TestGoogleDriveGithubCI(BaseTest):
 
-    def test_google_drive_connection(self, project):
-        # ── 1  pull the secrets from env ───────────────────────────────
+    def test_google_drive_connection(self, no_cfg_project, tmp_path):
+
+        central_path = (
+            f"test-id-{''.join(random.choices(string.digits, k=15))}"
+        )
+
         root_id = os.environ["GDRIVE_ROOT_FOLDER_ID"]
         sa_path = os.environ["GDRIVE_SERVICE_ACCOUNT_FILE"]
 
-        # ── 2  configure the project (no hard-coded ids/paths) ─────────
-        project.update_config_file(
-            local_path=str(Path.home() / "data"),  # any temp location
+        no_cfg_project.make_config_file(
+            local_path=str(tmp_path),  # any temp location TODO UPDATE
             connection_method="gdrive",
-            central_path="testGDrive",
+            central_path=central_path,
             gdrive_root_folder_id=root_id,
             gdrive_client_id=None,  # keep None
         )
 
-        # ── 3  feed the SA-file path to the interactive prompt ─────────
         state = {"first": True}
 
         def mock_input(_: str) -> str:
             if state["first"]:
                 state["first"] = False
-                return "n"  # ← tells setup to use file, not auth-browser
-            return sa_path  # ← absolute path written in the workflow
-
-        import builtins
+                return "n"
+            return sa_path
 
         original_input = builtins.input
         builtins.input = mock_input
 
-        try:
-            project.setup_google_drive_connection()
-        finally:
-            builtins.input = original_input  # always restore
+        no_cfg_project.setup_google_drive_connection()
+
+        builtins.input = original_input
 
-        # ── 4  run the usual checks ────────────────────────────────────
         subs, sessions = test_utils.get_default_sub_sessions_to_test()
+
         test_utils.make_and_check_local_project_folders(
-            project, "rawdata", subs, sessions, get_broad_datatypes()
+            no_cfg_project, "rawdata", subs, sessions, get_broad_datatypes()
         )
 
-        project.upload_entire_project()
+        no_cfg_project.upload_entire_project()
+
+        # get a list of files on gdrive and check they are as expected
+        # assert the test id if its failed
+
+        # only tidy up if as expected, otherwise we can leave the folder there to have a look
+        # and delete manually later
+        rclone.call_rclone(
+            f"purge central_{no_cfg_project.project_name}_gdrive:{central_path}"
+        )