PLaying around with github secrents and connections.

Author: JoeZiminski

.github/workflows/test_connections.yml

@@ -0,0 +1,25 @@
+# .github/workflows/ci.yml  (excerpt)
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install deps
+        run: pip install -r requirements.txt
+
+      - name: Expose Google-Drive secrets
+        run: |
+          # 1) materialise the JSON file
+          echo "${{ secrets.GDRIVE_SERVICE_ACCOUNT_JSON }}" > $HOME/gdrive.json
+          # 2) make both paths available to the test process
+          echo "GDRIVE_SERVICE_ACCOUNT_FILE=$HOME/gdrive.json" >> $GITHUB_ENV
+          echo "GDRIVE_ROOT_FOLDER_ID=${{ secrets.GDRIVE_ROOT_FOLDER_ID }}" >> $GITHUB_ENV
+      - name: Run pytest
+        env:
+          CI: "true"      # so your skip-marker sees CI=true
+        run: pytest -q -k test_gdrive_connection

datashuttle/datashuttle_class.py

@@ -928,17 +928,20 @@ def setup_google_drive_connection(self) -> None:
         browser_available = gdrive.ask_user_for_browser(log=True)
 
         if not browser_available:
-            config_token = gdrive.prompt_and_get_config_token(
-                self.cfg,
-                gdrive_client_secret,
-                self.cfg.get_rclone_config_name("gdrive"),
-                log=True,
-            )
+            service_account_filepath = utils.get_user_input(
+                "Please input the path to your credentials json"
+            )  # TODO: add more explanation
+        #         config_token = gdrive.prompt_and_get_config_token(
+        #            self.cfg,
+        #           gdrive_client_secret,
+        #          self.cfg.get_rclone_config_name("gdrive"),
+        #         log=True,
+        #    )
         else:
-            config_token = None
+            service_account_filepath = None
 
         self._setup_rclone_gdrive_config(
-            gdrive_client_secret, config_token, log=True
+            gdrive_client_secret, service_account_filepath, log=True
         )
 
         rclone.check_successful_connection_and_raise_error_on_fail(self.cfg)
@@ -1565,14 +1568,14 @@ def _setup_rclone_central_local_filesystem_config(self) -> None:
     def _setup_rclone_gdrive_config(
         self,
         gdrive_client_secret: str | None,
-        config_token: str | None,
+        service_account_filepath: str | None,
         log: bool,
     ) -> None:
         rclone.setup_rclone_config_for_gdrive(
             self.cfg,
             self.cfg.get_rclone_config_name("gdrive"),
             gdrive_client_secret,
-            config_token,
+            service_account_filepath,
             log=log,
         )
 

datashuttle/utils/folders.py

@@ -575,7 +575,7 @@ def search_gdrive_or_aws_for_folders(
 
     if output.returncode != 0:
         utils.log_and_message(
-            f"Error searching files at {search_path.as_posix()} \n {output.stderr.decode('utf-8') if output.stderr else ""}"
+            f"Error searching files at {search_path.as_posix()} \n {output.stderr.decode('utf-8') if output.stderr else ''}"
         )
         return all_folder_names, all_filenames
 

datashuttle/utils/rclone.py

@@ -160,7 +160,7 @@ def setup_rclone_config_for_gdrive(
     cfg: Configs,
     rclone_config_name: str,
     gdrive_client_secret: str | None,
-    config_token: Optional[str] = None,
+    service_account_filepath: Optional[str] = None,
     log: bool = True,
 ):
     """
@@ -197,10 +197,16 @@ def setup_rclone_config_for_gdrive(
         else ""
     )
 
-    extra_args = (
-        f"config_is_local=false config_token={config_token}"
-        if config_token
-        else ""
+    #    extra_args = (
+    #       f"config_is_local=false config_token={config_token}"
+    #      if config_token
+    #     else ""
+    # )
+
+    service_account_filepath_arg = (
+        ""
+        if service_account_filepath is None
+        else f"service_account_file {service_account_filepath} "
     )
     output = call_rclone(
         f"config create "
@@ -210,7 +216,8 @@ def setup_rclone_config_for_gdrive(
         f"{client_secret_key_value}"
         f"scope drive "
         f"root_folder_id {cfg['gdrive_root_folder_id']} "
-        f"{extra_args}",
+        f"{service_account_filepath_arg}",
+        #        f"{extra_args}",
         pipe_std=True,
     )
 

tests/tests_integration/test_gdrive_connection.py

@@ -0,0 +1,53 @@
+import os
+from pathlib import Path
+
+import pytest
+import test_utils
+from base import BaseTest
+
+from datashuttle.configs.canonical_configs import get_broad_datatypes
+
+
+@pytest.mark.skipif(os.getenv("CI") is None, reason="Only runs in CI")
+class TestGoogleDriveGithubCI(BaseTest):
+
+    def test_google_drive_connection(self, project):
+        # ── 1  pull the secrets from env ───────────────────────────────
+        root_id = os.environ["GDRIVE_ROOT_FOLDER_ID"]
+        sa_path = os.environ["GDRIVE_SERVICE_ACCOUNT_FILE"]
+
+        # ── 2  configure the project (no hard-coded ids/paths) ─────────
+        project.update_config_file(
+            local_path=str(Path.home() / "data"),  # any temp location
+            connection_method="gdrive",
+            central_path="testGDrive",
+            gdrive_root_folder_id=root_id,
+            gdrive_client_id=None,  # keep None
+        )
+
+        # ── 3  feed the SA-file path to the interactive prompt ─────────
+        state = {"first": True}
+
+        def mock_input(_: str) -> str:
+            if state["first"]:
+                state["first"] = False
+                return "n"  # ← tells setup to use file, not auth-browser
+            return sa_path  # ← absolute path written in the workflow
+
+        import builtins
+
+        original_input = builtins.input
+        builtins.input = mock_input
+
+        try:
+            project.setup_google_drive_connection()
+        finally:
+            builtins.input = original_input  # always restore
+
+        # ── 4  run the usual checks ────────────────────────────────────
+        subs, sessions = test_utils.get_default_sub_sessions_to_test()
+        test_utils.make_and_check_local_project_folders(
+            project, "rawdata", subs, sessions, get_broad_datatypes()
+        )
+
+        project.upload_entire_project()