Make Globus dependencies optional in test suite (#3980)

This PR makes Globus dependencies more optional:

In regular install, globus-sdk used by the very abandoned and probably
untested GlobusStaging provider is now no-longer installed, which
reduces the dependency footprint some.

In test installs, Globus Compute tests can be picked out or skipped
using `-k globus_compute`. This follows the style of `-k workqueue` and
`-k taskvine` used to allow Work Queue and Taskvine to not be installed.

As a consequence, the main CI workflow no longer runs the Globus Compute
site tests. This PR adds a `-k globus_compute` local test to the
Globus-specific workflow, again in the style of the Work Queue and
Taskvine tests.

The GlobusStaging uses the older "not configured implies skip implies
pass in CI" model of test configuration which I find personally quite
distasteful; but I have not peturbed that.

Some GlobusStaging related code now imports globus_sdk later on, rather
than at the top of modules, so that it will be importable even without
globus_sdk. This is because we can still generate documentation without
globus_sdk installed.

Relevant issues:

Debian bug #1116867
https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1116867#30 where
@emollier comments on some Globus related packaging impeding the Debian
packaging of Parsl.

Parsl issue #2218 

# Changed Behaviour

Dependencies will be loosened both in user installs and test environment
installs - this might reveal interesting package incompatibilities
although I am not aware of any.

Import-related failures in GlobusStaging will no longer appear when
importing GlobusStaging, but instead will appear when that provider
tries to use globus_sdk to login/perform a transfer.

## Type of change

- Code maintenance/cleanup

Author: benclifford

Makefile

@@ -51,6 +51,7 @@ mypy: ## run mypy checks
 .PHONY: gce_test
 gce_test: ## Run tests with GlobusComputeExecutor (--config .../globus_compute.py)
 	pytest -v -k "not shared_fs and not issue_3620 and not staging_required" --config parsl/tests/configs/globus_compute.py parsl/tests/ --random-order --durations 10
+	pytest -v -k "not shared_fs and not issue_3620 and not staging_required and globus_compute" --config local --random-order --durations 10
 
 .PHONY: local_thread_test
 local_thread_test: ## run all tests with local_thread config (--config .../local_threads.py)
@@ -86,7 +87,7 @@ radical_local_test:  ## Run the Radical local tests (-m radical --config local)
 .PHONY: config_local_test
 config_local_test:  ## run the config-local tests (--config local)
 	pip3 install ".[monitoring,visualization,proxystore,kubernetes]"
-	pytest parsl/tests/ -k "not cleannet and not workqueue and not taskvine" --config local --random-order --durations 10
+	pytest parsl/tests/ -k "not cleannet and not workqueue and not taskvine and not globus_compute" --config local --random-order --durations 10
 
 .PHONY: site_test
 site_test:  ## Run the site tests

parsl/data_provider/globus.py

@@ -4,7 +4,6 @@
 from functools import partial
 from typing import Optional
 
-import globus_sdk
 import typeguard
 
 import parsl
@@ -79,6 +78,7 @@ def get_authorizer(cls):
 
     @classmethod
     def transfer_file(cls, src_ep, dst_ep, src_path, dst_path):
+        import globus_sdk
         tc = globus_sdk.TransferClient(authorizer=cls.authorizer)
         td = globus_sdk.TransferData(tc, src_ep, dst_ep)
         td.add_item(src_path, dst_path)
@@ -140,6 +140,7 @@ def _update_tokens_file_on_refresh(cls, token_response):
     def _do_native_app_authentication(cls, client_id, redirect_uri,
                                       requested_scopes=None):
 
+        import globus_sdk
         client = globus_sdk.NativeAppAuthClient(client_id=client_id)
         client.oauth2_start_flow(
             requested_scopes=requested_scopes,
@@ -154,6 +155,7 @@ def _do_native_app_authentication(cls, client_id, redirect_uri,
 
     @classmethod
     def _get_native_app_authorizer(cls, client_id):
+        import globus_sdk
         tokens = None
         try:
             tokens = cls._load_tokens_from_file(cls.TOKEN_FILE)

parsl/tests/configs/local_threads_globus.py

@@ -1,6 +1,5 @@
 from parsl.config import Config
 from parsl.data_provider.data_manager import default_staging
-from parsl.data_provider.globus import GlobusStaging
 from parsl.executors.threads import ThreadPoolExecutor
 
 # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
@@ -10,19 +9,24 @@
 #          (i.e., user_opts['swan']['username'] -> 'your_username')
 from .user_opts import user_opts
 
-storage_access = default_staging + [GlobusStaging(
-                endpoint_uuid=user_opts['globus']['endpoint'],
-                endpoint_path=user_opts['globus']['path']
-            )]
 
-config = Config(
-    executors=[
-        ThreadPoolExecutor(
-            label='local_threads_globus',
-            working_dir=user_opts['globus']['path'],
-            storage_access=storage_access
-        )
-    ]
-)
+def fresh_config():
+    from parsl.data_provider.globus import GlobusStaging
+
+    storage_access = default_staging + [GlobusStaging(
+                    endpoint_uuid=user_opts['globus']['endpoint'],
+                    endpoint_path=user_opts['globus']['path']
+                )]
+
+    return Config(
+        executors=[
+            ThreadPoolExecutor(
+                label='local_threads_globus',
+                working_dir=user_opts['globus']['path'],
+                storage_access=storage_access
+            )
+        ]
+    )
+
 
 remote_writeable = user_opts['globus']['remote_writeable']

parsl/tests/test_python_apps/test_basic.py

@@ -14,12 +14,6 @@ def import_square(x):
     return math.pow(x, 2)
 
 
-@python_app
-def custom_exception():
-    from globus_sdk import GlobusError
-    raise GlobusError('foobar')
-
-
 def test_simple(n=2):
     x = double(n)
     assert x.result() == n * 2
@@ -38,11 +32,3 @@ def test_parallel_for(n):
 
     for i in d:
         assert d[i].result() == 2 * i
-
-
-def test_custom_exception():
-    from globus_sdk import GlobusError
-
-    x = custom_exception()
-    with pytest.raises(GlobusError):
-        x.result()

parsl/tests/test_python_apps/test_exception.py

@@ -0,0 +1,19 @@
+import pytest
+
+from parsl.app.app import python_app
+
+
+class CustomException(Exception):
+    pass
+
+
+@python_app
+def custom_exception():
+    from parsl.tests.test_python_apps.test_exception import CustomException
+    raise CustomException('foobar')
+
+
+def test_custom_exception():
+    x = custom_exception()
+    with pytest.raises(CustomException):
+        x.result()

parsl/tests/test_staging/test_staging_globus.py

@@ -3,9 +3,9 @@
 import parsl
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
-from parsl.tests.configs.local_threads_globus import config, remote_writeable
+from parsl.tests.configs.local_threads_globus import fresh_config, remote_writeable
 
-local_config = config
+local_config = fresh_config
 
 
 @python_app

parsl/tests/unit/test_globus_compute_executor.py

@@ -2,38 +2,44 @@
 from unittest import mock
 
 import pytest
-from globus_compute_sdk import Executor
 
 from parsl.executors import GlobusComputeExecutor
 
 
 @pytest.fixture
 def mock_ex():
-    # Not Parsl's job to test GC's Executor
+    # Not Parsl's job to test GC's Executor, although it
+    # still needs to be importable for these test cases.
+    from globus_compute_sdk import Executor
+
     yield mock.Mock(spec=Executor)
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_mock_spec(mock_ex):
     # a test of tests -- make sure we're using spec= in the mock
     with pytest.raises(AttributeError):
         mock_ex.aasdf()
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_label_default(mock_ex):
     gce = GlobusComputeExecutor(mock_ex)
     assert gce.label == type(gce).__name__, "Expect reasonable default label"
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_label(mock_ex, randomstring):
     exp_label = randomstring()
     gce = GlobusComputeExecutor(mock_ex, label=exp_label)
     assert gce.label == exp_label
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_resets_spec_after_submit(mock_ex, randomstring):
     submit_res = {randomstring(): "some submit res"}
     res = {"some": randomstring(), "spec": randomstring()}
@@ -57,6 +63,7 @@ def mock_submit(*a, **k):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_resets_uep_after_submit(mock_ex, randomstring):
     uep_conf = randomstring()
     res = {"some": randomstring()}
@@ -79,6 +86,7 @@ def mock_submit(*a, **k):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_happy_path(mock_ex, randomstring):
     mock_fn = mock.Mock()
     args = tuple(randomstring() for _ in range(random.randint(0, 3)))
@@ -95,6 +103,7 @@ def test_gc_executor_happy_path(mock_ex, randomstring):
 
 
 @pytest.mark.local
+@pytest.mark.globus_compute
 def test_gc_executor_shuts_down_asynchronously(mock_ex):
     gce = GlobusComputeExecutor(mock_ex)
     gce.shutdown()

requirements.txt

@@ -8,7 +8,6 @@ typeguard>=2.10,!=3.*,<5
 
 
 typing-extensions>=4.6,<5
-globus-sdk
 dill
 tblib
 requests

setup.py

@@ -46,6 +46,7 @@
     'proxystore': ['proxystore'],
     'radical-pilot': ['radical.pilot==1.90', 'radical.utils==1.90'],
     'globus_compute': ['globus_compute_sdk>=2.34.0'],
+    'globus_transfer': ['globus-sdk'],
     # Disabling psi-j since github direct links are not allowed by pypi
     # 'psij': ['psi-j-parsl@git+https://github.com/ExaWorks/psi-j-parsl']
 }

test-requirements.txt

@@ -10,7 +10,6 @@ types-mock
 types-python-dateutil
 types-requests
 mpi4py
-globus-compute-sdk>=2.34.0
 
 # sqlalchemy is needed for typechecking, so it's here
 # as well as at runtime for optional monitoring execution