Make portalocker optional dependency (#1007)

Summary:
Fixes #940 #978

## Changes:
- Make portalocker optional
- Add `workflow_dispatch` to `pull_release.yml` to enable manual triggering on nightly release testing.
- Remove unused `test/requirements.yml`

Pull Request resolved: #1007

Reviewed By: wenleix, NivekT

Differential Revision: D43238384

Pulled By: ejguan

fbshipit-source-id: b23bdd558bb2d97ba90c3297ac7d93ea8adafe5f

Author: ejguan

.github/workflows/pull_release.yml

@@ -1,6 +1,7 @@
 name: Test Release Pipelines
 
 on:
+  workflow_dispatch:
   pull_request:
     paths:
       - .github/workflows/pull_release.yml

mypy.ini

@@ -57,6 +57,9 @@ ignore_missing_imports = True
 [mypy-pandas.*]
 ignore_missing_imports = True
 
+[mypy-portalocker.*]
+ignore_missing_imports = True
+
 [mypy-psutil.*]
 ignore_missing_imports = True
 

packaging/torchdata/meta.yaml

@@ -24,7 +24,6 @@ requirements:
     - python
     - urllib3>=1.25
     - requests
-    - portalocker>=2.0.0
     {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }}
 
 build:

requirements.txt

@@ -1,3 +1,2 @@
 urllib3 >= 1.25
 requests
-portalocker >= 2.0.0

test/requirements.txt

@@ -5,6 +5,7 @@ s3fs
 iopath == 0.1.9
 numpy
 rarfile
+portalocker >= 2.0.0
 # Protobuf 4.0 is binary incompatible with what C++ TF uses.
 # See: https://github.com/tensorflow/tensorflow/blob/8dcaf6b98a6a49c85eb470140ba8506e71a3b5af/tensorflow/tools/pip_package/setup.py#L88-L94
 # Protobuf 3.20.2 is also broken on MacOS Python 3.10

test/requirements.yml

@@ -74,6 +74,14 @@
     HAS_RAR_TOOLS = False
 skipIfNoRarTools = unittest.skipIf(not HAS_RAR_TOOLS, "no rar tools")
 
+try:
+    import portalocker
+
+    HAS_PORTALOCKER = True
+except ImportError:
+    HAS_PORTALOCKER = False
+skipIfNoPortalocker = unittest.skipIf(not HAS_PORTALOCKER, "No portalocker installed")
+
 
 def filepath_fn(temp_dir_name, name: str) -> str:
     return os.path.join(temp_dir_name, os.path.basename(name))
@@ -645,6 +653,7 @@ def _slow_fn(tmpdirname, x):
         time.sleep(10)
         return (x, "str")
 
+    @skipIfNoPortalocker
     def test_disk_cache_locks(self):
         with tempfile.TemporaryDirectory() as tmpdirname:
             file_name = os.path.join(tmpdirname, "test.bin")

test/test_local_io.py

@@ -60,6 +60,14 @@
 skipIfAWS = unittest.skipIf(HAS_AWS, "AWSSDK Enabled")
 skipIfNoAWS = unittest.skipIf(not HAS_AWS, "No AWSSDK Enabled")
 
+try:
+    import portalocker
+
+    HAS_PORTALOCKER = True
+except ImportError:
+    HAS_PORTALOCKER = False
+skipIfNoPortalocker = unittest.skipIf(not HAS_PORTALOCKER, "No portalocker installed")
+
 
 class TestDataPipeRemoteIO(expecttest.TestCase):
     def setUp(self):
@@ -125,6 +133,7 @@ def test_http_reader_iterdatapipe(self):
                 allow_redirects=query_params["allow_redirects"],
             )
 
+    @skipIfNoPortalocker
     def test_on_disk_cache_holder_iterdatapipe(self):
         tar_file_url = "https://raw.githubusercontent.com/pytorch/data/main/test/_fakedata/csv.tar.gz"
         expected_file_name = os.path.join(self.temp_dir.name, "csv.tar.gz")

test/test_remote_io.py

@@ -18,14 +18,8 @@
 
 try:
     import portalocker
-except ImportError as e:
-    if os.name == "nt" and str(e).startswith("DLL load failed while importing"):
-        print(
-            "Please take a look at FAQ in https://github.com/pytorch/data#frequently-asked-questions-faq"
-            "for the solution of this Error."
-        )
-    raise
-
+except ImportError:
+    protalocker = None
 
 from torch.utils.data.datapipes.utils.common import _check_unpickable_fn, DILL_AVAILABLE
 
@@ -38,6 +32,26 @@
 
     dill.extend(use_dill=False)
 
+
+def _assert_portalocker() -> None:
+    try:
+        import portalocker  # noqa: F401
+    except ImportError as e:
+        if os.name == "nt" and str(e).startswith("DLL load failed while importing"):
+            print(
+                "Please take a look at FAQ in https://github.com/pytorch/data#frequently-asked-questions-faq"
+                "for the solution of this Error."
+            )
+            raise
+        else:
+            raise ModuleNotFoundError(
+                "Package `portalocker` is required to be installed to use this datapipe."
+                "Please use `pip install 'portalocker>=2.0.0'` or"
+                "`conda install -c conda-forge 'portalocker>=2/0.0'`"
+                "to install the package"
+            )
+
+
 T_co = TypeVar("T_co", covariant=True)
 
 PROMISE_FILE_DELETE_TIMEOUT = 30
@@ -190,6 +204,8 @@ def __init__(
         hash_type: str = "sha256",
         extra_check_fn: Optional[Callable[[str], bool]] = None,
     ):
+        _assert_portalocker()
+
         self.source_datapipe = source_datapipe
 
         if filepath_fn is not None: