[torchx/specs] Use default_factory for the default value of Role.resource and mlflow_test.Config.model_config to support python 3.11 clients (#768)

Author: kiukchung

.github/workflows/python-unittests.yaml

@@ -10,7 +10,7 @@ jobs:
   unittest:
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10']
+        python-version: [3.8, 3.9, "3.10", 3.11]
         platform: ["linux.20_04.4x"]
         include:
           - python-version: 3.9
@@ -32,7 +32,10 @@ jobs:
         run: |
           set -eux
           pip install pytest pytest-cov
-          pip install -e .[dev]
+          # use legacy resolver for python 3.11, otherwise pip will timeout trying to resolve deps
+          # TODO(kiukchung) long term we should narrowly scope dependency versions
+          # see: https://pip.pypa.io/en/latest/topics/dependency-resolution/
+          pip install --use-deprecated=legacy-resolver -e .[dev]
       - name: Run tests
         run: pytest --cov=./ --cov-report=xml
       - name: Upload coverage to Codecov

dev-requirements.txt

@@ -14,6 +14,8 @@ ipython
 kfp==1.8.22
 mlflow-skinny
 moto==4.1.6
+# kfp==1.8.22 needs protobuf < 4
+protobuf==3.20.3
 pyre-extensions
 pyre-check
 pytest

requirements.txt

@@ -4,7 +4,8 @@ importlib-metadata
 pyyaml
 docker
 filelock
-fsspec
+# more recent versions of fsspec causes torchx/workspace/test/dir_workspace_test#test_torchcxignore to fail
+fsspec==2023.1.0
 # To resolve confliciting dependencies for urllib3: https://github.com/pytorch/torchx/actions/runs/3484190429/jobs/5828784263#step:4:552
 urllib3<1.27,>=1.21.1
 tabulate

torchx/specs/api.py

@@ -104,6 +104,13 @@ def copy(original: "Resource", **capabilities: Any) -> "Resource":
 # sentinel value used for cases when resource does not matter (e.g. ignored)
 NULL_RESOURCE: Resource = Resource(cpu=-1, gpu=-1, memMB=-1)
 
+
+# no-arg static factory method to use with default_factory in @dataclass
+# needed to support python 3.11 since mutable defaults for dataclasses are not allowed in 3.11
+def _null_resource() -> Resource:
+    return NULL_RESOURCE
+
+
 # used as "*" scheduler backend
 ALL: str = "all"
 
@@ -333,7 +340,7 @@ class Role:
     num_replicas: int = 1
     max_retries: int = 0
     retry_policy: RetryPolicy = RetryPolicy.APPLICATION
-    resource: Resource = NULL_RESOURCE
+    resource: Resource = field(default_factory=_null_resource)
     port_map: Dict[str, int] = field(default_factory=dict)
     metadata: Dict[str, Any] = field(default_factory=dict)
     mounts: List[Union[BindMount, VolumeMount, DeviceMount]] = field(

torchx/tracker/test/mlflow_test.py

@@ -44,7 +44,7 @@ class Config:
     locales: List[str] = field(default_factory=lambda: ["us", "eu", "fr"])
     empty_list: List[str] = field(default_factory=list)
     empty_map: Dict[str, str] = field(default_factory=dict)
-    model_config: ModelConfig = ModelConfig()
+    model_config: ModelConfig = field(default_factory=ModelConfig)
     datasets: List[DatasetConfig] = field(
         default_factory=lambda: [
             DatasetConfig(url="s3://dataset1"),