Merge branch 'master' into feat/dynamo_export_onnx

Author: Borda

.github/checkgroup.yml

@@ -176,14 +176,15 @@ subprojects:
       - "lightning-fabric (GPUs) (testing Fabric | latest)"
       - "lightning-fabric (GPUs) (testing Lightning | latest)"
 
-  - id: "lightning_fabric: TPU workflow"
-    paths:
-      # tpu CI availability is very limited, so we only require tpu tests
-      # to pass when their configurations are modified
-      - ".github/workflows/tpu-tests.yml"
-      - "tests/tests_fabric/run_tpu_tests.sh"
-    checks:
-      - "test-on-tpus (pytorch, pjrt, v4-8)"
+  # Temporarily disabled
+  #  - id: "lightning_fabric: TPU workflow"
+  #    paths:
+  #      # tpu CI availability is very limited, so we only require tpu tests
+  #      # to pass when their configurations are modified
+  #      - ".github/workflows/tpu-tests.yml"
+  #      - "tests/tests_fabric/run_tpu_tests.sh"
+  #    checks:
+  #      - "test-on-tpus (pytorch, pjrt, v4-8)"
 
   # SECTION: common
 

.github/workflows/_legacy-checkpoints.yml

@@ -111,9 +111,13 @@ jobs:
           retention-days: ${{ env.KEEP_DAYS }}
           include-hidden-files: true
 
+      - name: access secrets
+        # export to env bool if secrets.AWS_REGION is not empty
+        run: echo "WITH_SECRETS=$([ -n '${{ secrets.AWS_REGION }}' ] && echo 1 || echo 0)" >> $GITHUB_ENV
+
       - run: pip install -r requirements/ci.txt
       - name: Upload checkpoints to S3
-        if: ${{ secrets[AWS_REGION] != '' }}
+        if: ${{ env.WITH_SECRETS == '1' }}
         working-directory: ${{ env.LEGACY_FOLDER }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}

.github/workflows/ci-checkpoints.yml.disabled renamed to .github/workflows/ci-checkpoints.yml

@@ -116,16 +116,24 @@ jobs:
           mkdir -p $PYPI_CACHE_DIR
           ls -lh $PYPI_CACHE_DIR
 
-      - name: Env. variables
+      - name: Expand Env. variables
         run: |
           # Switch PyTorch URL between stable and test/future
           python -c "print('TORCH_URL=' + str('${{env.TORCH_URL_TEST}}' if '${{ matrix.pytorch-version }}' == '2.7' else '${{env.TORCH_URL_STABLE}}'))" >> $GITHUB_ENV
           # Switch coverage scope
           python -c "print('COVERAGE_SCOPE=' + str('lightning' if '${{matrix.pkg-name}}' == 'lightning' else 'lightning_fabric'))" >> $GITHUB_ENV
           # if you install mono-package set dependency only for this subpackage
           python -c "print('EXTRA_PREFIX=' + str('' if '${{matrix.pkg-name}}' != 'lightning' else 'fabric-'))" >> $GITHUB_ENV
+      - name: Append Env. vars for MacOS
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          # trying to avoid "gloo" issue with SIGABRT
+          echo "GLOO_SOCKET_IFNAME=lo0" >> $GITHUB_ENV
+      - name: Append Env. vars for Windows
+        if: ${{ runner.os == 'windows' }}
+        run: |
           # Avoid issue on Windows with PyTorch 2.4: "RuntimeError: use_libuv was requested but PyTorch was build without libuv support"
-          python -c "print('USE_LIBUV=0' if '${{matrix.os}}' == 'windows-2022' and '${{matrix.pytorch-version}}' == '2.4' else '')" >> $GITHUB_ENV
+          echo "USE_LIBUV=0" >> $GITHUB_ENV
 
       - name: Install package & dependencies
         timeout-minutes: 20

.github/workflows/ci-tests-fabric.yml

@@ -179,12 +179,12 @@ jobs:
         with:
           pkg-folder: dist/${{ steps.folder.outputs.pkg }}
           pypi-token: ${{ secrets[format('PYPI_TOKEN_{0}', matrix.name)] }}
-# FIXME: this is not working suddenly, Unrecognized named-value: 'secrets'
-#  legacy-checkpoints:
-#    needs: [build-packages]
-#    uses: ./.github/workflows/_legacy-checkpoints.yml
-#    with:
-#      push_to_s3: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
-#      upload_local: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
-#      create_pr: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
-#    secrets: inherit
+
+  legacy-checkpoints:
+    needs: [build-packages]
+    uses: ./.github/workflows/_legacy-checkpoints.yml
+    with:
+      push_to_s3: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
+      upload_local: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
+      create_pr: ${{ startsWith(github.event.ref, 'refs/tags') || github.event_name == 'release' }}
+    secrets: inherit

.github/workflows/release-pkg.yml

@@ -5,7 +5,7 @@
 matplotlib>3.1, <3.10.0
 omegaconf >=2.2.3, <2.4.0
 hydra-core >=1.2.0, <1.4.0
-jsonargparse[signatures] >=4.39.0, <4.41.0
+jsonargparse[signatures,jsonnet] >=4.39.0, <4.41.0
 rich >=12.3.0, <14.1.0
 tensorboardX >=2.2, <2.7.0  # min version is set by torch.onnx missing attribute
 bitsandbytes >=0.45.2,<0.47.0; platform_system != "Darwin"

.github/workflows/tpu-tests.yml.disabled renamed to .github/workflows/tpu-tests.yml

@@ -554,7 +554,9 @@ def parse_arguments(self, parser: LightningArgumentParser, args: ArgsType) -> No
     def _dump_config(self) -> None:
         if hasattr(self, "config_dump"):
             return
-        self.config_dump = yaml.safe_load(self.parser.dump(self.config, skip_link_targets=False, skip_none=False))
+        self.config_dump = yaml.safe_load(
+            self.parser.dump(self.config, skip_link_targets=False, skip_none=False, format="yaml")
+        )
         if "subcommand" in self.config:
             self.config_dump = self.config_dump[self.config.subcommand]
 

requirements/pytorch/extra.txt

@@ -1858,3 +1858,23 @@ def test_lightning_cli_with_args_given(args):
 def test_lightning_cli_args_and_sys_argv_warning():
     with mock.patch("sys.argv", ["", "--model.foo=456"]), pytest.warns(Warning, match="LightningCLI's args parameter "):
         LightningCLI(TestModel, run=False, args=["--model.foo=789"])
+
+
+def test_lightning_cli_jsonnet(cleandir):
+    class MainModule(BoringModel):
+        def __init__(self, main_param: int = 1):
+            super().__init__()
+
+    config = """{
+        "model":{
+            "main_param": 2
+        }
+    }"""
+    config_path = Path("config.jsonnet")
+    config_path.write_text(config)
+
+    cli_args = [f"--config={config_path}"]
+    with mock.patch("sys.argv", ["any.py"] + cli_args):
+        cli = LightningCLI(MainModule, run=False, parser_kwargs={"parser_mode": "jsonnet"})
+
+    assert cli.config["model"]["main_param"] == 2