Merging master with mlx (#21152)

Author: fbadine

.github/workflows/actions.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: [3.10]
         backend: [tensorflow, jax, torch, numpy, openvino, mlx]
     name: Run tests
     runs-on: ubuntu-latest
@@ -116,10 +116,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.9
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -136,23 +136,7 @@ jobs:
           pip install -r requirements-openvino.txt --progress-bar off --upgrade
           pip uninstall -y keras keras-nightly
           pip install -e "." --progress-bar off --upgrade
-      - name: Check for API changes
-        run: |
-          bash shell/api_gen.sh
-          git status
-          clean=$(git status | grep "nothing to commit")
-          if [ -z "$clean" ]; then
-            echo "Please run shell/api_gen.sh to generate API."
-            exit 1
-          fi
-      - name: Lint
-        run: bash shell/lint.sh
-      - name: Check for API changes
-        run: |
-          bash shell/api_gen.sh
-          git status
-          clean=$(git status | grep "nothing to commit")
-          if [ -z "$clean" ]; then
-            echo "Please run shell/api_gen.sh to generate API."
-            exit 1
-          fi
+      - name: Install pre-commit
+        run: pip install pre-commit && pre-commit install
+      - name: Run pre-commit
+        run: pre-commit run --all-files --hook-stage manual

.github/workflows/nightly.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: ['3.10']
         backend: [tensorflow, jax, torch, numpy]
     name: Run tests
     runs-on: ubuntu-latest
@@ -62,10 +62,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.9
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.10'
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -113,7 +113,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: '3.10'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools

.github/workflows/scorecard.yml

@@ -30,7 +30,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
         with:
           results_file: results.sarif
           results_format: sarif
@@ -48,14 +48,14 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
+        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
         with:
           sarif_file: results.sarif

.kokoro/github/ubuntu/gpu/build.sh

@@ -3,9 +3,9 @@ set -x
 
 cd "${KOKORO_ROOT}/"
 
-sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
+sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1
 
-PYTHON_BINARY="/usr/bin/python3.9"
+PYTHON_BINARY="/usr/bin/python3.10"
 
 "${PYTHON_BINARY}" -m venv venv
 source venv/bin/activate

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+repos:
+  - repo: local
+    hooks:
+      - id: api-gen
+        name: api_gen
+        entry: |
+          bash shell/api_gen.sh
+          git status
+          clean=$(git status | grep "nothing to commit")
+          if [ -z "$clean" ]; then
+            echo "Please run shell/api_gen.sh to generate API."
+            exit 1
+          fi
+        language: system
+        stages: [pre-commit, manual]
+        require_serial: true
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.2
+    hooks:
+      - id: ruff
+        args: [--config, pyproject.toml, --fix, .]
+        stages: [pre-commit]
+      - id: ruff-format
+        args: [--config, pyproject.toml, .]
+        stages: [pre-commit]
+      - id: ruff
+        args: [--config, pyproject.toml, .]
+        stages: [manual]
+      - id: ruff-format
+        args: ["--check", --config, pyproject.toml, .]
+        stages: [manual]

CONTRIBUTING.md

@@ -107,22 +107,35 @@ You can also add GPU support to your environment, see the
 [Adding GPU support](https://github.com/keras-team/keras/blob/master/README.md#adding-gpu-support)
 section of the README.
 
-## Code style
+## Generating public API and formatting the code
 
-Keras uses [Ruff](https://docs.astral.sh/ruff/) to format the code. Please refer to
-[requirements-common.txt](https://github.com/keras-team/keras/blob/master/requirements-common.txt)
-for the required versions. Run the following command **at the root directory of
-the repo** to format your code.
+For the first time you are setting up the repo, please run `pre-commit install`.
+Note that this needs to be done only once at the beginning.
+
+Now, whenever you run `git commit -m "<message>"`, three things are
+automatically done:
+
+- Public API generation
+- Code formatting
+- Code linting
+
+If there's any error, the commit will not go through. Please fix the error (
+most of the times, the error is fixed automatically by the formatter/linter) and
+re-run the following:
 
 ```
-sh shell/format.sh
+git add .
+git commit -m "<message>" # This will not get logged as a duplicate commit.
 ```
 
-It will also display the errors that cannot be resolved by autoformatting. You
-need to follow the output of the command to resolve them manually.
+In case you want to run the above manually on all files, you can do the
+following:
+
+```
+pre-commit run --all-files
+```
 
-If you do not want to auto format the code but only show the lint errors, you
-can run `sh shell/lint.sh` **at the root directory of the repo**.
+KerasHub uses [Ruff](https://docs.astral.sh/ruff/) to format the code.
 
 ### Docstrings
 

conftest.py

@@ -25,13 +25,36 @@ def pytest_configure(config):
 
 
 def pytest_collection_modifyitems(config, items):
+    openvino_skipped_tests = []
+    if backend() == "openvino":
+        with open(
+            "keras/src/backend/openvino/excluded_concrete_tests.txt", "r"
+        ) as file:
+            openvino_skipped_tests = file.readlines()
+            # it is necessary to check if stripped line is not empty
+            # and exclude such lines
+            openvino_skipped_tests = [
+                line.strip() for line in openvino_skipped_tests if line.strip()
+            ]
+
     requires_trainable_backend = pytest.mark.skipif(
         backend() == "numpy" or backend() == "openvino",
         reason="Trainer not implemented for NumPy and OpenVINO backend.",
     )
     for item in items:
         if "requires_trainable_backend" in item.keywords:
             item.add_marker(requires_trainable_backend)
+        # also, skip concrete tests for openvino, listed in the special file
+        # this is more granular mechanism to exclude tests rather
+        # than using --ignore option
+        for skipped_test in openvino_skipped_tests:
+            if skipped_test in item.nodeid:
+                item.add_marker(
+                    skip_if_backend(
+                        "openvino",
+                        "Not supported operation by openvino backend",
+                    )
+                )
 
 
 def skip_if_backend(given_backend, reason):

integration_tests/import_test.py

@@ -8,7 +8,7 @@
 BACKEND_REQ = {
     "tensorflow": ("tensorflow-cpu", ""),
     "torch": (
-        "torch torchvision",
+        "torch",
         "--extra-index-url https://download.pytorch.org/whl/cpu ",
     ),
     "jax": ("jax[cpu]", ""),
@@ -29,11 +29,12 @@ def setup_package():
     whl_path = re.findall(
         r"[^\s]*\.whl",
         build_process.stdout,
-    )[-1]
+    )
     if not whl_path:
+        print(build_process.stdout)
         print(build_process.stderr)
         raise ValueError("Installing Keras package unsuccessful. ")
-    return whl_path
+    return whl_path[-1]
 
 
 def create_virtualenv():

keras/__init__.py

@@ -14,6 +14,7 @@
 from keras.api import Optimizer
 from keras.api import Quantizer
 from keras.api import Regularizer
+from keras.api import RematScope
 from keras.api import Sequential
 from keras.api import StatelessScope
 from keras.api import SymbolicScope
@@ -44,6 +45,7 @@
 from keras.api import quantizers
 from keras.api import random
 from keras.api import regularizers
+from keras.api import remat
 from keras.api import saving
 from keras.api import tree
 from keras.api import utils

keras/api/__init__.py

@@ -37,6 +37,8 @@
 from keras.src.backend import device
 from keras.src.backend import name_scope
 from keras.src.backend.common.keras_tensor import KerasTensor
+from keras.src.backend.common.remat import RematScope
+from keras.src.backend.common.remat import remat
 from keras.src.backend.common.stateless_scope import StatelessScope
 from keras.src.backend.common.symbolic_scope import SymbolicScope
 from keras.src.dtype_policies.dtype_policy import DTypePolicy