API Generation (#1608)

* API Gen Code

* Generated API

* Generate API

* Update API Gen by removing the init string

* Exclude Sentence piece tokenizer for Keras 2

* Exclude docstring test from Keras 2

Author: sampathweb

.github/workflows/actions.yml

@@ -104,3 +104,12 @@ jobs:
           pip install --no-deps -e "." --progress-bar off
       - name: Lint
         run: bash shell/lint.sh
+      - name: Check for API changes
+        run: |
+          bash shell/api_gen.sh
+          git status
+          clean=$(git status | grep "nothing to commit")
+          if [ -z "$clean" ]; then
+            echo "Please run shell/api_gen.sh to generate API."
+            exit 1
+          fi

CONTRIBUTING.md

@@ -117,13 +117,13 @@ has cuda support.
 conda create -y -n keras-nlp-cpu python=3.10
 conda activate keras-nlp-cpu
 pip install -r requirements.txt  # install deps
-python pip_build.py --install  # install keras-nlp
+pip install -e .  # install keras-nlp
 
 for backend in "jax" "torch" "tensorflow"; do
     conda create -y -n keras-nlp-${backend} python=3.10
     conda activate keras-nlp-${backend}
     pip install -r requirements-${backend}-cuda.txt  # install deps
-    python pip_build.py --install  # install keras-nlp
+    pip install -e .  # install keras-nlp
 done
 ```
 
@@ -158,6 +158,15 @@ This is automatically done if you clone using git inside WSL.
 Note that will not support Windows Shell/PowerShell for any scripts in this
 repository.
 
+## Update Public API
+
+Run API generation script when creating PRs that update `keras_nlp_export`
+public APIs. Add the files changed in `keras_nlp/api` to the same PR.
+
+```
+./shell/api_gen.sh
+```
+
 ## Testing changes
 
 KerasNLP is tested using [PyTest](https://docs.pytest.org/en/6.2.x/).

api_gen.py

@@ -0,0 +1,86 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Script to generate keras_nlp public API in `keras_nlp/api` directory.
+
+Usage:
+
+Run via `./shell/api_gen.sh`.
+It generates API and formats user and generated APIs.
+"""
+
+import os
+import shutil
+
+import namex
+
+package = "keras_nlp"
+
+
+def ignore_files(_, filenames):
+    return [f for f in filenames if f.endswith("_test.py")]
+
+
+def copy_source_to_build_directory(root_path):
+    # Copy sources (`keras_nlp/` directory and setup files) to build dir
+    build_dir = os.path.join(root_path, "tmp_build_dir")
+    if os.path.exists(build_dir):
+        shutil.rmtree(build_dir)
+    os.mkdir(build_dir)
+    shutil.copytree(
+        package, os.path.join(build_dir, package), ignore=ignore_files
+    )
+    return build_dir
+
+
+def export_version_string(api_init_fname):
+    with open(api_init_fname) as f:
+        contents = f.read()
+    with open(api_init_fname, "w") as f:
+        contents += "from keras_nlp.src.version_utils import __version__\n"
+        f.write(contents)
+
+
+def build():
+    # Backup the `keras_nlp/__init__.py` and restore it on error in api gen.
+    root_path = os.path.dirname(os.path.abspath(__file__))
+    code_api_dir = os.path.join(root_path, package, "api")
+    # Create temp build dir
+    build_dir = copy_source_to_build_directory(root_path)
+    build_api_dir = os.path.join(build_dir, package, "api")
+    build_init_fname = os.path.join(build_dir, package, "__init__.py")
+    build_api_init_fname = os.path.join(build_api_dir, "__init__.py")
+    try:
+        os.chdir(build_dir)
+        # Generates `keras_nlp/api` directory.
+        if os.path.exists(build_api_dir):
+            shutil.rmtree(build_api_dir)
+        if os.path.exists(build_init_fname):
+            os.remove(build_init_fname)
+        os.makedirs(build_api_dir)
+        namex.generate_api_files(
+            "keras_nlp", code_directory="src", target_directory="api"
+        )
+        # Add __version__ to keras package
+        export_version_string(build_api_init_fname)
+        # Copy back the keras_nlp/api and keras_nlp/__init__.py from build dir
+        if os.path.exists(code_api_dir):
+            shutil.rmtree(code_api_dir)
+        shutil.copytree(build_api_dir, code_api_dir)
+    finally:
+        # Clean up: remove the build directory (no longer needed)
+        shutil.rmtree(build_dir)
+
+
+if __name__ == "__main__":
+    build()

keras_nlp/conftest.py renamed to conftest.py

@@ -24,8 +24,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import config as backend_config
-from keras_nlp.backend import keras
+from keras_nlp.src.backend import config as backend_config
+from keras_nlp.src.backend import keras
 
 
 def pytest_addoption(parser):

keras_nlp/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2021 The KerasNLP Authors
+# Copyright 2023 The KerasNLP Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,21 +11,36 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""DO NOT EDIT.
 
-# sentencepiece is segfaulting on tf-nightly if tensorflow is imported first.
-# This is a temporary fix to restore our nightly testing to green, while we look
-# for a longer term solution.
-try:
-    import sentencepiece
-except ImportError:
-    pass
-
-from keras_nlp import layers
-from keras_nlp import metrics
-from keras_nlp import models
-from keras_nlp import samplers
-from keras_nlp import tokenizers
-from keras_nlp import utils
-from keras_nlp.utils.preset_utils import upload_preset
-from keras_nlp.version_utils import __version__
-from keras_nlp.version_utils import version
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+import os
+
+# Import everything from /api/ into keras.
+from keras_nlp.api import *  # noqa: F403
+from keras_nlp.api import __version__  # Import * ignores names start with "_".
+
+# Add everything in /api/ to the module search path.
+__path__.append(os.path.join(os.path.dirname(__file__), "api"))  # noqa: F405
+
+# Don't pollute namespace.
+del os
+
+
+# Never autocomplete `.src` or `.api` on an imported keras object.
+def __dir__():
+    keys = dict.fromkeys((globals().keys()))
+    keys.pop("src")
+    keys.pop("api")
+    return list(keys)
+
+
+# Don't import `.src` or `.api` during `from keras import *`.
+__all__ = [
+    name
+    for name in globals().keys()
+    if not (name.startswith("_") or name in ("src", "api"))
+]

keras_nlp/api/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.api import layers
+from keras_nlp.api import metrics
+from keras_nlp.api import models
+from keras_nlp.api import samplers
+from keras_nlp.api import tokenizers
+from keras_nlp.src.utils.preset_utils import upload_preset
+from keras_nlp.src.version_utils import __version__
+from keras_nlp.src.version_utils import version

keras_nlp/api/layers/__init__.py

@@ -0,0 +1,47 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.src.layers.modeling.alibi_bias import AlibiBias
+from keras_nlp.src.layers.modeling.cached_multi_head_attention import (
+    CachedMultiHeadAttention,
+)
+from keras_nlp.src.layers.modeling.f_net_encoder import FNetEncoder
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.layers.modeling.sine_position_encoding import (
+    SinePositionEncoding,
+)
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
+    TokenAndPositionEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
+    MaskedLMMaskGenerator,
+)
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
+    MultiSegmentPacker,
+)
+from keras_nlp.src.layers.preprocessing.random_deletion import RandomDeletion
+from keras_nlp.src.layers.preprocessing.random_swap import RandomSwap
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker

keras_nlp/api/metrics/__init__.py

@@ -0,0 +1,24 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.src.metrics.bleu import Bleu
+from keras_nlp.src.metrics.edit_distance import EditDistance
+from keras_nlp.src.metrics.perplexity import Perplexity
+from keras_nlp.src.metrics.rouge_l import RougeL
+from keras_nlp.src.metrics.rouge_n import RougeN