PY-314 Code generation for neptune-api (#190)

* neptune-api codegen

* Configure linting for the generated code

* check in neptune-api-spec to the repo

* code review

* Add README files to generated directories

* Remove the source README.md file from docker/rofiles/neptune_api

* Move neptune-api-spec to src/neptune_query/generated

* formatting fix

* Adding src/neptune_api_codegen/README.md

* Exclude generated/neptune_api_spec and generated/README.md from building Python package

Author: gabrys

.gitignore

@@ -136,3 +136,5 @@ poetry.lock
 # pytest-benchmark
 .benchmarks
 benchmark_results.json
+
+/tmp/

.pre-commit-config.yaml

@@ -1,21 +1,32 @@
-exclude: ^src/neptune_query/generated/
+exclude: /generated/neptune_api/proto/|/generated/neptune_api_spec/
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+  - repo: https://github.com/psf/black
+    rev: 25.12.0
     hooks:
-      - id: check-yaml
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
+      - id: black
+        args: [--config, pyproject.toml]
   - repo: https://github.com/pycqa/isort
     rev: 7.0.0
     hooks:
       - id: isort
         args: [--settings-path, pyproject.toml]
-  - repo: https://github.com/psf/black
-    rev: 25.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.9
     hooks:
-      - id: black
-        args: [--config, pyproject.toml]
+      - name: codegen-ruff
+        id: ruff
+        args: [--fix]
+        files: ^src/neptune_query/generated/.*py$
+      - name: codegen-ruff-format
+        id: ruff-format
+        files: ^src/neptune_query/generated/.*py$
+  - repo: https://github.com/pycqa/isort
+    rev: 7.0.0
+    hooks:
+      - name: codegen-isort
+        id: isort
+        args: [--settings-path, pyproject.toml]
+        files: /generated/neptune_api/
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
@@ -27,6 +38,12 @@ repos:
       - id: insert-license
         files: ^src/neptune_query.*[^/]+\.py$
         args: [ "--license-filepath", ".github/license_header.txt", "--allow-past-years"]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.19.1
     hooks:
@@ -35,5 +52,22 @@ repos:
         pass_filenames: false
         additional_dependencies:
           - attrs
+          - httpx
+          - types-PyJWT
+          - grpcio-tools
+          - types-Pygments
+          - types-protobuf
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: [--settings-path, pyproject.toml]
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.4
+    hooks:
+      - id: insert-license
+        files: ^src/neptune_query/generated/.*py$
+        args: [ "--license-filepath", ".github/license_header.txt", "--allow-past-years"]
+
 default_language_version:
     python: python3

pyproject.toml

@@ -56,6 +56,10 @@ keywords = [
     "ML Model Store",
     "ML Metadata Store",
 ]
+exclude = [
+    "src/neptune_query/generated/neptune_api_spec/**",
+    "src/neptune_query/generated/README.md",
+]
 packages = [
     { include = "neptune_query", from = "src" },
 ]
@@ -81,12 +85,19 @@ exclude = '''
 
 [tool.isort]
 profile = "black"
+split_on_trailing_comma = false
 line_length = 120
 force_grid_wrap = 2
 
 [tool.flake8]
 max-line-length = 120
+# E203: whitespace before ':' - conflicts with black
+# E501: line too long
 extend-ignore = "E203"
+per-file-ignores = """
+    src/neptune_api_codegen/docker/rofiles/neptune_api/client.py:E501
+    src/neptune_query/generated/neptune_api/*:E501
+"""
 
 [tool.mypy]
 mypy_path = "src/"
@@ -110,3 +121,11 @@ markers = [
 env_files = [
     ".env",
 ]
+
+# This is for neptune-api generated code only. We may remove this later
+[tool.ruff]
+line-length = 120
+target-version = "py38"
+
+[tool.ruff.lint]
+select = ["F", "UP"]

src/neptune_api_codegen/README.md

@@ -0,0 +1,56 @@
+# neptune_api_codegen
+
+Utility for generating Neptune API bindings and local API spec used by `neptune_query`.
+It orchestrates Dockerized generators to:
+
+- copy OpenAPI (swagger) and Protobuf sources from the private Neptune backend repository
+- build a unified OpenAPI document
+- generate Python client code from OpenAPI
+- generate Python modules from `.proto` definitions
+- place artifacts under `src/neptune_query/generated/`
+
+See also: `src/neptune_query/generated/README.md` for a map of generated outputs.
+
+## Prerequisites
+- Docker installed and running (required to build images and run generators)
+- Git installed (used to embed the backend commit reference)
+- Access to the Neptune backend repository (cloned locally)
+
+## Quick start
+From the `neptune-query` repository root:
+
+```
+poetry run python -m neptune_api_codegen.cli
+```
+
+By default this performs a dry-run:
+- copies swagger/proto inputs from the backend repo into a temporary work dir
+- generates OpenAPI, Python client code, and Python protobuf modules there
+- removes the temporary dir afterwards
+
+To apply the change to `src/neptune_query/generated/`, run with `--update`:
+
+```
+poetry run python -m neptune_api_codegen.cli --update
+```
+
+To keep the temporary directory for inspection:
+
+```
+poetry run python -m neptune_api_codegen.cli --keep-tmpdir
+```
+
+## Temporary working directory
+A unique working directory is created under `./tmp/` per run, for example:
+
+```
+./tmp/neptune-api__20260114_141229_2195/
+├── api_spec/          # swagger + proto copied from backend + GIT_REF
+└── generated_python/  # openapi client + protobuf python generated here
+```
+
+If `--keep-tmpdir` is not used, the work directory is removed.
+
+## Backend repository detection
+If `--neptune-repo-path` is not provided, the tool attempts to auto-detect the Neptune backend repository
+on your local filesystem next to this project (../neptune).

src/neptune_api_codegen/__init__.py

@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2025, Neptune Labs Sp. z o.o.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.