Merge branch 'develop' into atg-20250219-cs60

Author: minitriga

.github/file-filters.yml

@@ -19,6 +19,8 @@ poetry_files: &poetry_files
 
 doc_files: &doc_files
   - "docs/**"
+  - package.json
+  - package-lock.json
 
 python_all: &python_all
   - "**/*.py"
@@ -32,7 +34,14 @@ yaml_all: &yaml_all
 markdown_all: &markdown_all
   - "**/*.{md,mdx}"
 
+infrahub_reference_generated: &infrahub_reference_generated
+  - "docs/docs/infrahubctl/*.mdx"
+  - "docs/docs/python-sdk/reference/config.mdx"
+
 documentation_all:
   - *development_files
   - *doc_files
   - *markdown_all
+
+documentation_generated_all:
+  - *infrahub_reference_generated

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ env:
   INFRAHUB_IMAGE_VER: "local"
   PYTEST_XDIST_WORKER_COUNT: 4
   INFRAHUB_TEST_IN_DOCKER: 1
-  VALE_VERSION: "3.0.7"
+  VALE_VERSION: "3.7.1"
   GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
   METRICS_ENDPOINT: ${{ secrets.METRICS_ENDPOINT }}
 
@@ -36,6 +36,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       documentation: ${{ steps.changes.outputs.documentation_all }}
+      documentation_generated: ${{ steps.changes.outputs.documentation_generated_all }}
       helm: ${{ steps.changes.outputs.helm_all }}
       python: ${{ steps.changes.outputs.python_all }}
       yaml: ${{ steps.changes.outputs.yaml_all }}
@@ -75,7 +76,7 @@ jobs:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
       - name: "Setup environment"
-        run: "pip install ruff==0.8.6"
+        run: "pip install ruff==0.11.0"
       - name: "Linting: ruff check"
         run: "ruff check ."
       - name: "Linting: ruff format"
@@ -115,6 +116,91 @@ jobs:
           SHELLCHECK_OPTS: --exclude=SC2086 --exclude=SC2046 --exclude=SC2004 --exclude=SC2129
 
 
+  documentation:
+    defaults:
+      run:
+        working-directory: ./docs
+    if: |
+      always() && !cancelled() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled') &&
+      needs.files-changed.outputs.documentation == 'true'
+    needs: ["files-changed", "yaml-lint", "python-lint"]
+    runs-on: "ubuntu-22.04"
+    timeout-minutes: 5
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+        with:
+          submodules: true
+      - name: Install NodeJS
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+          cache-dependency-path: docs/package-lock.json
+      - name: "Install dependencies"
+        run: npm install
+      - name: "Setup Python environment"
+        run: "pip install invoke toml"
+      - name: "Build docs website"
+        run: "invoke docs"
+
+  validate-generated-documentation:
+    if: |
+      always() && !cancelled() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled') &&
+      (needs.files-changed.outputs.python == 'true') || (needs.files-changed.outputs.documentation_generated == 'true')
+    needs: ["files-changed", "yaml-lint", "python-lint"]
+    runs-on: "ubuntu-22.04"
+    timeout-minutes: 5
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+        with:
+          submodules: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: "Setup Python environment"
+        run: |
+          pipx install poetry==2.1
+          poetry config virtualenvs.create true --local
+          poetry env use 3.12
+      - name: "Install dependencies"
+        run: "poetry install --no-interaction --no-ansi --extras ctl"
+      - name: "Setup environment"
+        run: "pip install invoke toml"
+      - name: "Validate generated documentation"
+        run: "poetry run invoke docs-validate"
+
+  validate-documentation-style:
+    if: |
+      always() && !cancelled() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled')
+    needs: ["files-changed", "yaml-lint", "python-lint"]
+    runs-on: "ubuntu-22.04"
+    timeout-minutes: 5
+    steps:
+      - name: "Check out repository code"
+        uses: "actions/checkout@v4"
+        with:
+          submodules: true
+
+      # The official GitHub Action for Vale doesn't work, installing manually instead:
+      # https://github.com/errata-ai/vale-action/issues/103
+      - name: Download Vale
+        run: |
+          curl -sL "https://github.com/errata-ai/vale/releases/download/v${VALE_VERSION}/vale_${VALE_VERSION}_Linux_64-bit.tar.gz" -o vale.tar.gz
+          tar -xzf vale.tar.gz
+        env:
+          VALE_VERSION: ${{ env.VALE_VERSION }}
+      - name: "Validate documentation style"
+        run: ./vale $(find ./docs -type f \( -name "*.mdx" -o -name "*.md" \) )
+
   unit-tests:
     env:
       # workaround for Rich table column width
@@ -144,7 +230,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: "Setup environment"
         run: |
-          pipx install poetry==1.8.5
+          pipx install poetry==2.1 --python python${{ matrix.python-version }}
           poetry config virtualenvs.create true --local
           pip install invoke toml codecov
       - name: "Install Package"
@@ -174,7 +260,7 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   # ------------------------------------------ Integration Tests  ------------------------------------------
-  integration-tests:
+  integration-tests-latest-infrahub:
     if: |
       always() && !cancelled() &&
       !contains(needs.*.result, 'failure') &&
@@ -197,15 +283,93 @@ jobs:
           echo "PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp" >> $GITHUB_ENV
       - name: "Setup environment"
         run: |
-          pipx install poetry==1.8.5
+          pipx install poetry==2.1
           poetry config virtualenvs.create true --local
           pip install invoke toml codecov
       - name: "Install Package"
         run: "poetry install --all-extras"
       - name: "Integration Tests"
-        run: "poetry run pytest --cov infrahub_sdk tests/integration/"
+        run: |
+          poetry run pytest --cov infrahub_sdk tests/integration/
       - name: "Upload coverage to Codecov"
         run: |
           codecov --flags integration-tests
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  # NOTE: Disabling this test for now because it's expected that we can't start the latest version of infrahub
+  # with the current shipping version of infrahub-testcontainers
+  # integration-tests-local-infrahub:
+  #   if: |
+  #     always() && !cancelled() &&
+  #     !contains(needs.*.result, 'failure') &&
+  #     !contains(needs.*.result, 'cancelled') &&
+  #     needs.files-changed.outputs.python == 'true' &&
+  #     (github.base_ref == 'stable' || github.base_ref == 'develop')
+  #   needs: ["files-changed", "yaml-lint", "python-lint"]
+  #   runs-on:
+  #     group: "huge-runners"
+  #   timeout-minutes: 30
+  #   steps:
+  #     - name: "Check out repository code"
+  #       uses: "actions/checkout@v4"
+
+  #     - name: "Extract target branch name"
+  #       id: extract_branch
+  #       run: echo "TARGET_BRANCH=${{ github.base_ref }}" >> $GITHUB_ENV
+
+  #     - name: "Checkout infrahub repository"
+  #       uses: "actions/checkout@v4"
+  #       with:
+  #         repository: "opsmill/infrahub"
+  #         path: "infrahub-server"
+  #         ref: ${{ github.base_ref }}
+  #         submodules: true
+
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: "3.12"
+
+  #     - name: "Setup git credentials prior dev.build"
+  #       run: |
+  #         cd infrahub-server
+  #         git config --global user.name 'Infrahub'
+  #         git config --global user.email '[email protected]'
+  #         git config --global --add safe.directory '*'
+  #         git config --global credential.usehttppath true
+  #         git config --global credential.helper /usr/local/bin/infrahub-git-credential
+
+  #     - name: "Set environment variables prior dev.build"
+  #       run: |
+  #         echo "INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }}" >> $GITHUB_ENV
+  #         RUNNER_NAME=$(echo "${{ runner.name }}" | grep -o 'ghrunner[0-9]\+' | sed 's/ghrunner\([0-9]\+\)/ghrunner_\1/')
+  #         echo "PYTEST_DEBUG_TEMPROOT=/var/lib/github/${RUNNER_NAME}/_temp" >> $GITHUB_ENV
+  #         echo "INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }}" >> $GITHUB_ENV
+  #         echo "INFRAHUB_TESTING_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }}" >> $GITHUB_ENV
+  #         echo "INFRAHUB_TESTING_DOCKER_IMAGE=opsmill/infrahub" >> $GITHUB_ENV
+
+  #     - name: "Build container"
+  #       run: |
+  #         cd infrahub-server
+  #         inv dev.build
+
+  #     - name: "Setup environment"
+  #       run: |
+  #         pipx install poetry==2.1
+  #         poetry config virtualenvs.create true --local
+  #         pip install invoke toml codecov
+
+  #     - name: "Install Package"
+  #       run: "poetry install --all-extras"
+
+  #     - name: "Integration Tests"
+  #       run: |
+  #         echo "Running tests for version: $INFRAHUB_TESTING_IMAGE_VER"
+  #         poetry run pytest --cov infrahub_sdk tests/integration/
+
+  #     - name: "Upload coverage to Codecov"
+  #       run: |
+  #         codecov --flags integration-tests
+  #       env:
+  #         CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/sync-docs.yml

@@ -0,0 +1,45 @@
+---
+# yamllint disable rule:truthy rule:line-length
+name: Sync Folders
+
+on:
+  push:
+    branches:
+      - stable
+    paths:
+      - 'docs/docs/**'
+      - 'docs/sidebars-infrahubctl.ts'
+      - 'docs/sidebars-python-sdk.ts'
+  pull_request:
+    branches:
+      - stable
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source repository
+        uses: actions/checkout@v4
+        with:
+          path: source-repo
+
+      - name: Checkout target repository
+        uses: actions/checkout@v4
+        with:
+          repository: opsmill/infrahub-docs
+          token: ${{ secrets.PAT_TOKEN }}
+          path: target-repo
+
+      - name: Sync folders
+        run: |
+          rm -rf target-repo/docs/docs-python-sdk/*
+          rm -f target-repo/docs/sidebars-python-sdk.ts
+          rm -f target-repo/docs/sidebars-infrahubctl.ts
+          cp -r source-repo/docs/docs/* target-repo/docs/docs-python-sdk/
+          cp source-repo/docs/sidebars-infrahubctl.ts target-repo/docs/
+          cp source-repo/docs/sidebars-python-sdk.ts target-repo/docs/
+          cd target-repo
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add .
+          if ! (git diff --quiet && git diff --staged --quiet); then git commit -m "Sync doc Python SDK from infrahub repo" && git push; fi

.gitignore

@@ -27,4 +27,5 @@ dist/*
 **/*.csv
 
 # Generated files
-generated/
+generated/
+sandbox/

.vale.ini

@@ -0,0 +1,15 @@
+StylesPath = .vale/styles
+
+MinAlertLevel = warning
+
+[formats]
+mdx = md
+
+[docs/**/*.md]
+BasedOnStyles = Infrahub
+;(import.*?\n) to ignore import statement in .mdx
+;(```.*?```\n) to ignore code block in .mdx
+BlockIgnores = (?s) *((import.*?\n)|(```.*?```\n))
+
+[*]
+BasedOnStyles = Infrahub

.vale/styles/Infrahub/branded-terms-case-swap.yml

@@ -0,0 +1,36 @@
+---
+extends: substitution
+message: Use '%s' instead of '%s'
+level: error
+ignorecase: false
+action:
+  name: replace
+swap:
+  (?:ansible): Ansible
+  (?:[^"]docker): Docker
+  (?:[Dd]ockerhub): DockerHub
+  (?:[^/][Gg]ithub): GitHub
+  (?:[Gg]itlab): GitLab
+  (?:gitpod): GitPod
+  (?:grafana): Grafana
+  (?:[^/][Gg]raphql): GraphQL
+  (?:[Ii]nflux[Dd]b): InfluxDB
+  infrahub(?:\s|$): Infrahub
+  (?:jinja2): Jinja2
+  (?:k3s): K3s
+  (?:k8s): K8s
+  (?:kubernetes): Kubernetes
+  (?:[^/][Mm]y[Ss]ql): MySQL
+  (?:neo4j): Neo4j
+  (?:[^/][Nn]ginx): NGINX
+  (?:[Nn]odejs): Node.js
+  (?:[^/][Oo]penapi): OpenAPI
+  (?:Openconfig): OpenConfig
+  opsmill(?:\s|$): OpsMill
+  (?:[Pp]ostgre[Ss]ql): PostgreSQL
+  (?:[^/]prometheus): Prometheus
+  (?:[^/-]python): Python
+  (?:[Rr]abbitmq): RabbitMQ
+  (?:[^/]terraform): Terraform
+  (?:ubuntu): Ubuntu
+  (?:[Vv]s\W?[Cc]ode): VS Code

.vale/styles/Infrahub/colon-space.yml

@@ -0,0 +1,8 @@
+---
+extends: substitution
+message: "Use a space after a colon, but not before"
+level: warning
+ignorecase: true
+nonword: true
+swap:
+  '(?<=\s):(?=\s+)': ': '

.vale/styles/Infrahub/eg-ie.yml

@@ -0,0 +1,12 @@
+---
+extends: substitution
+message: "Instead of %s, use ',i.e.,' or 'for example:'."
+level: warning
+ignorecase: true
+nonword: true
+action:
+  name: replace
+swap:
+  - e\.g\.: i.e.
+  - e\. g\.: i.e.
+  - i\. e\.: i.e.

.vale/styles/Infrahub/oxford-comma.yml

@@ -0,0 +1,8 @@
+---
+extends: existence
+message: "Use a comma before the last 'and' or 'or' in a list of items."
+level: suggestion
+scope: sentence
+nonword: true
+tokens:
+  - '(?:[^\s,]+,){1,} \w+ (?:and|or) \w+[.?!]'