.gitlab-ci.yml

workflow:
  rules:
    - if: $CI_COMMIT_TAG
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_TITLE =~ /^chore\(release\)/
      when: never
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

include:
  - template: Jobs/SAST.gitlab-ci.yml
  - template: Jobs/Secret-Detection.gitlab-ci.yml
  - project: 'gitlab-org/orbit/experiments/ai-review-bot'
    file: '/template.yml'
  - component: ${CI_SERVER_FQDN}/gitlab-org/orbit/components/lgtm-agent/lgtm-agent@main

stages:
  - lint
  - docs
  - security
  - test
  - ai
  - build
  - publish
  - release
  - review

# Job Templates

.job-template:
  tags:
    - saas-linux-medium-amd64
  image: registry.gitlab.com/gitlab-org/orbit/build-images/rust-builder:latest
  variables:
    MISE_DISABLE_TOOLS: lefthook
  before_script:
    - mise install

.sccache-template:
  variables:
    CARGO_INCREMENTAL: "0"
    SCCACHE_GCS_BUCKET: "gl-knowledgegraph-sccache"
    SCCACHE_GCS_KEY_PATH: "${SCCACHE_GCS_KEY}"
    SCCACHE_GCS_RW_MODE: "READ_WRITE"
  before_script:
    - mise install
    - |
      if [ -n "${SCCACHE_GCS_KEY}" ]; then
        export RUSTC_WRAPPER="sccache"
        sccache --start-server || true
      fi

.test-template:
  extends:
    - .job-template
    - .sccache-template
  variables:
    RUST_BACKTRACE: "1"
  artifacts:
    when: always
    reports:
      junit:
        - target/nextest/ci/junit.xml

.mr-only-rules:
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

.mr-and-main-rules:
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

# Lint Stage

agent-file-sync-check:
  stage: lint
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/alpine:latest
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - diff AGENTS.md CLAUDE.md || (echo "AGENTS.md and CLAUDE.md are out of sync. Please ensure they have identical content." && exit 1)

mr-title-check:
  stage: lint
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/oven/bun:1.3-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-only-rules
  before_script:
    - bun install -g @commitlint/cli @commitlint/config-conventional
  script:
    - ./scripts/check-mr-title.sh

ontology-schema-validate:
  stage: lint
  image: ghcr.io/astral-sh/uv:python3.12-alpine
  tags:
    - saas-linux-medium-amd64
  extends:
    - .mr-and-main-rules
  needs: []
  script:
    - test "$(wc -c < config/schemas/ontology.schema.json)" -le 32768 || (echo "config/schemas/ontology.schema.json must stay at or below 32 KB" && exit 1)
    - find config/ontology -name '*.yaml' ! -name 'reference.yaml' | xargs uvx check-jsonschema --verbose --schemafile config/schemas/ontology.schema.json

fmt-check:
  stage: lint
  extends:
    - .job-template
    - .mr-only-rules
  script:
    - cargo fmt --all -- --check

lint-check:
  stage: lint
  extends:
    - .job-template
    - .sccache-template
    - .mr-only-rules
  script:
    - cargo clippy --all-targets --all-features -- -D warnings
    - sccache --show-stats || true

# Security Stage

cargo-audit:
  stage: security
  extends:
    - .job-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo audit

cargo-deny:
  stage: security
  extends:
    - .job-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo deny check

cargo-geiger:
  stage: security
  extends:
    - .job-template
    - .mr-and-main-rules
  needs: []
  script:
    - for pkg in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name'); do cargo geiger -p "$pkg" --all-features || true; done
  allow_failure: true

generate-sbom:
  stage: security
  extends:
    - .job-template
  needs: []
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  script:
    - cargo cyclonedx --format json > sbom.cdx.json
  artifacts:
    paths:
      - sbom.cdx.json
    reports:
      cyclonedx: sbom.cdx.json

semgrep-sast:
  stage: security
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      exists:
        - "**/*.rs"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      exists:
        - "**/*.rs"

# Override the default secret_detection rules so the job runs on MR pipelines.
# The Jobs/Secret-Detection.gitlab-ci.yml template defaults to branch pipelines
# only, which conflicts with this project's workflow (MR + main branch only).
secret_detection:
  stage: security
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

# Docs stage

check_docs_markdown:
  stage: docs
  extends:
    - .mr-only-rules
  image: registry.gitlab.com/gitlab-org/technical-writing/docs-gitlab-com/lint-markdown:alpine-3.22-vale-3.13.0-markdownlint2-0.19.0-lychee-0.21.0
  script:
    # Lint prose
    - vale --minAlertLevel error docs/source
    # Lint Markdown
    - markdownlint-cli2 'docs/source/**/*.md'
    # Check links
    - lychee --offline --include-fragments docs/source/**/*.md

docs_hugo_build:
  image: registry.gitlab.com/gitlab-org/technical-writing/docs-gitlab-com/docs-gitlab-com-builder:hugo-0.150.1
  stage: docs
  extends:
    - .mr-only-rules
  variables:
    DOCS_BRANCH: "main"
  before_script:
    # Check if this a release branch, which would be the case for a backport.
    # If this is a backport MR, we need to checkout the appropriate version
    # of the Docs website.
    - |
      if [[ $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ [0-9]+-[0-9]+-stable ]]; then
        BRANCH_NAME=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME
        echo "Detected merge request to stable branch: $BRANCH_NAME"
      # Check if we're directly on a stable branch (direct push/commit)
      elif [[ $CI_COMMIT_BRANCH =~ [0-9]+-[0-9]+-stable ]]; then
        BRANCH_NAME=$CI_COMMIT_BRANCH
        echo "Detected direct commit to stable branch: $BRANCH_NAME"
      fi

      # Extract version info if we found a stable branch
      if [[ -n $BRANCH_NAME ]]; then
        MAJOR=$(echo $BRANCH_NAME | cut -d '-' -f 1)
        MINOR=$(echo $BRANCH_NAME | cut -d '-' -f 2)
        # Convert GitLab style (17-9-stable-ee) to Docs style (17.9)
        DOCS_BRANCH_CANDIDATE="$MAJOR.$MINOR"

        # Check if the branch exists in the Docs website repo, fallback to main if not
        if git ls-remote --heads --exit-code https://gitlab.com/gitlab-org/technical-writing/docs-gitlab-com.git "refs/heads/$DOCS_BRANCH_CANDIDATE" >/dev/null 2>&1; then
          DOCS_BRANCH="$DOCS_BRANCH_CANDIDATE"
          echo "Using docs-gitlab-com branch $DOCS_BRANCH for release branch"
        else
          DOCS_BRANCH="main"
          echo "Branch $DOCS_BRANCH_CANDIDATE does not exist, falling back to main"
        fi
      fi
    # Clone the Docs website project
    - git clone --depth 1 --filter=tree:0 --branch $DOCS_BRANCH https://gitlab.com/gitlab-org/technical-writing/docs-gitlab-com.git
    - cd docs-gitlab-com
    - make add-latest-icons
  script:
    # Test that Hugo will build
    - hugo --gc --printPathWarnings --panicOnWarning  --environment test
    # Test for invalid index pages
    # See https://gitlab.com/gitlab-org/technical-writing/docs-gitlab-com/-/blob/main/scripts/check-index-filenames.sh
    - make check-index-pages SEARCH_DIR="../docs/source"
    #- make check-index-pages SEARCH_DIR="../doc-locale/source" # Enable when translation is added

# Test Stage

unit-test:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  needs: []
  script:
    - cargo nextest run --profile ci --lib --bins --test compiler
    - sccache --show-stats || true

integration-test:
  stage: test
  extends:
    - .test-template
    - .mr-and-main-rules
  tags:
    - saas-linux-large-amd64
  services:
    - name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    TESTCONTAINERS_HOST_OVERRIDE: docker
  needs: []
  before_script:
    - !reference [.sccache-template, before_script]
    - echo "$CI_DEPENDENCY_PROXY_PASSWORD" | docker login $CI_DEPENDENCY_PROXY_SERVER -u $CI_DEPENDENCY_PROXY_USER --password-stdin
    - |
      docker pull ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/clickhouse/clickhouse-server:25.12 &
      docker pull ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/nats:2.11-alpine &
      wait
    - |
      docker tag ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/clickhouse/clickhouse-server:25.12 clickhouse/clickhouse-server:25.12
      docker tag ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/nats:2.11-alpine nats:2.11-alpine
  script:
    - cargo nextest run --profile ci --all-features --test docker
    - sccache --show-stats || true

# Build Stage

.docker-build:
  stage: build
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:29
  needs: []
  services:
    - name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:29-dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    IMAGE_NAME: gkg
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  script:
    - ./scripts/docker-build.sh "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:develop-${ARCH}"

docker-build-amd64:
  extends: .docker-build
  tags:
    - saas-linux-large-amd64
  variables:
    ARCH: amd64

docker-build-arm64:
  extends: .docker-build
  tags:
    - saas-linux-large-arm64
  variables:
    ARCH: arm64

docker-manifest:
  stage: build
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:29
  tags:
    - saas-linux-medium-amd64
  needs:
    - docker-build-amd64
    - docker-build-arm64
  variables:
    IMAGE_NAME: gkg
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  script:
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
    - docker buildx imagetools create
        -t "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:develop"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:develop-amd64"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:develop-arm64"

docker-build-mr:
  stage: build
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:29
  tags:
    - saas-linux-large-amd64
  needs: []
  services:
    - name: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:29-dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    IMAGE_NAME: gkg
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  script:
    - ./scripts/docker-build.sh "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:develop-mr-${CI_MERGE_REQUEST_IID}-${CI_COMMIT_SHORT_SHA}-amd64"

build-proto-gem:
  needs: []
  image: ruby:3.3
  tags:
    - saas-linux-medium-amd64
  stage: build
  before_script: []
  script:
    - BUNDLE_GEMFILE=tools/protogem/Gemfile bundle install --quiet
    - |
      if [ -n "$CI_COMMIT_TAG" ]; then
        ruby tools/protogem/build-proto-gem --version "${CI_COMMIT_TAG#v}"
      else
        ruby tools/protogem/build-proto-gem
      fi
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/
    - changes:
        - crates/gkg-server/proto/**/*
        - tools/protogem/**/*
  artifacts:
    paths:
      - _build/gitlab-gkg-proto-*.gem
    expire_in: 30 days

publish-proto-gem:
  needs:
    - build-proto-gem
  image: ruby:3.3
  tags:
    - saas-linux-medium-amd64
  stage: publish
  before_script: []
  script:
    - gem push "_build/gitlab-gkg-proto-${CI_COMMIT_TAG#v}.gem"
  rules:
    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org/orbit" && $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
  environment:
    name: rubygems
    url: https://rubygems.org/gems/gitlab-gkg-proto

# Release Stage

semantic-release:
  stage: release
  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/node:22
  tags:
    - saas-linux-medium-amd64
  variables:
    GIT_STRATEGY: clone
  needs:
    - job: build-proto-gem
      optional: true
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: manual
  before_script:
    - npm install -g semantic-release @semantic-release/gitlab @semantic-release/git @semantic-release/commit-analyzer @semantic-release/release-notes-generator @semantic-release/changelog conventional-changelog-conventionalcommits semantic-release-slack-bot
  script:
    - npx semantic-release

.docker-release:
  stage: build
  image: docker:29
  services:
    - name: docker:29-dind
      alias: docker
  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    IMAGE_NAME: gkg
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/
  script:
    - export GKG_VERSION="${CI_COMMIT_TAG#v}"
    - ./scripts/docker-build.sh "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${GKG_VERSION}-${ARCH}"

release-build-amd64:
  extends: .docker-release
  tags:
    - saas-linux-large-amd64
  variables:
    ARCH: amd64

release-build-arm64:
  extends: .docker-release
  tags:
    - saas-linux-large-arm64
  variables:
    ARCH: arm64

release-manifest:
  stage: build
  image: docker:29
  tags:
    - saas-linux-medium-amd64
  needs:
    - release-build-amd64
    - release-build-arm64
  variables:
    IMAGE_NAME: gkg
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/
  script:
    - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
    - VERSION="${CI_COMMIT_TAG#v}"
    - docker buildx imagetools create
        -t "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${VERSION}"
        -t "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:latest"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${VERSION}-amd64"
        "${CI_REGISTRY_IMAGE}/${IMAGE_NAME}:${VERSION}-arm64"

# Review stage

.review-docs:
  image: ruby:3.3-alpine3.23
  extends:
    - .mr-only-rules
  stage: review
  cache: {}
  dependencies: []
  before_script:
  - gem install gitlab --no-doc
  # We need to download the script rather than clone the repo since the
  # review-docs-cleanup job will not be able to run when the branch gets
  # deleted (when merging the MR).
  - apk add --update openssl
  - wget https://gitlab.com/gitlab-org/gitlab/-/raw/master/scripts/trigger-build.rb
  - chmod 755 trigger-build.rb
  variables:
    GIT_STRATEGY: none
    DOCS_GITLAB_REPO_SUFFIX: orbit
    # By default, deploy the Review App using the `main` branch of the `gitlab-org/gitlab-docs` project
    DOCS_BRANCH: main
  when: manual

# Trigger a docs build in docs-gitlab-com project
# Useful to preview the docs changes live
# https://docs.gitlab.com/development/documentation/review_apps/
review-docs-deploy:
  extends:
  - .review-docs
  environment:
    name: review/mr-${CI_MERGE_REQUEST_IID}
    url: https://docs.gitlab.com/upstream-review-mr-${DOCS_GITLAB_REPO_SUFFIX}-${CI_MERGE_REQUEST_IID}/${DOCS_GITLAB_REPO_SUFFIX}
    auto_stop_in: 2 weeks
    on_stop: review-docs-cleanup
  script:
  - ./trigger-build.rb docs deploy

# Cleanup remote environment of docs-gitlab-com
review-docs-cleanup:
  extends:
  - .review-docs
  environment:
    name: review/mr-${CI_MERGE_REQUEST_IID}
    action: stop
  script:
  - ./trigger-build.rb docs cleanup

# AI Review Stage
#
# Manually-triggered AI agents for MR review. Uses the shared
# ai-review-bot framework from gitlab-org/orbit/experiments/ai-review-bot.
#
# Required CI/CD variables (Settings > CI/CD > Variables):
#   GITLAB_REVIEW_TOKEN  — Masked + Hidden
#   AI_PROXY_INIT_TOKEN  — Masked + Hidden
#   GOOGLE_CLOUD_PROJECT — GCP project ID for Vertex AI
#   VERTEX_SA_KEY_B64    — Masked + Hidden

ai:performance:
  extends: .ai-review-base
  stage: ai
  timeout: 1 hour
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  environment:
    name: ai-review
    action: access
  variables:
    AI_AGENT: performance
    AI_MODEL_OVERRIDE: "google-vertex-anthropic/claude-opus-4-6@default"
    GOOGLE_CLOUD_LOCATION: "global"
    AI_REFS: "https://gitlab.com/gitlab-org/gitlab.git,https://github.com/ClickHouse/clickhouse-docs.git,https://gitlab.com/gitlab-org/analytics-section/siphon.git"

ai:security:
  extends: .ai-review-base
  stage: ai
  timeout: 1 hour
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  environment:
    name: ai-review
    action: access
  variables:
    AI_AGENT: security
    AI_MODEL_OVERRIDE: "google-vertex-anthropic/claude-opus-4-6@default"
    GOOGLE_CLOUD_LOCATION: "global"
    AI_REFS: "https://gitlab.com/gitlab-org/gitlab.git,https://github.com/ClickHouse/clickhouse-docs.git,https://gitlab.com/gitlab-org/analytics-section/siphon.git"

ai:docs:
  extends: .ai-review-base
  stage: ai
  timeout: 1 hour
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
      when: manual
      allow_failure: true
  environment:
    name: ai-review
    action: access
  variables:
    AI_AGENT: docs
    AI_MODEL_OVERRIDE: "google-vertex-anthropic/claude-opus-4-6@default"
    GOOGLE_CLOUD_LOCATION: "global"