.gitlab-ci.yml

# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

variables:
  GIT_SUBMODULE_STRATEGY: recursive
  GIT_SUBMODULE_FORCE_HTTPS: "true"

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == 'main'
    - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/

stages:
  - check
  - test
  - docs
  - package
  - upload

variables:
  BUILD_AIQ_COMPAT: "true"
  BUILD_NAT_COMPAT: "true"
  CLICKHOUSE_USER: clickhouse
  CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
  GIT_SUBMODULE_FORCE_HTTPS: "true"
  GIT_SUBMODULE_STRATEGY: recursive
  LANGFUSE_NEXTAUTH_SECRET: ${LANGFUSE_NEXTAUTH_SECRET}
  LANGFUSE_SALT: ${LANGFUSE_SALT}
  MYSQL_ROOT_PASSWORD: "${MYSQL_ROOT_PASSWORD}"
  OPENAI_BASE_URL: "http://nginx-rewrite-models:8088"
  POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
  POSTGRES_USER: postgres
  POSTGRES_DB: postgres
  REDIS_PASSWORD: ${REDIS_PASSWORD}
  NAT_CI_SANDBOX_URL: "http://local-sandbox:6000"
  NAT_CI_ETCD_HOST: "etcd"
  NAT_CI_LANGFUSE_BUCKET: "langfuse"
  NAT_CI_LANGFUSE_HOST: "langfuse"
  NAT_CI_MILVUS_HOST: "milvus"
  NAT_CI_MINIO_HOST: "minio"
  NAT_CI_MYSQL_HOST: "mysql"
  NAT_CI_OAUTH2_HOST: "oauth2-server"
  NAT_CI_OAUTH2_PORT: "5000"
  NAT_CI_OPENSEARCH_URL: "http://opensearch:9200"
  NAT_CI_PHOENIX_URL: "http://phoenix:6006"
  NAT_CI_PISTON_URL: "http://piston:2000/api/v2"
  NAT_CI_PROXIED_OPENAI_BASE_URL: "${NAT_CI_PROXIED_OPENAI_BASE_URL}"
  NAT_CI_REDIS_HOST: "redis"
  UV_CACHE_DIR: .uv-cache
  WORKSPACE_TMP: "${CI_PROJECT_DIR}/.tmp"

default:
  image: ghcr.io/astral-sh/uv:0.9.28-python3.13-bookworm
  cache:
    - key: $CI_COMMIT_REF_SLUG
      paths:
        - $UV_CACHE_DIR
  before_script:
    - mkdir -p ${WORKSPACE_TMP}

  after_script:
    # Your `uv` commands
    - uv cache prune --ci

check:style:
  stage: check
  script:
    - echo "Runing checks"
    - ./ci/scripts/gitlab/checks.sh

  rules:
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
  cache:
    key:
      files:
        - .pre-commit-config.yaml
    paths:
      - .cache/pre-commit

check:model_health:
  stage: check
  timeout: 10 minutes
  allow_failure: true
  script:
    - ./ci/scripts/gitlab/model_health_check.sh
  artifacts:
    when: always
    paths:
      - model_health_results.json
  rules:
    - if: $CI_CRON_NIGHTLY == "1"
      when: always

test:python_tests:
  stage: test
  timeout: 2 hours
  services:
    - name: clickhouse/clickhouse-server
      alias: clickhouse
      pull_policy: if-not-present
      variables:
        CLICKHOUSE_DB: default

    - name: $CI_REGISTRY_IMAGE/local-sandbox
      alias: local-sandbox
      pull_policy: if-not-present

    - name: minio/minio:RELEASE.2025-07-18T21-56-31Z
      alias: minio
      command: ["server", "/data", "--console-address", ":9001"]
      pull_policy: if-not-present

    - name: mysql:9.3
      alias: mysql
      pull_policy: if-not-present

    - name: arizephoenix/phoenix:latest
      alias: phoenix
      pull_policy: if-not-present

    - name: $CI_REGISTRY_IMAGE/nginx-rewrite-models:20260303
      alias: nginx-rewrite-models
      pull_policy: if-not-present

    - name: postgres:17
      alias: postgres
      pull_policy: if-not-present
      variables:
        TZ: UTC
        PGTZ: UTC

    - name: redis:8.0
      alias: redis
      # Explicitly invoke the shell to expand the environment variable
      entrypoint: [ "sh", "-c", "/usr/local/bin/docker-entrypoint.sh --requirepass ${REDIS_PASSWORD}"]
      pull_policy: if-not-present

    - name: quay.io/coreos/etcd:v3.5.5
      alias: etcd
      command: ["etcd", "--advertise-client-urls", "http://0.0.0.0:2379", "--listen-client-urls", "http://0.0.0.0:2379"]
      pull_policy: if-not-present

    - name: milvusdb/milvus:v2.3.1
      alias: milvus
      variables:
        ETCD_ENDPOINTS: etcd:2379
        MINIO_ADDRESS: minio:9000
      command: ["milvus", "run", "standalone"]
      pull_policy: if-not-present

    - name: $CI_REGISTRY_IMAGE/oauth2_server:20251212
      alias: oauth2-server
      pull_policy: if-not-present
      variables:
        AUTHLIB_INSECURE_TRANSPORT: "1"
        FLASK_APP: app.py
        FLASK_ENV: development

    - name: opensearchproject/opensearch:2.11.1
      alias: opensearch
      pull_policy: if-not-present
      variables:
        discovery.type: "single-node"
        plugins.security.disabled: "true"

    - name: $CI_REGISTRY_IMAGE/piston:nat_test_20251212
      alias: piston
      pull_policy: if-not-present

    - name: langfuse/langfuse-worker:3
      alias: langfuse-worker
      pull_policy: if-not-present
      variables:
        CLICKHOUSE_MIGRATION_URL: clickhouse://clickhouse:9000
        CLICKHOUSE_URL: http://clickhouse:8123
        CLICKHOUSE_CLUSTER_ENABLED: false
        DATABASE_HOST: postgres
        DATABASE_USERNAME: ${POSTGRES_USER}
        DATABASE_PASSWORD: ${POSTGRES_PASSWORD}
        DATABASE_NAME: ${POSTGRES_DB}
        DATABASE_ARGS: ""
        LANGFUSE_USE_AZURE_BLOB: false
        LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${NAT_CI_LANGFUSE_BUCKET}
        LANGFUSE_S3_EVENT_UPLOAD_REGION: auto
        LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: minioadmin
        LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: minioadmin
        LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: http://minio:9000
        LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: true
        LANGFUSE_S3_EVENT_UPLOAD_PREFIX: events/
        LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${NAT_CI_LANGFUSE_BUCKET}
        LANGFUSE_S3_MEDIA_UPLOAD_REGION: auto
        LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: minioadmin
        LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: minioadmin
        LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: http://minio:9000
        LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: true
        LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: media/
        LANGFUSE_S3_BATCH_EXPORT_ENABLED: false
        LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${NAT_CI_LANGFUSE_BUCKET}
        LANGFUSE_S3_BATCH_EXPORT_PREFIX: exports/
        LANGFUSE_S3_BATCH_EXPORT_REGION: auto
        LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: http://minio:9000
        LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: http://minio:9000
        LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: minioadmin
        LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: minioadmin
        LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: true
        NEXTAUTH_SECRET: ${LANGFUSE_NEXTAUTH_SECRET}
        NEXTAUTH_URL: http://langfuse:3000
        REDIS_AUTH: ${REDIS_PASSWORD}
        REDIS_HOST: ${NAT_CI_REDIS_HOST}
        REDIS_PORT: 6379
        SALT: ${LANGFUSE_SALT}

    - name: langfuse/langfuse:3
      alias: langfuse
      pull_policy: if-not-present
      variables:
        CLICKHOUSE_MIGRATION_URL: clickhouse://clickhouse:9000
        CLICKHOUSE_URL: http://clickhouse:8123
        CLICKHOUSE_CLUSTER_ENABLED: false
        DATABASE_HOST: postgres
        DATABASE_USERNAME: ${POSTGRES_USER}
        DATABASE_PASSWORD: ${POSTGRES_PASSWORD}
        DATABASE_NAME: ${POSTGRES_DB}
        DATABASE_ARGS: ""
        LANGFUSE_USE_AZURE_BLOB: false
        LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${NAT_CI_LANGFUSE_BUCKET}
        LANGFUSE_S3_EVENT_UPLOAD_REGION: auto
        LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: minioadmin
        LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: minioadmin
        LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: http://minio:9000
        LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: true
        LANGFUSE_S3_EVENT_UPLOAD_PREFIX: events/
        LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${NAT_CI_LANGFUSE_BUCKET}
        LANGFUSE_S3_MEDIA_UPLOAD_REGION: auto
        LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: minioadmin
        LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: minioadmin
        LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: http://minio:9000
        LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: true
        LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: media/
        LANGFUSE_S3_BATCH_EXPORT_ENABLED: false
        LANGFUSE_S3_BATCH_EXPORT_BUCKET: ${NAT_CI_LANGFUSE_BUCKET}
        LANGFUSE_S3_BATCH_EXPORT_PREFIX: exports/
        LANGFUSE_S3_BATCH_EXPORT_REGION: auto
        LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: http://minio:9000
        LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: http://minio:9000
        LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: minioadmin
        LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY: minioadmin
        LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: true
        NEXTAUTH_SECRET: ${LANGFUSE_NEXTAUTH_SECRET}
        NEXTAUTH_URL: http://langfuse:3000
        REDIS_AUTH: ${REDIS_PASSWORD}
        REDIS_HOST: ${NAT_CI_REDIS_HOST}
        REDIS_PORT: 6379
        SALT: ${LANGFUSE_SALT}
        LANGFUSE_INIT_ORG_ID: test-org
        LANGFUSE_INIT_PROJECT_ID: test-project
        LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_PUBLIC_KEY}
        LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_SECRET_KEY}
        LANGFUSE_INIT_USER_EMAIL: test@localhost.dev
        LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_USER_PW}
        HOSTNAME: 0.0.0.0

  script:
    - echo "Running tests"
    - ./ci/scripts/gitlab/tests.sh
  coverage: /TOTAL.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
  artifacts:
    when: always
    reports:
      junit: pytest_junit_report*.xml
      coverage_report:
        coverage_format: cobertura  # or jacoco
        path: pytest_coverage_report*.xml

docs:build:
  stage: docs
  script:
    - echo "Building docs"
    - ./ci/scripts/gitlab/docs.sh
    # Copy the built docs to the public folder for artifact (required path for Gitlab Pages)
    - echo "Copying built docs to public folder for artifact"
    - cp -r docs/build/html public
  artifacts:
    paths:
      - public
    expire_in: 1 week

package:wheel:
  stage: package
  script:
    - echo "Building wheel"
    - ./ci/scripts/gitlab/build_wheel.sh
  artifacts:
    paths:
      # match the following wheels:
      # - root metapackage: .tmp/wheels/nvidia-nat/nvidia-nat/nvidia_nat-<VERSION>-py3-none-any.whl
      # - subpackages: .tmp/wheels/nvidia-nat/nvidia_nat_<PACKAGE>/nvidia_nat_<PACKAGE>-<VERSION>-py3-none-any.whl
      # - example wheels: .tmp/wheels/nvidia-nat/examples/*.whl
      - .tmp/wheels/nvidia-nat/*/*.whl
    expire_in: 1 week
  rules:
    - if: $CI_CRON_NIGHTLY == "1"
      when: always
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH == 'develop-test'
    - if: $CI_COMMIT_BRANCH == 'main'
    - if: $CI_COMMIT_BRANCH =~ /^release\/.*$/

upload:artifactory:
  stage: upload
  script:
    - echo "Uploading wheel to artifactory"
    - ./ci/scripts/gitlab/artifactory_upload.sh
  needs:
    - job: package:wheel
  rules:
    - if: $CI_COMMIT_TAG
    - if: $CI_CRON_NIGHTLY == "1"