frameworks-sdk/.gitlab-ci.yml at main · argonne-lcf/frameworks-sdk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
# Build, package, and test the Frameworks SDK from {Aurora,Sunspot}
spec:
  inputs:
    runner:
      options: [aurora, aurora-eval, sunspot]
      default: aurora
      description: CI runners to use
    root_dir:
      rules:
        - if: $[[ inputs.runner ]] =~ '/^aurora.*/'
          default: '/lus/flare/projects/datascience/frameworks-ci'
        - if: $[[ inputs.runner ]] == 'sunspot'
          default: '/lus/tegu/projects/datascience/frameworks-ci'
      description: Directory to store built artifacts
    python_version:
      default: 3.12
      description: Python version to build against
    torch_version:
      default: main
      description: PyTorch version to build
    triton_version:
      default: main
      description: Triton-XPU version to build
    torchccl_version:
      default: ''
      description: torchCCL version to build
    ipex_version:
      default: ''
      description: IPEX version to build
    vllm_version:
      default: main
      description: vLLM version to build
    vllm_xpu_kernels_version:
      default: main
      description: vLLM XPU kernels version to build
    sdk_tests_version:
      default: main
      description: Frameworks SDK validation tests version to test
---

variables:
  FRAMEWORKS_ROOT_DIR: $[[ inputs.root_dir ]]
  FRAMEWORKS_PYTHON_VERSION: $[[ inputs.python_version ]]
  FRAMEWORKS_TORCH_VERSION: $[[ inputs.torch_version ]]
  FRAMEWORKS_TRITON_VERSION: $[[ inputs.triton_version ]]
  FRAMEWORKS_TORCHCCL_VERSION: $[[ inputs.torchccl_version ]]
  FRAMEWORKS_IPEX_VERSION: $[[ inputs.ipex_version ]]
  FRAMEWORKS_VLLM_VERSION: $[[ inputs.vllm_version ]]
  FRAMEWORKS_VLLM_XPU_KERNELS_VERSION: $[[ inputs.vllm_xpu_kernels_version ]]
  FRAMEWORKS_SDK_TESTS_VERSION: $[[ inputs.sdk_tests_version ]]

include:
  - local: 'templates/common.yml'
    inputs:
      runner: $[[ inputs.runner ]]

  # Nightly Wheels
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: triton
      needs: [llvm]
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: torch
      needs: [triton, pti-gpu]
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: intel_extension_for_pytorch
      needs: [torch]
    rules:
      - if: '"$[[ inputs.ipex_version ]]" != ""'
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: oneccl_bind_pt
      needs: [torch]
    rules:
      - if: '"$[[ inputs.torchccl_version ]]" != ""'
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: mpi4py
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: vllm
      needs: [torch]
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: vllm_xpu_kernels
      needs: [torch]
  - local: 'templates/nightly-wheel.yml'
    inputs:
      wheel: torchvision
      needs: [torch]

# Custom LLVM build for `triton-xpu`
llvm:
  stage: build
  extends: .common-runner
  script:
    - ./deps/llvm
  artifacts:
    when: always
    expire_in: never
    paths:
      - "*.log"

# Updated `pti-gpu` build
pti-gpu:
  stage: build
  extends: .common-runner
  script:
    - ./deps/pti-gpu
  artifacts:
    when: always
    expire_in: never
    paths:
      - "*.log"

test:
  stage: test
  extends: .common-runner
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
  script:
    - ./tests/bats/bin/bats --report-formatter junit --output . tests/*.bats
  after_script:
    # Convert the frameworks-sdk-tests `results/*/summary.json` (written to
    # the workspace by the `--results-dir` flag in the bats test) into a JUnit
    # XML report; `after_script` runs on failure so failing results are still
    # reported. The job status stays driven by the `bats` exit code.
    - python3 tools/frameworks-sdk-tests-junit.py || true
  artifacts:
    when: always
    expire_in: never
    paths:
      - "*.xml"
    reports:
      junit:
        - "*.xml"