feat!: separate CIM data models into standalone splunk-cim-models package [BREAKING] #2428
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "develop" | |
| tags: | |
| - "v[0-9]+.[0-9]+.[0-9]+" | |
| pull_request: | |
| branches: | |
| - "**" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| meta: | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| matrix_supportedSplunk: ${{ steps.matrix.outputs.latestSplunk }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: matrix | |
| uses: splunk/addonfactory-test-matrix-action@v3.1.2 | |
| fossa-scan: | |
| continue-on-error: true | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: run fossa anlyze and create report | |
| run: | | |
| curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash | |
| fossa analyze --debug | |
| fossa report attribution --format text > /tmp/THIRDPARTY | |
| env: | |
| FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} | |
| - name: upload THIRDPARTY file | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: THIRDPARTY | |
| path: /tmp/THIRDPARTY | |
| - name: run fossa test | |
| run: | | |
| fossa test --debug | |
| env: | |
| FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} | |
| compliance-copyrights: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: apache/skywalking-eyes@v0.6.0 | |
| pre-commit: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - uses: pre-commit/action@v3.0.1 | |
| semgrep: | |
| uses: splunk/sast-scanning/.github/workflows/sast-scan.yml@main | |
| secrets: | |
| SEMGREP_KEY: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} | |
| test-splunk-unit: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Install dependencies | |
| run: | | |
| git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf https://github.com | |
| git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf ssh://git@github.com | |
| curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1 | |
| poetry install | |
| poetry run pytest -v tests/unit | |
| test-splunk-external: | |
| runs-on: ubuntu-22.04 | |
| needs: | |
| - meta | |
| - pre-commit | |
| - fossa-scan | |
| - compliance-copyrights | |
| - test-splunk-unit | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - name: Setup for testing | |
| run: | | |
| mkdir test-results-${{ matrix.splunk.version }} | |
| - name: Test | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} | |
| run: | | |
| export SPLUNK_APP_PACKAGE=./tests/e2e/addons/TA_fiction_indextime | |
| export SPLUNK_ADDON=TA_fiction_indextime | |
| export SPLUNK_APP_ID=TA_fiction_indextime | |
| export SPLUNK_VERSION=${{ matrix.splunk.version }} | |
| export SPLUNK_HEC_TOKEN="9b741d03-43e9-4164-908b-e09102327d22" | |
| echo $SPLUNK_VERSION | |
| docker compose -f "docker-compose-ci.yml" build | |
| SPLUNK_PASSWORD=Chang3d! docker compose -f docker-compose-ci.yml up --abort-on-container-exit | |
| docker volume ls | |
| - name: Collect Results | |
| run: | | |
| docker volume ls | |
| docker container create --name dummy \ | |
| -v pytest-splunk-addon_results:/work/test-results \ | |
| registry.access.redhat.com/ubi7/ubi | |
| docker cp dummy:/work/test-results/test.xml test-results-${{ matrix.splunk.version }} | |
| - uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: splunk ${{ matrix.splunk.version }} external test artifacts | |
| path: | | |
| test-results-${{ matrix.splunk.version }} | |
| test-splunk-matrix: | |
| needs: | |
| - meta | |
| - pre-commit | |
| - fossa-scan | |
| - compliance-copyrights | |
| - test-splunk-unit | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} | |
| test-marker: [ | |
| "splunk_connection_docker", | |
| "splunk_app_fiction", | |
| "splunk_app_broken", | |
| "splunk_app_cim_fiction", | |
| "splunk_app_cim_broken", | |
| "splunk_fiction_indextime", | |
| "splunk_fiction_indextime_broken", | |
| "splunk_setup_fixture", | |
| "splunk_app_req", | |
| "splunk_app_req_broken", | |
| "splunk_cim_model", | |
| "splunk_app_fiction_with_ep", | |
| "splunk_app_req_with_ep", | |
| ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.7 | |
| - run: | | |
| git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf https://github.com | |
| git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf ssh://git@github.com | |
| curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1 | |
| poetry install | |
| poetry run pytest -v --splunk-version=${{ matrix.splunk.version }} -m docker -m ${{ matrix.test-marker }} tests/e2e | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} | |
| # --------------------------------------------------------------------------- | |
| # Validate PSA against multiple splunk-cim-models package versions. | |
| # | |
| # Matrix axes: | |
| # splunk – every supported Splunk version (from meta job) | |
| # cim-models – every package git ref listed below | |
| # | |
| # How to add a new CIM models release: | |
| # 1. Append a new entry to the cim-models list with the git ref (branch | |
| # or tag) and a short human-readable label. | |
| # 2. The matrix will automatically run the splunk_cim_model e2e test | |
| # for each (Splunk version, CIM models version) combination. | |
| # | |
| # Only the splunk_cim_model marker is used here because the other CIM | |
| # e2e markers (splunk_app_cim_fiction, splunk_app_cim_broken) always pass | |
| # --splunk-dm-path with custom fiction models and are therefore unaffected | |
| # by the installed splunk-cim-models version. | |
| # --------------------------------------------------------------------------- | |
| test-cim-models-versions: | |
| needs: | |
| - meta | |
| - pre-commit | |
| - fossa-scan | |
| - compliance-copyrights | |
| - test-splunk-unit | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| splunk: ${{ fromJson(needs.meta.outputs.matrix_supportedSplunk) }} | |
| cim-models: | |
| - ref: "v1" | |
| label: "1.x" | |
| - ref: "v2" | |
| label: "2.x" | |
| # Add future releases here, e.g.: | |
| # - ref: "v3" | |
| # label: "3.x" | |
| name: "CIM models ${{ matrix.cim-models.label }} / Splunk ${{ matrix.splunk.version }}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.7" | |
| - name: Install PSA with CIM models ${{ matrix.cim-models.label }} | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} | |
| run: | | |
| git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf https://github.com | |
| git config --global --add url."https://${{ secrets.GH_TOKEN_ADMIN }}@github.com".insteadOf ssh://git@github.com | |
| curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1 | |
| poetry install | |
| # Override the dev-dependency CIM models with the matrix version. | |
| # Using pip inside the Poetry venv so the pytester subprocess sees | |
| # the same environment. | |
| poetry run pip install \ | |
| "git+https://github.com/splunk/psa-cim-models.git@${{ matrix.cim-models.ref }}" | |
| - name: Verify splunk-cim-models ${{ matrix.cim-models.label }} | |
| run: | | |
| poetry run python - <<'EOF' | |
| import splunk_cim_models as scm | |
| print("DATA_MODELS_PATH:", scm.DATA_MODELS_PATH) | |
| print("CIM versions available:", list(scm.datamodels.keys())) | |
| EOF | |
| - name: Run CIM compatibility e2e tests | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} | |
| run: | | |
| # splunk_app_cim_fiction uses --splunk-dm-path with local fiction | |
| # data models, so expected test outcomes are identical across all | |
| # splunk-cim-models versions. It still exercises all PSA import | |
| # paths (COMMON_FIELDS_PATH, DATAMODEL_SCHEMA_PATH, datamodels) so | |
| # an incompatible package version will surface as a collection or | |
| # runtime error rather than a wrong-field assertion. | |
| # | |
| # splunk_cim_model (real Change/Network_Traffic models) is left to | |
| # test-splunk-matrix, which always uses the default v2 dev | |
| # dependency and asserts version-specific field outcomes. | |
| poetry run pytest -v \ | |
| --splunk-version=${{ matrix.splunk.version }} \ | |
| -m docker \ | |
| -m splunk_app_cim_fiction \ | |
| tests/e2e | |
| publish: | |
| needs: | |
| - test-splunk-external | |
| - test-splunk-matrix | |
| - test-cim-models-versions | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Very Important semantic-release won't trigger a tagged | |
| # build if this is not set false | |
| submodules: false | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.7" | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: THIRDPARTY | |
| - name: Update Notices | |
| run: cp -f THIRDPARTY NOTICE | |
| - name: Install Poetry | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - --version 1.5.1 | |
| - id: semantic | |
| uses: splunk/semantic-release-action@v1.3 | |
| with: | |
| git_committer_name: ${{ secrets.SA_GH_USER_NAME }} | |
| git_committer_email: ${{ secrets.SA_GH_USER_EMAIL }} | |
| gpg_private_key: ${{ secrets.SA_GPG_PRIVATE_KEY }} | |
| passphrase: ${{ secrets.SA_GPG_PASSPHRASE }} | |
| extra_plugins: | | |
| semantic-release-replace-plugin | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_TOKEN_ADMIN }} | |
| - if: ${{ steps.semantic.outputs.new_release_published == 'true' }} | |
| run: | | |
| poetry build | |
| poetry publish -n -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_TOKEN }} |