sv-tests-ci #1832
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: sv-tests-ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 1 * * *' # run daily at 01:00 am (UTC) | |
| jobs: | |
| Run: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tool: | |
| - name: icarus | |
| deps: autoconf autotools-dev bison flex libfl-dev gperf | |
| - name: moore | |
| rust_ver: "1.61" | |
| - name: odin | |
| repo: odin_ii | |
| deps: autoconf autotools-dev bison flex libfl-dev cmake pkg-config | |
| - name: slang | |
| deps: cmake pkg-config | |
| - name: surelog | |
| repo: Surelog | |
| submodules: third_party/UHDM third_party/antlr4 third_party/googletest | |
| deps: cmake default-jre pkg-config tclsh uuid-dev | |
| - name: sv-parser | |
| deps: cargo | |
| rust_ver: "1.81" | |
| - name: tree-sitter-systemverilog | |
| deps: gcc | |
| - name: tree-sitter-verilog | |
| deps: gcc | |
| - name: verible | |
| deps: bazel=7.6.1 bison flex libfl-dev | |
| skip-ccache: 1 | |
| - name: verilator | |
| deps: autoconf autotools-dev bison flex help2man libfl-dev libelf-dev | |
| make_jobs_max: 4 | |
| - name: yosys | |
| deps: bison clang tcl-dev flex libfl-dev pkg-config libreadline-dev | |
| - name: yosys-synlig | |
| repo: synlig | |
| submodules: third_party/yosys third_party/surelog | |
| deps: cmake clang tcl-dev bison default-jre flex libfl-dev libreadline-dev pkg-config tclsh uuid-dev | |
| runners_filter: SynligYosys | |
| - name: zachjs-sv2v | |
| deps: haskell-stack | |
| - name: yosys-slang | |
| deps: bison clang tcl-dev flex libfl-dev pkg-config libreadline-dev cmake pkg-config | |
| submodules: third_party/slang | |
| - name: circt-verilog | |
| deps: cmake clang ninja-build lld | |
| submodules: llvm | |
| env: | |
| RUNNERS_FILTER: ${{ matrix.tool.runners_filter }} | |
| CCACHE_DIR: "/root/sv-tests/sv-tests/.cache/" | |
| # those generators can use a lot of RAM/cpu and starve other tests | |
| # tests from those generators are run without "-j" flag | |
| BIG_GENERATORS: "fusesoc black-parrot" | |
| DEBIAN_FRONTEND: "noninteractive" | |
| GHA_MACHINE_TYPE: "n2-highmem-16" | |
| name: ${{ matrix.tool.name }} | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| container: ubuntu:jammy-20221130 | |
| steps: | |
| - name: Cancel previous | |
| uses: styfle/cancel-workflow-action@0.8.0 | |
| with: | |
| access_token: ${{ github.token }} | |
| - name: Show shell | |
| run: | | |
| echo "$SHELL" | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Update apt repos | |
| run: | | |
| apt-get update -qq | |
| - name: Setup Node | |
| run: | | |
| apt -qq -y install curl | |
| touch "$HOME/.bash_profile" | |
| # From https://nodejs.org/en/download | |
| curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash | |
| . "$HOME/.nvm/nvm.sh" | |
| nvm install 22 | |
| - name: Add bazel repo (if needed) | |
| if: ${{ contains(matrix.tool.deps, 'bazel') }} | |
| run: | | |
| apt -qq -y install apt-transport-https curl gnupg | |
| curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg | |
| chmod a+r bazel.gpg | |
| mv bazel.gpg /etc/apt/trusted.gpg.d/ | |
| echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list | |
| apt-get update -qq | |
| - name: Install common dependencies | |
| run: | | |
| apt install -y python3 python3-pip git wget file | |
| update-alternatives --install /usr/bin/python python /usr/bin/python3 1 | |
| update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 | |
| pip install --upgrade setuptools | |
| pip install -r conf/requirements.txt | |
| - name: Install VeeR dependencies | |
| run: | | |
| apt install -y cpanminus | |
| cpanm Bit::Vector JSON | |
| - name: Install ccache | |
| if: ${{ !matrix.tool.skip-ccache }} | |
| run: | | |
| apt install -y ccache | |
| - name: Install tool-specific dependencies | |
| if: ${{ matrix.tool.deps }} | |
| run: | | |
| apt -qq -y install ${{ matrix.tool.deps }} | |
| - name: Setup Rust (if needed) | |
| if: ${{ matrix.tool.rust_ver }} | |
| run: | | |
| apt -y install curl | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > setup.sh | |
| sh setup.sh -y | |
| source $HOME/.cargo/env | |
| rustup install ${{ matrix.tool.rust_ver }} | |
| rustup default ${{ matrix.tool.rust_ver }} | |
| - name: Update haskell (if needed) | |
| if: ${{ contains(matrix.tool.deps, 'haskell') }} | |
| run: | | |
| stack upgrade | |
| - name: Checkout the tool submodule | |
| run: | | |
| # Github dropped support for unauthorized git: https://github.blog/2021-09-01-improving-git-protocol-security-github/ | |
| # Make sure we always use https:// instead of git:// | |
| git config --global url.https://github.com/.insteadOf git://github.com/ | |
| # take verilator from github (some tested tools still use a submodule | |
| # from veripool) | |
| git config --global url."https://github.com/verilator/verilator".insteadOf http://git.veripool.org/git/verilator | |
| REPOSITORY_NAME=${{ matrix.tool.name }} | |
| if [[ ! -z "${{ matrix.tool.repo }}" ]]; then | |
| REPOSITORY_NAME=${{ matrix.tool.repo }} | |
| fi | |
| git submodule update --init --depth 1 third_party/tools/${REPOSITORY_NAME} | |
| if [[ ! -z "${{ matrix.tool.submodules }}" ]]; then | |
| pushd third_party/tools/${REPOSITORY_NAME} | |
| git submodule update --init --recursive --depth 1 ${{ matrix.tool.submodules }} | |
| popd | |
| fi | |
| # yosys tool contains tests, is a dependency of other tools | |
| git submodule update --init --recursive --depth 1 third_party/tools/yosys | |
| # icarus contains tests | |
| git submodule update --init --depth 1 third_party/tools/icarus | |
| - name: Create Cache Timestamp | |
| id: cache_timestamp | |
| uses: nanzm/get-time-action@v1.1 | |
| with: | |
| format: 'YYYY-MM-DD-HH-mm-ss' | |
| - name: Setup cache | |
| uses: actions/cache@v4 | |
| timeout-minutes: 3 | |
| continue-on-error: true | |
| with: | |
| path: "/root/sv-tests/sv-tests/.cache/" | |
| key: cache_${{ matrix.tool.name }}_${{ steps.cache_timestamp.outputs.time }} | |
| restore-keys: cache_${{ matrix.tool.name }}_ | |
| - name: Build | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| [[ -f $HOME/.cargo/env ]] && source $HOME/.cargo/env | |
| . "$HOME/.bash_profile" # GitHub Actions runs bash with --noprofile | |
| make ${{ matrix.tool.name }} -j$(nproc) | |
| - name: Check | |
| run: | |
| make info | |
| - name: Checkout third party tests and cores | |
| run: | | |
| git submodule update --init --recursive --depth 1 third_party/tests | |
| git submodule update --init --recursive --depth 1 third_party/cores | |
| - name: Run | |
| run: | | |
| BIG_GENERATORS_EXPR=$(echo $BIG_GENERATORS | sed 's/ /\\|/g') | |
| export STABLE_GENERATORS=$(make list-generators | tr ' ' '\n' | grep -v "${BIG_GENERATORS_EXPR}") | |
| export UNSTABLE_GENERATORS=$(make list-generators | tr ' ' '\n' | grep "${BIG_GENERATORS_EXPR}") | |
| export JOBS=${{ matrix.tool.make_jobs_max }} | |
| if [[ "${JOBS}x" == "x" ]]; then export JOBS=$(nproc); fi | |
| for gen in ${STABLE_GENERATORS}; do echo "==GEN stable $gen -j $JOBS" ; make generate-$gen -j${JOBS}; make -j ${JOBS}; done | |
| for gen in ${UNSTABLE_GENERATORS}; do echo "==GEN unstable $gen" ; make generate-$gen; make; done | |
| - name: Prepare Report | |
| run: | |
| mv out/report/report.csv out/report/${{ matrix.tool.name }}_report.csv | |
| - name: Pack results | |
| run: | | |
| tar -cvf out_${{ matrix.tool.name }}.tar ./out/report/${{ matrix.tool.name }}_report.csv ./out/logs/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: report_${{ matrix.tool.name }} | |
| path: | | |
| out_${{ matrix.tool.name }}.tar | |
| **/plot_*.svg | |
| Summary: | |
| name: Summary | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| container: ubuntu:jammy-20221130 | |
| needs: Run | |
| env: | |
| ANALYZER: "$PWD/tools/report_analyzer.py" | |
| GRAPHER: "$PWD/tools/history-graph" | |
| OUT_REPORT_DIR: "$PWD/out/report/" | |
| COMPARE_REPORT: "$OUT_REPORT_DIR/report.csv" | |
| REPORTS_HISTORY: "$(mktemp -d --suffix='.history')" | |
| BASE_REPORT: "$REPORTS_HISTORY/report.csv" | |
| CHANGES_SUMMARY_JSON: "$OUT_REPORT_DIR/tests_summary.json" | |
| CHANGES_SUMMARY_MD: "$OUT_REPORT_DIR/tests_summary.md" | |
| TESTS_SUMMARY_DIR: "tests_summary/" | |
| DEBIAN_FRONTEND: "noninteractive" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Setup python | |
| run: | | |
| apt-get update -qq | |
| apt install -y python3 python3-pip wget git curl jq | |
| update-alternatives --install /usr/bin/python python /usr/bin/python3 1 | |
| update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 | |
| pip install --upgrade setuptools | |
| pip install -r conf/requirements.txt | |
| - name: Install VeeR dependencies | |
| run: | | |
| apt install -y cpanminus | |
| cpanm Bit::Vector JSON | |
| - name: Prepare output directories | |
| run: | | |
| mkdir -p out/report | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: ./out/ | |
| - name: Extract | |
| run: | | |
| for file in $(find out/ -name *.tar -print); do tar -xf $file --strip-components=2 -C $(dirname $file); done | |
| - name: Checkout third party tests and cores | |
| run: | | |
| # take verilator from github | |
| git config --global url."https://github.com/verilator/verilator".insteadOf http://git.veripool.org/git/verilator | |
| git submodule update --init --recursive --depth 1 third_party/tests | |
| git submodule update --init --recursive --depth 1 third_party/cores | |
| # yosys tool also contains tests | |
| git submodule update --init --recursive --depth 1 third_party/tools/yosys | |
| # icarus contains tests | |
| git submodule update --init --depth 1 third_party/tools/icarus | |
| - name: Summary | |
| run: | | |
| ./.github/workflows/summary.sh | |
| ./.github/workflows/report.sh | |
| - name: Update sv-tests-results repository | |
| if: github.ref == 'refs/heads/master' | |
| run: | | |
| eval $(ssh-agent -s) | |
| ssh-add - <<< "${{ secrets.REPORT_DEPLOY_KEY }}" | |
| mkdir -p ~/.ssh | |
| chmod 700 ~/.ssh | |
| ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts | |
| ./.github/workflows/update_report.sh | |
| - name: Prepare artifacts for PR commenter | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| mkdir $TESTS_SUMMARY_DIR | |
| echo ${{ github.event.number }} > $TESTS_SUMMARY_DIR/issue_num | |
| cp $COMPARE_REPORT $TESTS_SUMMARY_DIR/report_base.csv | |
| cp $OUT_REPORT_DIR/report.csv $TESTS_SUMMARY_DIR/report_new.csv | |
| cp $OUT_REPORT_DIR/new_*csv $TESTS_SUMMARY_DIR | |
| cp $OUT_REPORT_DIR/tests_summary.json $TESTS_SUMMARY_DIR | |
| cp $OUT_REPORT_DIR/tests_summary.md $TESTS_SUMMARY_DIR | |
| find out -name plot_*.svg -exec cp {} $TESTS_SUMMARY_DIR \; | |
| - name: Post GitHub summary | |
| run: | | |
| cat $CHANGES_SUMMARY_MD > $GITHUB_STEP_SUMMARY | |
| - name: Upload artifacts for summary | |
| uses: actions/upload-artifact@v4 | |
| if: github.event_name == 'pull_request' | |
| with: | |
| name: tests_summary | |
| path: | | |
| ./tests_summary/ | |
| - name: Find artifacts that are no longer needed | |
| id: get-artifacts-to-delete | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| artifacts=$(find ./out -type d -name 'report_*' -exec basename {} \;) | |
| echo $artifacts | |
| artifacts="${artifacts//'%'/'%25'}" | |
| artifacts="${artifacts//$'\n'/'%0A'}" | |
| artifacts="${artifacts//$'\r'/'%0D'}" | |
| echo "artifacts=$artifacts" >> GITHUB_OUTPUT | |
| echo $artifacts | |
| - name: Delete Old Artifacts | |
| if: github.event_name == 'pull_request' | |
| uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: ${{ steps.get-artifacts-to-delete.outputs.artifacts }} | |
| Automerge: | |
| name: Automerge dependabot pull requests | |
| permissions: | |
| pull-requests: write | |
| contents: write | |
| runs-on: ubuntu-latest | |
| needs: Summary | |
| if: ${{ github.event_name == 'pull_request' && github.event.pull_request.user.login == 'dependabot[bot]' }} | |
| steps: | |
| - name: Dependabot metadata | |
| id: metadata | |
| uses: dependabot/fetch-metadata@v1.1.1 | |
| with: | |
| github-token: "${{ secrets.GITHUB_TOKEN }}" | |
| - name: Enable auto-merge for Dependabot PRs | |
| run: gh pr merge --auto --merge "$PR_URL" | |
| env: | |
| PR_URL: ${{ github.event.pull_request.html_url }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |