diff --git a/.github/actions/apt-packages/action.yaml b/.github/actions/apt-packages/action.yaml new file mode 100644 index 0000000000..616f17fea3 --- /dev/null +++ b/.github/actions/apt-packages/action.yaml @@ -0,0 +1,27 @@ +--- +name: Cache and Install APT Dependencies +description: + Light wrapper around the actions/cache action and our script + to maintain the input vars in only one place for all workflows. + +runs: + using: "composite" + steps: + - name: Cache APT Dependencies + id: cache-apt-deps + uses: actions/cache@v4 + with: + path: | + ~/apt_cache + key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} + restore-keys: | + ${{ runner.os }}-v8-apt- + + - name: Install APT Depedencies + shell: bash + env: + CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} + run: | + # install dev dependencies for Python YAML and LDAP packages + # https://github.com/StackStorm/st2-auth-ldap + ./scripts/github/install-apt-packages-use-cache.sh diff --git a/.github/actions/init-pants/action.yaml b/.github/actions/init-pants/action.yaml new file mode 100644 index 0000000000..6f2fd35dfe --- /dev/null +++ b/.github/actions/init-pants/action.yaml @@ -0,0 +1,36 @@ +--- +name: Initialize Pants and its GHA caches +description: + Light wrapper around the pantsbuild/actions/init-pants action + to maintain the input vars in only one place for all workflows. + +inputs: + gha-cache-key: + description: Qualify all cache keys with this string. Useful for invalidating everything. + required: true + +runs: + using: "composite" + steps: + - name: Initialize Pants and its GHA caches + uses: pantsbuild/actions/init-pants@v8 + # This action adds an env var to make pants use both pants.ci.toml & pants.toml. + # This action also creates 3 GHA caches (1 is optional). + # - `pants-setup` has the bootsrapped pants install + # - `pants-named-caches` has pip/wheel and PEX caches + # - `pants-lmdb-store` has the fine-grained process cache. + # If we ever use a remote cache, then we can drop this. + # Otherwise, we may need an additional workflow or job to delete old caches + # if they are not expiring fast enough, and we hit the GHA 10GB per repo max. + with: + base-branch: master + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: ${{ inputs.gha-cache-key }} + # This hash should include all of our lockfiles so that the pip/pex caches + # get invalidated on any transitive dependency update. + named-caches-hash: ${{ hashFiles('lockfiles/*.lock') }} + # enable the optional lmdb_store cache since we're not using remote caching. + cache-lmdb-store: 'true' + # install whatever version of python we need for our in-repo pants-plugins + setup-python-for-plugins: 'true' + diff --git a/.github/actions/setup-python/action.yaml b/.github/actions/setup-python/action.yaml new file mode 100644 index 0000000000..1026f32927 --- /dev/null +++ b/.github/actions/setup-python/action.yaml @@ -0,0 +1,34 @@ +--- +name: Install Python and Cache Deps +description: + Light wrapper around the actions/setup-python and actions/cache actions + to maintain the input vars in only one place for all workflows. + +input: + python-version: + description: Which version of python to install. + required: true + +runs: + using: "composite" + steps: + - name: 'Set up Python (${{ inputs.python-version }})' + uses: actions/setup-python@v5 + with: + python-version: '${{ inputs.python-version }}' + + - name: Cache Python Dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + virtualenv + ~/virtualenv + # TODO: maybe make the virtualenv a partial cache to exclude st2*? + # !virtualenv/lib/python*/site-packages/st2* + # !virtualenv/bin/st2* + key: ${{ runner.os }}-v5-python-${{ inputs.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }} + # Don't use alternative key as if requirements.txt has altered we + # don't want to retrieve previous cache + #restore-keys: | + # ${{ runner.os }}-v5-python-${{ inputs.python }}- diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 160af875a8..12e0b61ff2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,41 +81,12 @@ jobs: - name: Custom Environment Setup run: | ./scripts/github/setup-environment.sh - - name: 'Set up Python (${{ matrix.python-version }})' - uses: actions/setup-python@v5 + - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps' + uses: ./.github/actions/setup-python with: python-version: '${{ matrix.python-version }}' - - name: Cache Python Dependencies - uses: actions/cache@v4 - with: - path: | - ~/.cache/pip - virtualenv - ~/virtualenv - # TODO: maybe make the virtualenv a partial cache to exclude st2*? - # !virtualenv/lib/python*/site-packages/st2* - # !virtualenv/bin/st2* - key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }} - # Don't use alternative key as if requirements.txt has altered we - # don't want to retrieve previous cache - #restore-keys: | - # ${{ runner.os }}-v5-python-${{ matrix.python }}- - - name: Cache APT Dependencies - id: cache-apt-deps - uses: actions/cache@v4 - with: - path: | - ~/apt_cache - key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-v8-apt- - - name: Install APT Depedencies - env: - CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - # install dev dependencies for Python YAML and LDAP packages - # https://github.com/StackStorm/st2-auth-ldap - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Install virtualenv run: | ./scripts/github/install-virtualenv.sh @@ -180,40 +151,12 @@ jobs: - name: Custom Environment Setup run: | ./scripts/github/setup-environment.sh - - name: 'Set up Python (${{ matrix.python-version }})' - uses: actions/setup-python@v5 + - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps' + uses: ./.github/actions/setup-python with: python-version: '${{ matrix.python-version }}' - - name: Cache Python Dependencies - uses: actions/cache@v4 - with: - path: | - ~/.cache/pip - virtualenv - ~/virtualenv - # TODO: maybe make the virtualenv a partial cache to exclude st2*? - # !virtualenv/lib/python*/site-packages/st2* - # !virtualenv/bin/st2* - key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }} - restore-keys: | - ${{ runner.os }}-python-${{ matrix.python }}- - - name: Cache APT Dependencies - id: cache-apt-deps - uses: actions/cache@v4 - with: - path: | - ~/apt_cache - key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-v8-apt- - - name: Install APT Depedencies - env: - CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - cat /etc/environment - # install dev dependencies for Python YAML and LDAP packages - # https://github.com/StackStorm/st2-auth-ldap - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Install virtualenv run: | ./scripts/github/install-virtualenv.sh @@ -384,41 +327,12 @@ jobs: - name: Custom Environment Setup run: | ./scripts/github/setup-environment.sh - - name: 'Set up Python (${{ matrix.python-version }})' - uses: actions/setup-python@v5 + - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps' + uses: ./.github/actions/setup-python with: python-version: '${{ matrix.python-version }}' - - name: Cache Python Dependencies - uses: actions/cache@v4 - with: - path: | - ~/.cache/pip - virtualenv - ~/virtualenv - # TODO: maybe make the virtualenv a partial cache to exclude st2*? - # !virtualenv/lib/python*/site-packages/st2* - # !virtualenv/bin/st2* - key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }} - # Don't use alternative key as if requirements.txt has altered we - # don't want to retrieve previous cache - #restore-keys: | - # ${{ runner.os }}-v5-python-${{ matrix.python }}- - - name: Cache APT Dependencies - id: cache-apt-deps - uses: actions/cache@v4 - with: - path: | - ~/apt_cache - key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-v8-apt- - - name: Install APT Depedencies - env: - CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - # install dev dependencies for Python YAML and LDAP packages - # https://github.com/StackStorm/st2-auth-ldap - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Install virtualenv run: | ./scripts/github/install-virtualenv.sh @@ -604,41 +518,12 @@ jobs: - name: Custom Environment Setup run: | ./scripts/github/setup-environment.sh - - name: 'Set up Python (${{ matrix.python-version }})' - uses: actions/setup-python@v5 + - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps' + uses: ./.github/actions/setup-python with: python-version: '${{ matrix.python-version }}' - - name: Cache Python Dependencies - uses: actions/cache@v4 - with: - path: | - ~/.cache/pip - virtualenv - ~/virtualenv - # TODO: maybe make the virtualenv a partial cache to exclude st2*? - # !virtualenv/lib/python*/site-packages/st2* - # !virtualenv/bin/st2* - key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }} - # Don't use alternative key as if requirements.txt has altered we - # don't want to retrieve previous cache - #restore-keys: | - # ${{ runner.os }}-v5-python-${{ matrix.python }}- - - name: Cache APT Dependencies - id: cache-apt-deps - uses: actions/cache@v4 - with: - path: | - ~/apt_cache - key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-v8-apt- - - name: Install APT Depedencies - env: - CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - # install dev dependencies for Python YAML and LDAP packages - # https://github.com/StackStorm/st2-auth-ldap - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Install virtualenv run: | ./scripts/github/install-virtualenv.sh @@ -709,6 +594,7 @@ jobs: if: always() needs: - lint-checks + - self-check - unit-tests - integration-tests runs-on: ubuntu-20.04 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 851cb3e1c0..8d0eff683a 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -38,45 +38,14 @@ jobs: # a test uses a submodule, and pants needs access to it to calculate deps. submodules: 'true' - #- name: Cache APT Dependencies - # id: cache-apt-deps - # uses: actions/cache@v4 - # with: - # path: | - # ~/apt_cache - # key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - # restore-keys: | - # ${{ runner.os }}-v8-apt- - - name: Install APT Depedencies - env: - CACHE_HIT: 'false' # cache doesn't work - #CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - # install dev dependencies for Python YAML and LDAP packages - # https://github.com/StackStorm/st2-auth-ldap - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Initialize Pants and its GHA caches - uses: pantsbuild/actions/init-pants@v8 - # This action adds an env var to make pants use both pants.ci.toml & pants.toml. - # This action also creates 3 GHA caches (1 is optional). - # - `pants-setup` has the bootsrapped pants install - # - `pants-named-caches` has pip/wheel and PEX caches - # - `pants-lmdb-store` has the fine-grained process cache. - # If we ever use a remote cache, then we can drop this. - # Otherwise, we may need an additional workflow or job to delete old caches - # if they are not expiring fast enough, and we hit the GHA 10GB per repo max. + uses: ./.github/actions/init-pants with: - base-branch: master # To ignore a bad cache, bump the cache* integer. gha-cache-key: cache0 - # This hash should include all of our lockfiles so that the pip/pex caches - # get invalidated on any transitive dependency update. - named-caches-hash: ${{ hashFiles('requirements.txt') }} - # enable the optional lmdb_store cache since we're not using remote caching. - cache-lmdb-store: 'true' - # install whatever version of python we need for our in-repo pants-plugins - setup-python-for-plugins: 'true' - name: Lint run: | diff --git a/.github/workflows/microbenchmarks.yaml b/.github/workflows/microbenchmarks.yaml index 5aa26ab4c5..667a252a94 100644 --- a/.github/workflows/microbenchmarks.yaml +++ b/.github/workflows/microbenchmarks.yaml @@ -75,36 +75,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - - name: 'Set up Python (${{ matrix.python-version }})' - uses: actions/setup-python@v5 + - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps' + uses: ./.github/actions/setup-python with: python-version: '${{ matrix.python-version }}' - - name: Cache Python Dependencies - uses: actions/cache@v4 - with: - path: | - ~/.cache/pip - virtualenv - ~/virtualenv - key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }} - # Don't use alternative key as if requirements.txt has altered we - # don't want to retrieve previous cache - #restore-keys: | - # ${{ runner.os }}-v5-python-${{ matrix.python }}- - - name: Cache APT Dependencies - id: cache-apt-deps - uses: actions/cache@v4 - with: - path: | - ~/apt_cache - key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-v8-apt- - - name: Install APT Dependencies - env: - CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Install virtualenv run: | ./scripts/github/install-virtualenv.sh @@ -122,7 +98,7 @@ jobs: - name: Upload Histograms uses: actions/upload-artifact@v4 with: - name: benchmark_histograms + name: benchmark_histograms-py${{ matrix.python-version }} path: benchmark_histograms/ retention-days: 30 diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml index d7f54a4715..fe3e855fc3 100644 --- a/.github/workflows/orquesta-integration-tests.yaml +++ b/.github/workflows/orquesta-integration-tests.yaml @@ -127,41 +127,12 @@ jobs: - name: Custom Environment Setup run: | ./scripts/github/setup-environment.sh - - name: 'Set up Python (${{ matrix.python-version }})' - uses: actions/setup-python@v5 + - name: 'Set up Python (${{ matrix.python-version }}) and Cache Deps' + uses: ./.github/actions/setup-python with: python-version: '${{ matrix.python-version }}' - - name: Cache Python Dependencies - uses: actions/cache@v4 - with: - path: | - ~/.cache/pip - virtualenv - ~/virtualenv - # TODO: maybe make the virtualenv a partial cache to exclude st2*? - # !virtualenv/lib/python*/site-packages/st2* - # !virtualenv/bin/st2* - key: ${{ runner.os }}-v5-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt', 'lockfiles/*.lock') }} - # Don't use alternative key as if requirements.txt has altered we - # don't want to retrieve previous cache - #restore-keys: | - # ${{ runner.os }}-v5-python-${{ matrix.python }}- - - name: Cache APT Dependencies - id: cache-apt-deps - uses: actions/cache@v4 - with: - path: | - ~/apt_cache - key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - restore-keys: | - ${{ runner.os }}-v8-apt- - - name: Install APT Depedencies - env: - CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - # install dev dependencies for Python YAML and LDAP packages - # https://github.com/StackStorm/st2-auth-ldap - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Install virtualenv run: | ./scripts/github/install-virtualenv.sh @@ -214,6 +185,7 @@ jobs: - name: Compress Service Logs Before upload if: ${{ failure() }} run: | + ./tools/launchdev.sh stop # stop st2 before collecting logs tar cvzpf logs.tar.gz logs/* - name: Upload StackStorm services Logs if: ${{ failure() }} diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index d695aefc23..0d3e85be81 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -30,26 +30,10 @@ jobs: submodules: 'true' - name: Initialize Pants and its GHA caches - uses: pantsbuild/actions/init-pants@v8 - # This action adds an env var to make pants use both pants.ci.toml & pants.toml. - # This action also creates 3 GHA caches (1 is optional). - # - `pants-setup` has the bootsrapped pants install - # - `pants-named-caches` has pip/wheel and PEX caches - # - `pants-lmdb-store` has the fine-grained process cache. - # If we ever use a remote cache, then we can drop this. - # Otherwise, we may need an additional workflow or job to delete old caches - # if they are not expiring fast enough, and we hit the GHA 10GB per repo max. + uses: ./.github/actions/init-pants with: - base-branch: master # To ignore a bad cache, bump the cache* integer. gha-cache-key: cache0-BUILD - # This hash should include all of our lockfiles so that the pip/pex caches - # get invalidated on any transitive dependency update. - named-caches-hash: ${{ hashFiles('requirements.txt') }} - # enable the optional lmdb_store cache since we're not using remote caching. - cache-lmdb-store: 'true' - # install whatever version of python we need for our in-repo pants-plugins - setup-python-for-plugins: 'true' - name: Check BUILD files run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8d7e94a6b4..3f2145828c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -83,45 +83,14 @@ jobs: python-version: '${{ matrix.python-version }}' - #- name: Cache APT Dependencies - # id: cache-apt-deps - # uses: actions/cache@v4 - # with: - # path: | - # ~/apt_cache - # key: ${{ runner.os }}-v8-apt-${{ hashFiles('scripts/github/apt-packages.txt') }} - # restore-keys: | - # ${{ runner.os }}-v8-apt- - - name: Install APT Depedencies - env: - CACHE_HIT: 'false' # cache doesn't work - #CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} - run: | - # install dev dependencies for Python YAML and LDAP packages - # https://github.com/StackStorm/st2-auth-ldap - ./scripts/github/install-apt-packages-use-cache.sh + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages - name: Initialize Pants and its GHA caches - uses: pantsbuild/actions/init-pants@v8 - # This action adds an env var to make pants use both pants.ci.toml & pants.toml. - # This action also creates 3 GHA caches (1 is optional). - # - `pants-setup` has the bootsrapped pants install - # - `pants-named-caches` has pip/wheel and PEX caches - # - `pants-lmdb-store` has the fine-grained process cache. - # If we ever use a remote cache, then we can drop this. - # Otherwise, we may need an additional workflow or job to delete old caches - # if they are not expiring fast enough, and we hit the GHA 10GB per repo max. + uses: ./.github/actions/init-pants with: - base-branch: master # To ignore a bad cache, bump the cache* integer. gha-cache-key: cache0-py${{ matrix.python-version }} - # This hash should include all of our lockfiles so that the pip/pex caches - # get invalidated on any transitive dependency update. - named-caches-hash: ${{ hashFiles('requirements.txt') }} - # enable the optional lmdb_store cache since we're not using remote caching. - cache-lmdb-store: 'true' - # install whatever version of python we need for our in-repo pants-plugins - setup-python-for-plugins: 'true' - name: Test # We do not support running pytest everywhere yet. When we do it will be simply: