diff --git a/.github/action.yml b/.github/action.yml index 99ea473c1..e9608c298 100644 --- a/.github/action.yml +++ b/.github/action.yml @@ -59,6 +59,7 @@ runs: shell: bash if: ${{ !fromJson(inputs.no-start) }} run: | + docker compose version REGISTRY=${{ inputs.registry }} \ REPO=${{ inputs.repo }} \ COMPOSE_PROJECT_NAME=${{ env.project-number }} \ diff --git a/.github/workflows/container-ci.yaml b/.github/workflows/container-ci.yaml index bbb8707f7..2b48e16c5 100644 --- a/.github/workflows/container-ci.yaml +++ b/.github/workflows/container-ci.yaml @@ -85,7 +85,7 @@ jobs: build-containers: needs: [setup-build] env: ${{ matrix }} - runs-on: [self-hosted, Linux, X64, clx] + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} strategy: matrix: ${{ fromJson(needs.setup-build.outputs.matrix) }} fail-fast: false @@ -111,7 +111,7 @@ jobs: - name: Build Container Group if: ${{ !inputs.no_build }} id: build-group - uses: intel/ai-containers/.github@9ecbccaae1fd9006f3e5c2e6ce4145a0d6ece76f + uses: intel/ai-containers/.github@sramakr1/add_pytorch_test_check with: group_dir: ${{ inputs.group_dir }} env_overrides: ${{ inputs.env_overrides || env.env_overrides || '' }} @@ -197,56 +197,57 @@ jobs: #################################################################################################### # Generic Test Runner #################################################################################################### - # setup-test: - # needs: [build-containers] - # runs-on: ubuntu-latest - # outputs: - # matrix: ${{ steps.test-matrix.outputs.matrix }} - # steps: - # - name: Harden Runner - # uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - # with: - # egress-policy: audit - # - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - # with: - # persist-credentials: false - # - name: Get Recipes - # id: test-matrix - # run: echo "matrix=$(find ${INPUTS_GROUP_DIR} -type f -name 'tests.yaml' -exec dirname {} \; | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT - # env: - # INPUTS_GROUP_DIR: ${{ inputs.group_dir }} - # test-containers: - # needs: [setup-build, setup-test] - # if: ${{ needs.setup-test.outputs.matrix != '[]' }} - # runs-on: [self-hosted, Linux, "${{ inputs.runner_label || fromJson(needs.setup-build.outputs.matrix).runner_label }}"] - # strategy: - # matrix: - # tests: ${{ fromJson(needs.setup-test.outputs.matrix) }} - # experimental: [true] - # fail-fast: false - # steps: - # - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - # with: - # egress-policy: audit - # - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - # with: - # persist-credentials: false - # - uses: azure/docker-login@15c4aadf093404726ab2ff205b2cdd33fa6d054c # v2 - # with: - # login-server: ${{ secrets.REGISTRY }} - # username: ${{ secrets.REGISTRY_USER }} - # password: ${{ secrets.REGISTRY_TOKEN }} - # # - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - # # with: - # # registry: ${{ secrets.REGISTRY }} - # # username: ${{ secrets.REGISTRY_USER }} - # # password: ${{ secrets.REGISTRY_TOKEN }} - # - name: Test Container Group - # uses: intel/ai-containers/test-runner@b47fdb7521ea71f718f582ed6dc1de488b546d6c - # with: - # cache_registry: ${{ secrets.CACHE_REGISTRY }} - # recipe_dir: ${{ inputs.group_dir }} - # registry: ${{ secrets.REGISTRY }} - # repo: ${{ secrets.REPO }} - # test_dir: ${{ matrix.tests }} - # token: ${{ secrets.GITHUB_TOKEN }} + setup-test: + needs: [build-containers] + if: ${{ inputs.group_dir != 'pytorch' }} + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.test-matrix.outputs.matrix }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + with: + egress-policy: audit + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - name: Get Recipes + id: test-matrix + run: echo "matrix=$(find ${INPUTS_GROUP_DIR} -type f -name 'tests.yaml' -exec dirname {} \; | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT + env: + INPUTS_GROUP_DIR: ${{ inputs.group_dir }} + test-containers: + needs: [setup-build, setup-test] + if: ${{ needs.setup-test.outputs.matrix != '[]' && inputs.group_dir != 'pytorch'}} + runs-on: [self-hosted, Linux, "${{ inputs.runner_label || fromJson(needs.setup-build.outputs.matrix).runner_label }}"] + strategy: + matrix: + tests: ${{ fromJson(needs.setup-test.outputs.matrix) }} + experimental: [true] + fail-fast: false + steps: + - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + with: + egress-policy: audit + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - uses: azure/docker-login@15c4aadf093404726ab2ff205b2cdd33fa6d054c # v2 + with: + login-server: ${{ secrets.REGISTRY }} + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_TOKEN }} + # - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + # with: + # registry: ${{ secrets.REGISTRY }} + # username: ${{ secrets.REGISTRY_USER }} + # password: ${{ secrets.REGISTRY_TOKEN }} + - name: Test Container Group + uses: intel/ai-containers/test-runner@b47fdb7521ea71f718f582ed6dc1de488b546d6c + with: + cache_registry: ${{ secrets.CACHE_REGISTRY }} + recipe_dir: ${{ inputs.group_dir }} + registry: ${{ secrets.REGISTRY }} + repo: ${{ secrets.REPO }} + test_dir: ${{ matrix.tests }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml index d4c996cca..3699825b2 100644 --- a/.github/workflows/integration-test.yaml +++ b/.github/workflows/integration-test.yaml @@ -68,74 +68,74 @@ jobs: group: ${{ fromJson(needs.group-diff.outputs.groups) }} experimental: [true] fail-fast: false - uses: intel/ai-containers/.github/workflows/container-ci.yaml@main + uses: intel/ai-containers/.github/workflows/container-ci.yaml@sramakr1/add_pytorch_test_check with: group_dir: ${{ matrix.group }} secrets: inherit - # merge-logs: - # # download all artifacts across containers - # needs: [pipeline-ci] - # if: success() || failure() - # runs-on: ubuntu-latest - # permissions: - # pull-requests: write - # steps: - # - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.0 - # id: download_artifact_outputs - # with: - # pattern: test-runner-summary* - # merge-multiple: true - # - name: Find Summary - # id: summary - # shell: bash - # run: | - # SUMMARY=$(find . -maxdepth 1 -name '*summary.json' -print) - # if [[ -n "$SUMMARY" ]]; then - # echo "summary=true" >> $GITHUB_OUTPUT - # echo "Files matching the pattern ./*summary.json" - # jq -s '[.[] | .[]]' ./*summary.json > combined.json - # echo "Files found in the directory" - # else - # echo "summary=false" >> $GITHUB_OUTPUT - # echo "No files matching the pattern ./*summary.json" - # fi - # - name: Generate TXT file - # if: ${{ steps.summary.outputs.summary != 'false' }} - # run: | - # { - # echo "### Integration Test Results" - # echo "Groups Tested: $(jq -r 'map(.Group) | unique | join(", ")' combined.json)" - # echo -e "\n
" - # echo -e " Results\n" - # echo " | Test-Group | Test | Status |" - # echo " |:----:|:---:|:---:|" - # jq -r '.[] | " | \(.Group) | \(.Test) | \(.Status) |"' combined.json - # echo -e "\n
\n" - # if jq -e 'all(.[]; .Status == "PASS")' combined.json > /dev/null; then - # echo "#### Overall Result: PASS ✅" - # else - # echo "#### Overall Result: FAIL ❌" - # fi - # } >> output.txt - # - name: PR-comment - # if: ${{ steps.summary.outputs.summary != 'false' }} - # uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 - # with: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # number: ${{ github.event.pull_request.number }} - # header: test-runner combined summary - # path: output.txt - # recreate: true - # status-check: - # needs: [group-diff, pipeline-ci] - # runs-on: ubuntu-latest - # if: always() - # steps: - # - run: exit 1 - # if: >- - # ${{ - # contains(needs.*.result, 'failure') - # || contains(needs.*.result, 'cancelled') - # || contains(needs.*.result, 'skipped') - # && needs.group-diff.outputs.groups != '[""]' - # }} + merge-logs: + # download all artifacts across containers + needs: [group-diff, pipeline-ci] + if: success() || failure() && needs.group-diff.outputs.groups != 'pytorch' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v4.0 + id: download_artifact_outputs + with: + pattern: test-runner-summary* + merge-multiple: true + - name: Find Summary + id: summary + shell: bash + run: | + SUMMARY=$(find . -maxdepth 1 -name '*summary.json' -print) + if [[ -n "$SUMMARY" ]]; then + echo "summary=true" >> $GITHUB_OUTPUT + echo "Files matching the pattern ./*summary.json" + jq -s '[.[] | .[]]' ./*summary.json > combined.json + echo "Files found in the directory" + else + echo "summary=false" >> $GITHUB_OUTPUT + echo "No files matching the pattern ./*summary.json" + fi + - name: Generate TXT file + if: ${{ steps.summary.outputs.summary != 'false' }} + run: | + { + echo "### Integration Test Results" + echo "Groups Tested: $(jq -r 'map(.Group) | unique | join(", ")' combined.json)" + echo -e "\n
" + echo -e " Results\n" + echo " | Test-Group | Test | Status |" + echo " |:----:|:---:|:---:|" + jq -r '.[] | " | \(.Group) | \(.Test) | \(.Status) |"' combined.json + echo -e "\n
\n" + if jq -e 'all(.[]; .Status == "PASS")' combined.json > /dev/null; then + echo "#### Overall Result: PASS ✅" + else + echo "#### Overall Result: FAIL ❌" + fi + } >> output.txt + - name: PR-comment + if: ${{ steps.summary.outputs.summary != 'false' }} + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + number: ${{ github.event.pull_request.number }} + header: test-runner combined summary + path: output.txt + recreate: true + status-check: + needs: [group-diff, pipeline-ci] + runs-on: ubuntu-latest + if: always() + steps: + - run: exit 1 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + && needs.group-diff.outputs.groups != '[""]' + }} diff --git a/python/requirements.txt b/python/requirements.txt index 3ee2bbab4..212df3533 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,6 +1,6 @@ numpy==1.26.4 setuptools>=70.0.0 -psutil==7.0.0 +psutil==7.1.0 mkl==2025.2.0 mkl-include==2025.2.0 intel-openmp==2025.2.1 diff --git a/pytorch/docker-compose.yaml b/pytorch/docker-compose.yaml index a2c07f350..a1b2eec1b 100644 --- a/pytorch/docker-compose.yaml +++ b/pytorch/docker-compose.yaml @@ -13,8 +13,7 @@ # limitations under the License. include: - - path: - - ../python/docker-compose.yaml + - path: ../python/docker-compose.yaml services: ipex-base: build: