Skip to content

Commit 4048ba3

Browse files
SS-JIASS-JIA
committed
[DO NOT LAND] Try to debug Mac CI failure
ghstack-source-id: de627a9 Pull Request resolved: #5844
1 parent 20a157f commit 4048ba3

File tree

3 files changed

+389
-0
lines changed

3 files changed

+389
-0
lines changed

‎.github/actions/setup-miniconda/action.yml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
name: Set up conda environment for testing
2+
3+
description: Clean workspace and check out PyTorch
4+
5+
inputs:
6+
python-version:
7+
description: If set to any value, dont use sudo to clean the workspace
8+
required: false
9+
type: string
10+
default: "3.9"
11+
miniconda-version:
12+
description: Miniconda version to install
13+
required: false
14+
type: string
15+
default: "23.1.0-1"
16+
environment-file:
17+
description: Environment file to install dependencies from
18+
required: false
19+
type: string
20+
default: ""
21+
pip-requirements-file:
22+
description: An optional pip requirements file to be installed in the conda environment
23+
required: false
24+
type: string
25+
default: ""
26+
27+
runs:
28+
using: composite
29+
steps:
30+
# Use the same trick from https://github.com/marketplace/actions/setup-miniconda
31+
# to refresh the cache daily. This is kind of optional though
32+
- name: Get date
33+
id: get-date
34+
shell: bash
35+
run: |
36+
echo "today=$(/bin/date -u '+%Y%m%d')d" >> "${GITHUB_OUTPUT}"
37+
38+
- name: Setup miniconda cache
39+
id: miniconda-cache
40+
uses: actions/cache@v3
41+
with:
42+
path: ${{ runner.temp }}/miniconda
43+
key: miniconda-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }}
44+
45+
- name: Install miniconda (${{ inputs.miniconda-version }})
46+
if: steps.miniconda-cache.outputs.cache-hit != 'true'
47+
env:
48+
MINICONDA_VERSION: ${{ inputs.miniconda-version }}
49+
shell: bash -l {0}
50+
run: |
51+
MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda"
52+
mkdir -p "${MINICONDA_INSTALL_PATH}"
53+
case ${RUNNER_OS}-${RUNNER_ARCH} in
54+
Linux-X64)
55+
MINICONDA_ARCH="Linux-x86_64"
56+
;;
57+
macOS-ARM64)
58+
MINICONDA_ARCH="MacOSX-arm64"
59+
;;
60+
macOS-X64)
61+
MINICONDA_ARCH="MacOSX-x86_64"
62+
;;
63+
*)
64+
echo "::error::Platform ${RUNNER_OS}-${RUNNER_ARCH} currently unsupported using this action"
65+
exit 1
66+
;;
67+
esac
68+
MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py39_${MINICONDA_VERSION}-${MINICONDA_ARCH}.sh"
69+
curl -fsSL "${MINICONDA_URL}" -o "${MINICONDA_INSTALL_PATH}/miniconda.sh"
70+
bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}"
71+
rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh"
72+
73+
- name: Update GitHub path to include miniconda install
74+
shell: bash
75+
run: |
76+
set -x
77+
78+
MINICONDA_INSTALL_PATH="${RUNNER_TEMP}/miniconda"
79+
echo "${MINICONDA_INSTALL_PATH}/bin" >> $GITHUB_PATH
80+
# NB: GITHUB_PATH has a lower priority than PATH, so also set the path
81+
# here to make sure that the correct conda is used
82+
echo "PATH=${MINICONDA_INSTALL_PATH}/bin:${PATH}" >> $GITHUB_ENV
83+
84+
# When the environment-file or pip-requirements-file inputs are not set or are set to invalid paths, the hashFiles
85+
# function will return an empty string without failing the step. This works out nicely and we can have a various
86+
# combination of cache key such as:
87+
# - Both are missing or invalid: miniconda-env-macOS-ARM64-20221022d--
88+
# - Both are set: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)-HASH(pip-requirements-file)
89+
# - The first one is missing or invalid: miniconda-env-macOS-ARM64-20221022d--HASH(pip-requirements-file)
90+
# - The second one is missing or invalid: miniconda-env-macOS-ARM64-20221022d-HASH(environment-file)-
91+
#
92+
# There is no need to skip or run actions/cache with complicated logic
93+
- name: Setup miniconda env cache
94+
id: miniconda-env-cache
95+
uses: actions/cache@v3
96+
with:
97+
path: ${{ runner.temp }}/conda-python-${{ inputs.python-version }}
98+
key: miniconda-env-${{ runner.os }}-${{ runner.arch }}-${{ inputs.python-version }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(inputs.environment-file) }}-${{ hashFiles(inputs.pip-requirements-file) }}
99+
100+
- name: Setup conda environment with python (v${{ inputs.python-version }})
101+
if: steps.miniconda-env-cache.outcome == 'success' && steps.miniconda-env-cache.outputs.cache-hit != 'true'
102+
shell: bash
103+
env:
104+
PYTHON_VERSION: ${{ inputs.python-version }}
105+
ENV_FILE: ${{ inputs.environment-file }}
106+
PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }}
107+
run: |
108+
set -x
109+
110+
CONDA_BASE_ENV="${RUNNER_TEMP}/conda-python-${PYTHON_VERSION}"
111+
ENV_FILE_FLAG=""
112+
if [[ -f "${ENV_FILE}" ]]; then
113+
ENV_FILE_FLAG="--file ${ENV_FILE}"
114+
elif [[ -n "${ENV_FILE}" ]]; then
115+
echo "::warning::Specified env file (${ENV_FILE}) not found, not going to include it"
116+
fi
117+
118+
CONDA_EXTRA_FLAGS=""
119+
if [[ "${PYTHON_VERSION}" == "3.11" ]]; then
120+
CONDA_EXTRA_FLAGS=" -c pytorch-nightly"
121+
fi
122+
123+
# Print the conda we are using here in case we need debugging information
124+
CONDA_RUNTIME=$(which conda)
125+
"${CONDA_RUNTIME}" --version
126+
127+
"${CONDA_RUNTIME}" create \
128+
--yes --quiet \
129+
--prefix "${CONDA_BASE_ENV}" \
130+
${ENV_FILE_FLAG} \
131+
python="${PYTHON_VERSION}" \
132+
cmake=3.22 \
133+
ninja=1.10 \
134+
pkg-config=0.29 \
135+
wheel=0.37 \
136+
${CONDA_EXTRA_FLAGS}
137+
138+
if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then
139+
"${CONDA_RUNTIME}" run -p "${CONDA_BASE_ENV}" --no-capture-output python3 -mpip install -r "${PIP_REQUIREMENTS_FILE}"
140+
elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then
141+
echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it"
142+
fi
143+
144+
- name: Clone the base conda environment and update GitHub env
145+
shell: bash
146+
env:
147+
PYTHON_VERSION: ${{ inputs.python-version }}
148+
CONDA_BASE_ENV: ${{ runner.temp }}/conda-python-${{ inputs.python-version }}
149+
PIP_REQUIREMENTS_FILE: ${{ inputs.pip-requirements-file }}
150+
run: |
151+
set -x
152+
153+
# Print the conda we are using here in case we need debugging information
154+
CONDA_RUNTIME=$(which conda)
155+
"${CONDA_RUNTIME}" --version
156+
157+
CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
158+
"${CONDA_RUNTIME}" create \
159+
--yes --quiet \
160+
--prefix "${CONDA_ENV}" \
161+
--clone "${CONDA_BASE_ENV}"
162+
163+
set +e
164+
# NB: Cloning sometimes doesn't copied pip dependencies (untracked files) over. If this
165+
# happens, let's attempt to install the pip requirements directly on top of the cloned
166+
# environment. This is to make sure that no dependency is missing.
167+
UNTRACKED_FILES_COUNT=$("${CONDA_RUNTIME}" package -p "${CONDA_ENV}" -u | grep -v "^#" | wc -l | xargs)
168+
set -e
169+
170+
if [[ -z "${UNTRACKED_FILES_COUNT}" ]] || [[ "${UNTRACKED_FILES_COUNT}" == "0" ]]; then
171+
if [[ -f "${PIP_REQUIREMENTS_FILE}" ]]; then
172+
# NB: Force reinstall and don't use the cache, as the installation would still fail
173+
# when reporting that all requirements have been satisfied
174+
"${CONDA_RUNTIME}" run -p "${CONDA_ENV}" --no-capture-output python3 -mpip install --ignore-installed --no-cache-dir -r "${PIP_REQUIREMENTS_FILE}"
175+
elif [[ -n "${PIP_REQUIREMENTS_FILE}" ]]; then
176+
echo "::warning::Specified pip requirements file (${PIP_REQUIREMENTS_FILE}) not found, not going to include it"
177+
fi
178+
fi
179+
180+
echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}"
181+
echo "CONDA_RUN=${CONDA_RUNTIME} run -p ${CONDA_ENV} --no-capture-output" >> "${GITHUB_ENV}"
182+
if [[ "${PYTHON_VERSION}" == "3.11" ]]; then
183+
# TODO: Remove me, when more packages will be available on default channel
184+
echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV} -c pytorch-nightly" >> "${GITHUB_ENV}"
185+
else
186+
echo "CONDA_INSTALL=${CONDA_RUNTIME} install --yes --quiet -p ${CONDA_ENV}" >> "${GITHUB_ENV}"
187+
fi
188+
189+
- name: Reset channel priority
190+
shell: bash
191+
run: |
192+
CONDA_RUNTIME=$(which conda)
193+
194+
set -euxo pipefail
195+
"${CONDA_RUNTIME}" config --set channel_priority false

‎.github/workflows/macos-job.yml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
name: Run a macOS job
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
script:
7+
description: 'Script to utilize'
8+
default: "python setup.py bdist_wheel"
9+
type: string
10+
timeout:
11+
description: 'Timeout for the job (in minutes)'
12+
default: 30
13+
type: number
14+
runner:
15+
description: 'Runner type to utilize'
16+
default: "macos-12"
17+
type: string
18+
upload-artifact:
19+
description: 'Name to give artifacts uploaded from ${RUNNER_ARTIFACT_DIR}'
20+
default: ""
21+
type: string
22+
download-artifact:
23+
description: 'Name to download artifacts to ${RUNNER_ARTIFACT_DIR}'
24+
default: ""
25+
type: string
26+
repository:
27+
description: 'Repository to checkout, defaults to ""'
28+
default: ""
29+
type: string
30+
fetch-depth:
31+
description: 'Number of commits to fetch, defaults to 1 similar to actions/checkout'
32+
default: 1
33+
type: number
34+
submodules:
35+
description:
36+
Same as actions/checkout, set to `true` to checkout submodules or `recursive` to
37+
recursively checkout everything
38+
default: ""
39+
type: string
40+
ref:
41+
description: 'Reference to checkout, defaults to "nightly"'
42+
default: ""
43+
type: string
44+
test-infra-repository:
45+
description: "Test infra repository to use"
46+
default: "pytorch/test-infra"
47+
type: string
48+
test-infra-ref:
49+
description: "Test infra reference to use"
50+
default: ""
51+
type: string
52+
job-name:
53+
description: "Name for the job, which is displayed in the GitHub UI"
54+
default: "macos-job"
55+
type: string
56+
continue-on-error:
57+
description: "Prevents a job from failing when a step fails. Set to true to allow a job to pass when exec script step fails."
58+
default: false
59+
type: boolean
60+
binary-matrix:
61+
description: "If we are calling this workflow with binary build matrix entry, will initialize matrix entries and env vars"
62+
required: false
63+
default: ''
64+
type: string
65+
66+
jobs:
67+
job:
68+
name: ${{ inputs.job-name }}
69+
env:
70+
REPOSITORY: ${{ inputs.repository || github.repository }}
71+
SCRIPT: ${{ inputs.script }}
72+
runs-on: ${{ inputs.runner }}
73+
timeout-minutes: ${{ inputs.timeout }}
74+
steps:
75+
- name: Clean workspace
76+
run: |
77+
echo "::group::Cleanup debug output"
78+
rm -rfv "${GITHUB_WORKSPACE}"
79+
mkdir -p "${GITHUB_WORKSPACE}"
80+
echo "::endgroup::"
81+
82+
- name: Checkout repository (${{ inputs.test-infra-repository }}@${{ inputs.test-infra-ref }})
83+
uses: actions/checkout@v3
84+
with:
85+
# Support the use case where we need to checkout someone's fork
86+
repository: ${{ inputs.test-infra-repository }}
87+
ref: ${{ inputs.test-infra-ref }}
88+
path: test-infra
89+
90+
- name: Setup miniconda
91+
uses: ./.github/actions/setup-miniconda
92+
93+
- name: Checkout repository (${{ inputs.repository || github.repository }}@${{ inputs.ref }})
94+
uses: actions/checkout@v3
95+
with:
96+
# Support the use case where we need to checkout someone's fork
97+
repository: ${{ inputs.repository || github.repository }}
98+
ref: ${{ inputs.ref || github.ref }}
99+
path: ${{ inputs.repository || github.repository }}
100+
fetch-depth: ${{ inputs.fetch-depth }}
101+
submodules: ${{ inputs.submodules }}
102+
103+
- name: Setup useful environment variables
104+
working-directory: ${{ inputs.repository }}
105+
run: |
106+
RUNNER_ARTIFACT_DIR="${RUNNER_TEMP}/artifacts"
107+
mkdir -p "${RUNNER_ARTIFACT_DIR}"
108+
echo "RUNNER_ARTIFACT_DIR=${RUNNER_ARTIFACT_DIR}" >> "${GITHUB_ENV}"
109+
110+
RUNNER_TEST_RESULTS_DIR="${RUNNER_TEMP}/test-results"
111+
mkdir -p "${RUNNER_TEST_RESULTS_DIR}"
112+
echo "RUNNER_TEST_RESULTS_DIR=${RUNNER_TEST_RESULTS_DIR}" >> "${GITHUB_ENV}"
113+
114+
- name: Download artifacts (if any)
115+
uses: actions/download-artifact@v3
116+
if: ${{ inputs.download-artifact != '' }}
117+
with:
118+
name: ${{ inputs.download-artifact }}
119+
path: ${{ runner.temp }}/artifacts/
120+
121+
- name: Run script
122+
shell: bash -l {0}
123+
continue-on-error: ${{ inputs.continue-on-error }}
124+
working-directory: ${{ inputs.repository }}
125+
run: |
126+
{
127+
echo "#!/usr/bin/env bash";
128+
echo "set -eou pipefail";
129+
# Source conda so it's available to the script environment
130+
echo 'eval "$(conda shell.bash hook)"';
131+
echo "${SCRIPT}";
132+
} > "${RUNNER_TEMP}/exec_script"
133+
while read line; do
134+
eval "export ${line}"
135+
done < "${RUNNER_TEMP}/github_env_${GITHUB_RUN_ID}"
136+
bash "${RUNNER_TEMP}/exec_script"
137+
138+
- name: Surface failing tests
139+
if: always()
140+
uses: pmeier/[email protected]
141+
with:
142+
path: ${{ env.RUNNER_TEST_RESULTS_DIR }}
143+
fail-on-empty: false
144+
145+
- name: Check if there are potential artifacts and move them to the correct artifact location
146+
shell: bash -l {0}
147+
working-directory: ${{ inputs.repository }}
148+
id: check-artifacts
149+
if: ${{ inputs.upload-artifact != '' }}
150+
env:
151+
UPLOAD_ARTIFACT_NAME: ${{ inputs.upload-artifact }}
152+
run: |
153+
# If the default execution path is followed then we should get a wheel in the dist/ folder
154+
# attempt to just grab whatever is in there and scoop it all up
155+
if find "dist/" -name "*.whl" >/dev/null 2>/dev/null; then
156+
mv -v dist/*.whl "${RUNNER_ARTIFACT_DIR}/"
157+
fi
158+
# Set to fail upload step if there are no files for upload and expected files for upload
159+
echo 'if-no-files-found=error' >> "${GITHUB_OUTPUT}"
160+
161+
- name: Upload artifacts to GitHub (if any)
162+
uses: actions/upload-artifact@v3
163+
if: ${{ inputs.upload-artifact != '' }}
164+
with:
165+
name: ${{ inputs.upload-artifact }}
166+
path: ${{ runner.temp }}/artifacts/
167+
if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }}

‎.github/workflows/pull.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,3 +441,30 @@ jobs:
441441
442442
# run e2e (export, tokenizer and runner)
443443
PYTHON_EXECUTABLE=python bash .ci/scripts/test_phi_3_mini.sh
444+
445+
debug:
446+
uses: ./.github/workflows/macos-job.yml
447+
with:
448+
runner: macos-m1-stable
449+
python-version: '3.11'
450+
submodules: 'true'
451+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
452+
timeout: 90
453+
script: |
454+
BUILD_TOOL=cmake
455+
BACKEND=coreml
456+
457+
bash .ci/scripts/setup-conda.sh
458+
459+
# Setup MacOS dependencies as there is no Docker support on MacOS atm
460+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
461+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh
462+
echo "Finishing installing coreml."
463+
464+
# Build and test coreml model
465+
MODELS=(mv3 ic4 resnet50 edsr mobilebert w2l)
466+
for MODEL_NAME in "${MODELS[@]}"; do
467+
echo "::group::Exporting coreml model: $MODEL_NAME"
468+
PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"
469+
echo "::endgroup::"
470+
done

0 commit comments

Comments
 (0)