Skip to content

Commit 0c1d726

Browse files
huydhnfacebook-github-bot
huydhn
authored and
facebook-github-bot
committed
Run lintrunner on CI (#266)
Summary: There are a bunch of flake8 errors which I'll fix in a different PR and land them together to avoid complicating this one ### Testing https://github.com/pytorch/executorch/actions/runs/6152384722 Pull Request resolved: #266 Reviewed By: larryliu0820 Differential Revision: D49168984 Pulled By: huydhn fbshipit-source-id: 91788ca6cefe457ddfe5c81521ca094ac62ef9e6
1 parent 951425d commit 0c1d726

File tree

11 files changed

+129
-32
lines changed

11 files changed

+129
-32
lines changed

‎.ci/docker/build.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,18 @@ PYTHON_VERSION=3.10
1919
MINICONDA_VERSION=23.5.1-0
2020
BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
2121

22+
case "${IMAGE_NAME}" in
23+
executorch-ubuntu-22.04-clang12)
24+
LINTRUNNER=""
25+
;;
26+
executorch-ubuntu-22.04-linter)
27+
LINTRUNNER=yes
28+
;;
29+
*)
30+
echo "Invalid image name ${IMAGE_NAME}"
31+
exit 1
32+
esac
33+
2234
NIGHTLY=$(cat ci_commit_pins/nightly.txt)
2335
TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt)
2436
TORCHAUDIO_VERSION=$(cat ci_commit_pins/audio.txt)
@@ -35,6 +47,7 @@ docker build \
3547
--build-arg "TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION}.${NIGHTLY}" \
3648
--build-arg "TORCHVISION_VERSION=${TORCHVISION_VERSION}.${NIGHTLY}" \
3749
--build-arg "BUCK2_VERSION=${BUCK2_VERSION}" \
50+
--build-arg "LINTRUNNER=${LINTRUNNER}" \
3851
-f "${OS}"/Dockerfile \
3952
"$@" \
4053
.

‎.ci/docker/common/install_linter.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -eux
9+
10+
# shellcheck source=/dev/null
11+
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
12+
13+
# NB: Install all linter dependencies, the caching of lintrunner init could be
14+
# done after Executorch becomes public
15+
pip_install -r requirements-lintrunner.txt

‎.ci/docker/common/utils.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ as_ci_user() {
1010
# https://github.com/conda/conda/issues/6576
1111
# NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
1212
# NB: This must be run from a directory that the user has access to
13-
sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=${PATH}" "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" "$@"
13+
sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=${PATH}" "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}" "$@"
1414
}
1515

1616
conda_install() {

‎.ci/docker/requirements-lintrunner.txt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Lintrunner itself
2+
lintrunner==0.11.0
3+
lintrunner-adapters==0.9.0
4+
5+
# Flake 8 and its dependencies
6+
flake8==6.0.0
7+
flake8-breakpoint==1.1.0
8+
flake8-bugbear==23.6.5
9+
flake8-comprehensions==3.12.0
10+
flake8-pyi==23.5.0
11+
mccabe==0.7.0
12+
pycodestyle==2.10.0
13+
14+
# UFMT
15+
black==22.12.0
16+
ufmt==2.0.1
17+
usort==1.0.5
18+
19+
# Other linters
20+
clang-format==12.0.1
21+
cmakelint==1.4.1

‎.ci/docker/ubuntu/Dockerfile‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,13 @@ COPY ./common/install_conda.sh install_conda.sh
3838
COPY ./common/utils.sh utils.sh
3939
RUN bash ./install_conda.sh && rm install_conda.sh utils.sh /opt/conda/requirements-ci.txt /opt/conda/conda-env-ci.txt
4040

41+
ARG LINTRUNNER
42+
# Install lintrunner if needed
43+
COPY ./common/install_linter.sh install_linter.sh
44+
COPY ./common/utils.sh utils.sh
45+
COPY ./requirements-lintrunner.txt requirements-lintrunner.txt
46+
RUN if [ -n "${LINTRUNNER}" ]; then bash ./install_linter.sh; fi
47+
RUN rm install_linter.sh utils.sh requirements-lintrunner.txt
48+
4149
USER ci-user
4250
CMD ["bash"]

‎.github/workflows/_unittest.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ jobs:
2929
timeout: 60
3030
script: |
3131
set -eux
32-
export PATH="/opt/conda/envs/py_${{ inputs.python-version }}/bin:${PATH}"
32+
33+
# The generic Linux job chooses to use base env, not the one setup by the image
34+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
35+
conda activate "${CONDA_ENV}"
3336
3437
# Just need to install executorch, everything else has been setup
3538
pip install .

‎.github/workflows/docker-builds.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ jobs:
2525
matrix:
2626
include:
2727
- docker-image-name: executorch-ubuntu-22.04-clang12
28+
- docker-image-name: executorch-ubuntu-22.04-linter
2829
env:
2930
DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
3031
steps:

‎.github/workflows/lint.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Lint
2+
3+
on:
4+
pull_request:
5+
branches-ignore:
6+
- nightly
7+
push:
8+
branches:
9+
- main
10+
- release/*
11+
- landchecks/*
12+
workflow_dispatch:
13+
14+
concurrency:
15+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
16+
cancel-in-progress: true
17+
18+
jobs:
19+
lintrunner:
20+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
21+
with:
22+
runner: linux.2xlarge
23+
docker-image: executorch-ubuntu-22.04-linter
24+
fetch-depth: 0
25+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
26+
script: |
27+
# The generic Linux job chooses to use base env, not the one setup by the image
28+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
29+
conda activate "${CONDA_ENV}"
30+
31+
CACHE_DIRECTORY="/tmp/.lintbin"
32+
# Try to recover the cached binaries
33+
if [[ -d "${CACHE_DIRECTORY}" ]]; then
34+
# It's ok to fail this as lintrunner init would download these binaries
35+
# again if they do not exist
36+
cp -r "${CACHE_DIRECTORY}" . || true
37+
fi
38+
39+
# This has already been cached in the docker image
40+
lintrunner init 2> /dev/null
41+
42+
RC=0
43+
# Run lintrunner on all files
44+
if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then
45+
echo ""
46+
echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m"
47+
echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m"
48+
RC=1
49+
fi
50+
51+
# Use jq to massage the JSON lint output into GitHub Actions workflow commands.
52+
jq --raw-output \
53+
'"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \
54+
lint.json || true
55+
56+
exit $RC

‎.github/workflows/pull.yml‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,9 @@ jobs:
4949
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
5050
timeout: 60
5151
script: |
52-
PYTHON_VERSION=3.10
53-
# TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path
54-
# here, as it's there in the container
55-
export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
52+
# The generic Linux job chooses to use base env, not the one setup by the image
53+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
54+
conda activate "${CONDA_ENV}"
5655
5756
MODEL_NAME=${{ matrix.model }}
5857
BUILD_TOOL=${{ matrix.build-tool }}
@@ -105,10 +104,9 @@ jobs:
105104
submodules: 'true'
106105
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
107106
script: |
108-
PYTHON_VERSION=3.10
109-
# TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path
110-
# here, as it's there in the container
111-
export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
107+
# The generic Linux job chooses to use base env, not the one setup by the image
108+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
109+
conda activate "${CONDA_ENV}"
112110
113111
BUILD_TOOL=${{ matrix.build-tool }}
114112
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
@@ -155,10 +153,9 @@ jobs:
155153
submodules: 'true'
156154
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
157155
script: |
158-
PYTHON_VERSION=3.10
159-
# TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path
160-
# here, as it's there in the container
161-
export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
156+
# The generic Linux job chooses to use base env, not the one setup by the image
157+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
158+
conda activate "${CONDA_ENV}"
162159
163160
BUILD_TOOL=${{ matrix.build-tool }}
164161
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"

‎examples/quantization/utils.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
import copy
87
import logging
98

109
from torch.ao.quantization.quantize_pt2e import convert_pt2e, prepare_pt2e

0 commit comments

Comments
 (0)