Run lintrunner on CI (#266)

Summary:
There are a bunch of flake8 errors which I'll fix in a different PR and land them together to avoid complicating this one

### Testing

https://github.com/pytorch/executorch/actions/runs/6152384722

Pull Request resolved: #266

Reviewed By: larryliu0820

Differential Revision: D49168984

Pulled By: huydhn

fbshipit-source-id: 91788ca6cefe457ddfe5c81521ca094ac62ef9e6

Author: huydhn

.ci/docker/build.sh

@@ -19,6 +19,18 @@ PYTHON_VERSION=3.10
 MINICONDA_VERSION=23.5.1-0
 BUCK2_VERSION=$(cat ci_commit_pins/buck2.txt)
 
+case "${IMAGE_NAME}" in
+  executorch-ubuntu-22.04-clang12)
+    LINTRUNNER=""
+    ;;
+  executorch-ubuntu-22.04-linter)
+    LINTRUNNER=yes
+    ;;
+  *)
+    echo "Invalid image name ${IMAGE_NAME}"
+    exit 1
+esac
+
 NIGHTLY=$(cat ci_commit_pins/nightly.txt)
 TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt)
 TORCHAUDIO_VERSION=$(cat ci_commit_pins/audio.txt)
@@ -35,6 +47,7 @@ docker build \
   --build-arg "TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION}.${NIGHTLY}" \
   --build-arg "TORCHVISION_VERSION=${TORCHVISION_VERSION}.${NIGHTLY}" \
   --build-arg "BUCK2_VERSION=${BUCK2_VERSION}" \
+  --build-arg "LINTRUNNER=${LINTRUNNER}" \
   -f "${OS}"/Dockerfile \
   "$@" \
   .

.ci/docker/common/install_linter.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -eux
+
+# shellcheck source=/dev/null
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+# NB: Install all linter dependencies, the caching of lintrunner init could be
+# done after Executorch becomes public
+pip_install -r requirements-lintrunner.txt

.ci/docker/common/utils.sh

@@ -10,7 +10,7 @@ as_ci_user() {
   #     https://github.com/conda/conda/issues/6576
   # NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
   # NB: This must be run from a directory that the user has access to
-  sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=${PATH}" "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" "$@"
+  sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=${PATH}" "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}" "$@"
 }
 
 conda_install() {

.ci/docker/requirements-lintrunner.txt

@@ -0,0 +1,21 @@
+# Lintrunner itself
+lintrunner==0.11.0
+lintrunner-adapters==0.9.0
+
+# Flake 8 and its dependencies
+flake8==6.0.0
+flake8-breakpoint==1.1.0
+flake8-bugbear==23.6.5
+flake8-comprehensions==3.12.0
+flake8-pyi==23.5.0
+mccabe==0.7.0
+pycodestyle==2.10.0
+
+# UFMT
+black==22.12.0
+ufmt==2.0.1
+usort==1.0.5
+
+# Other linters
+clang-format==12.0.1
+cmakelint==1.4.1

.ci/docker/ubuntu/Dockerfile

@@ -38,5 +38,13 @@ COPY ./common/install_conda.sh install_conda.sh
 COPY ./common/utils.sh utils.sh
 RUN bash ./install_conda.sh && rm install_conda.sh utils.sh /opt/conda/requirements-ci.txt /opt/conda/conda-env-ci.txt
 
+ARG LINTRUNNER
+# Install lintrunner if needed
+COPY ./common/install_linter.sh install_linter.sh
+COPY ./common/utils.sh utils.sh
+COPY ./requirements-lintrunner.txt requirements-lintrunner.txt
+RUN if [ -n "${LINTRUNNER}" ]; then bash ./install_linter.sh; fi
+RUN rm install_linter.sh utils.sh requirements-lintrunner.txt
+
 USER ci-user
 CMD ["bash"]

.github/workflows/_unittest.yml

@@ -29,7 +29,10 @@ jobs:
       timeout: 60
       script: |
         set -eux
-        export PATH="/opt/conda/envs/py_${{ inputs.python-version }}/bin:${PATH}"
+
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
 
         # Just need to install executorch, everything else has been setup
         pip install .

.github/workflows/docker-builds.yml

@@ -25,6 +25,7 @@ jobs:
       matrix:
         include:
           - docker-image-name: executorch-ubuntu-22.04-clang12
+          - docker-image-name: executorch-ubuntu-22.04-linter
     env:
       DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }}
     steps:

.github/workflows/lint.yml

@@ -0,0 +1,56 @@
+name: Lint
+
+on:
+  pull_request:
+    branches-ignore:
+      - nightly
+  push:
+    branches:
+      - main
+      - release/*
+      - landchecks/*
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
+  cancel-in-progress: true
+
+jobs:
+  lintrunner:
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-linter
+      fetch-depth: 0
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        CACHE_DIRECTORY="/tmp/.lintbin"
+        # Try to recover the cached binaries
+        if [[ -d "${CACHE_DIRECTORY}" ]]; then
+          # It's ok to fail this as lintrunner init would download these binaries
+          # again if they do not exist
+          cp -r "${CACHE_DIRECTORY}" . || true
+        fi
+
+        # This has already been cached in the docker image
+        lintrunner init 2> /dev/null
+
+        RC=0
+        # Run lintrunner on all files
+        if ! lintrunner --force-color --all-files --tee-json=lint.json 2> /dev/null; then
+          echo ""
+          echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`. (If you don't get the same results, run \'lintrunner init\' to update your local linter)\e[0m"
+          echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m"
+          RC=1
+        fi
+
+        # Use jq to massage the JSON lint output into GitHub Actions workflow commands.
+        jq --raw-output \
+          '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \
+          lint.json || true
+
+        exit $RC

.github/workflows/pull.yml

@@ -49,10 +49,9 @@ jobs:
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 60
       script: |
-        PYTHON_VERSION=3.10
-        # TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path
-        # here, as it's there in the container
-        export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
 
         MODEL_NAME=${{ matrix.model }}
         BUILD_TOOL=${{ matrix.build-tool }}
@@ -105,10 +104,9 @@ jobs:
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
-        PYTHON_VERSION=3.10
-        # TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path
-        # here, as it's there in the container
-        export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
 
         BUILD_TOOL=${{ matrix.build-tool }}
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
@@ -155,10 +153,9 @@ jobs:
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
-        PYTHON_VERSION=3.10
-        # TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path
-        # here, as it's there in the container
-        export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}"
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
 
         BUILD_TOOL=${{ matrix.build-tool }}
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"

examples/quantization/utils.py

@@ -4,7 +4,6 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-import copy
 import logging
 
 from torch.ao.quantization.quantize_pt2e import convert_pt2e, prepare_pt2e