diff --git a/.buildbot/jenkins-build-llvm-project.py b/.buildbot/jenkins-build-llvm-project.py index 06c5bb115e..079d8686a1 100755 --- a/.buildbot/jenkins-build-llvm-project.py +++ b/.buildbot/jenkins-build-llvm-project.py @@ -7,8 +7,8 @@ LLVM_PROJECT_DOCKERFILE = "docker/Dockerfile.llvm-project" LLVM_PROJECT_GITHUB_URL = "https://api.github.com/repos/llvm/llvm-project" LLVM_PROJECT_BASE_IMAGE = { - "static": "ghcr.io/onnxmlir/ubuntu:jammy", - "shared": "registry.access.redhat.com/ubi8-minimal:latest", + "static": "ghcr.io/onnxmlir/ubuntu:noble", + "shared": "registry.access.redhat.com/ubi9-minimal:latest", } LLVM_PROJECT_IMAGE = { "static": docker_static_image_name, diff --git a/docker/Dockerfile.llvm-project b/docker/Dockerfile.llvm-project index 40f14a79cd..9c3fe0231f 100644 --- a/docker/Dockerfile.llvm-project +++ b/docker/Dockerfile.llvm-project @@ -1,5 +1,5 @@ -# By default, use ubuntu:jammy, remember to change Jenkins build script as well -ARG BASE_IMAGE="ghcr.io/onnxmlir/ubuntu:jammy" +# By default, use ubuntu:noble, remember to change Jenkins build script as well +ARG BASE_IMAGE="ghcr.io/onnxmlir/ubuntu:noble" FROM ${BASE_IMAGE} # Label the image for various checking and cleanup @@ -30,12 +30,11 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] apt-get install -qq -y --no-install-recommends \ autoconf automake ca-certificates clang cmake cppcheck \ curl default-jdk-headless gcc g++ git libncurses-dev \ - libtool make maven ninja-build openjdk-11-jdk-headless \ - python3 python3-dev python3-distutils python3-numpy \ - python3-pip python3-pytest-xdist python3-setuptools \ - python3-typing-extensions unzip zip zlib1g-dev && \ - rm -rf /var/lib/apt/lists/* && \ - ln -sf /usr/bin/pytest-3 /usr/bin/pytest; \ + libtool make maven ninja-build python3 python3-dev \ + python3-numpy python3-pip python3-pytest-xdist \ + python3-setuptools python3-typing-extensions \ + unzip zip zlib1g-dev && \ + rm -rf /var/lib/apt/lists/*; \ elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \ ln -sf /usr/share/zoneinfo/${TZ} /etc/localtime && \ ([ -x /usr/bin/microdnf ] && microdnf install -y yum) && \ @@ -45,27 +44,23 @@ RUN distro=$(cat /etc/os-release|grep -Po '(?<=^ID=").*(?=")|(?<=^ID=)[^"].*[^"] yum update -q -y && \ yum install -q -y \ autoconf automake ca-certificates clang cmake diffutils \ - file java-11-openjdk-devel java-11-openjdk-headless \ - gcc gcc-c++ git libtool make ncurses-devel ninja-build \ - python39 python39-devel python39-pip python39-setuptools \ - python39-wheel tzdata-java unzip which zip zlib-devel && \ - # Workaround broken ubi8 amd64 image installing python3.12 as - # dependency of clang, which also breaks the /usr/bin/pip3 - # symlink creation - ([ -f /usr/bin/python3.12 ] && yum remove -y python3.12 || true) && \ - # Use same versions as those in ubuntu:jammy + file java-21-devel gcc gcc-c++ git libtool make ncurses-devel \ + ninja-build python3 python3-devel python3-pip python3-setuptools \ + python3-wheel tzdata-java unzip which zip zlib-devel && \ + # Upgrade setuptools to avoid UNKNOWN wheel when building onnx + # Use same package versions as those in ubuntu:noble + pip3 install -q setuptools --upgrade && \ pip3 install -q \ - Cython pytest==6.2.5 numpy==1.21.5 pytest-forked==1.4.0 \ - pytest-xdist==2.5.0 typing-extensions==3.10.0.2 && \ + Cython pytest==7.4.4 numpy==1.26.4 pytest-xdist==3.4.0 \ + typing-extensions==4.10.0 && \ rm -rf /var/cache/dnf/* && \ echo -e "/usr/local/lib" > /etc/ld.so.conf.d/local.conf; \ fi \ && ln -sf /usr/bin/python3 /usr/bin/python \ - && pip3 install --upgrade pip \ && rm -rf ${HOME}/.cache # Install bazel -ARG BAZEL_VERSION=6.5.0 +ARG BAZEL_VERSION=7.4.1 ARG BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download RUN curl -sL ${BAZEL_URL}/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip \ -o bazel-${BAZEL_VERSION}-dist.zip \ @@ -76,7 +71,8 @@ RUN curl -sL ${BAZEL_URL}/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip \ --color=no \ --curses=yes \ --copt=-Wno-error=implicit-fallthrough \ - --host_javabase=@local_jdk//:jdk \ + --tool_java_runtime_version=local_jdk \ + --java_runtime_version=local_jdk \ --verbose_failures" \ BAZEL_JAVAC_OPTS="-J-Xmx4g -J-Xms1g" ./compile.sh \ && cp output/bazel /usr/local/bin \ @@ -92,34 +88,69 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ && cargo install cargo-bazel --version ${CARGO_BAZEL_VERSION} # Install protobuf -ARG PROTOBUF_VERSION=6.31.1 +ARG PROTOBUF_VERSION=33.5 ARG PROTOBUF_URL=https://github.com/protocolbuffers/protobuf.git ARG PROTOBUF_DIR=protobuf RUN git clone -b v${PROTOBUF_VERSION} --recursive ${PROTOBUF_URL} ${PROTOBUF_DIR} \ + # + # Fix protoc-gen-upb_minitable_stage0 segfault in + # _upb_DefPool_SetPlatform on s390x until upstreamed + # + && sed -i -Ez 's/(.*\n)([[:blank:]]*if \(_upb_Decoder_Reserve\(d, arr, 1\)\) \{)/\1 _upb_Decoder_MungeInt32\(\&elem\);\n\2/' ${PROTOBUF_DIR}/upb/wire/decode.c \ + && sed -i 'N; /} else {\n _upb_Decoder_Munge(field, val);/ { s/^/ _upb_Decoder_MungeInt32(val);\n/; b }; P; D' ${PROTOBUF_DIR}/upb/wire/decode.c \ + # && mkdir -p ${PROTOBUF_DIR}/build \ && cd ${PROTOBUF_DIR}/build \ + # # Must specify -Dprotobuf_BUILD_TESTS=OFF otherwise find_package(absl) # in onnx will fail due to missing protobuf::gmock target - # Must specify -DCMAKE_CXX_STANDARD=17 since protobuf requires c++17 but - # clang 14 on Ubuntu Jammy defaults to c++14 + # If clang default C++ standard is lower than 17, you must specify + # -DCMAKE_CXX_STANDARD=17 (or higher) which is required by protobuf + # && CC=clang CXX=clang++ \ cmake -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_CXX_STANDARD=17 \ -DBUILD_SHARED_LIBS=ON \ -Dprotobuf_BUILD_TESTS=OFF .. \ && make -j${NPROC} install && ldconfig \ && cd .. \ - # New version of python protobuf can no longer be built with setup.py. - # Must use bazel to build. protobuf v6.31.1 is the first version using - # rules_rust 0.56.0 which has s390x support. rules_buf still needs a - # small patch. + # + # New version of python protobuf can no longer be built with setup.py + # and must be built with bazel. + # + # v31.1 is the first version using rules_rust 0.56.0 which has s390x + # support. + # v33.5 includes the bzlmod fix for rules_python version bug + # (commit fa9ceb3d4f3fbb1542059d931bf4151a0bee316c). + # + # MODULE.bazel needs to be patched to bump rules_python to 1.8.0 + # since s390x no longer has python 3.8 interpreter. And rules_buf + # needs to be patched to support s390x arch. + # + # clang-18 crashes on s390x when compiling absl/time/duration.cc under + # bazel but works fine under cmake. This is because cmake and bazel + # are using different optimization levels. Override compiler options + # for duration.cc with --host_per_file_copt since abseil is being + # built as a tool dependency. If abseil is being built as a target + # dependency, we would use --per_file_copt instead. + # + && sed -i -e 's/"rules_python", version = "1.6.0"/"rules_python", version = "1.8.0"/g' \ + MODULE.bazel \ + && printf "\n\ +archive_override(\n\ + module_name = \"rules_buf\",\n\ + integrity = \"sha256-Hr64Q/CaYr0E3ptAjEOgdZd1yc+cBjp7OG1wzuf3DIs=\",\n\ + strip_prefix = \"rules_buf-0.3.0\",\n\ + urls = [ \"https://github.com/bufbuild/rules_buf/archive/refs/tags/v0.3.0.zip\", ],\n\ + patch_cmds = [\n\ + '''sed -i -e 's|\\\\\\\\[\"arm64\", \"amd64\"\\\\\\\\]|\\\\\\\\[\"arm64\", \"amd64\", \"s390x\"\\\\\\\\]|g' buf/internal/toolchain.bzl''',\n\ + ],\n\ +)" >> MODULE.bazel \ && export CARGO_BAZEL_GENERATOR_URL=file:///root/.cargo/bin/cargo-bazel \ && export CARGO_BAZEL_REPIN=true \ - && CC=clang CXX=clang++ bazel fetch //python/dist:binary_wheel \ - && sed -i -e 's/\["arm64", "amd64"\]/\["arm64", "amd64", "s390x"\]/g' \ - ${HOME}/.cache/bazel/_bazel_root/*/external/rules_buf/buf/internal/toolchain.bzl \ - && CC=clang CXX=clang++ bazel build //python/dist:binary_wheel \ - && pip3 install bazel-bin/python/dist/protobuf-${PROTOBUF_VERSION}-*.whl \ + && CC=clang CXX=clang++ \ + bazel build --host_per_file_copt=".*/absl/time/duration.cc@-O0" \ + //python/dist:binary_wheel \ + && pip3 install bazel-bin/python/dist/protobuf-*.whl --prefix=/usr \ && cd .. && rm -rf ${PROTOBUF_DIR} ${HOME}/.cache # Install jsoniter diff --git a/docker/Dockerfile.onnx-mlir b/docker/Dockerfile.onnx-mlir index 3753b11f1d..cbb398fd90 100644 --- a/docker/Dockerfile.onnx-mlir +++ b/docker/Dockerfile.onnx-mlir @@ -21,17 +21,26 @@ COPY . onnx-mlir # CMAKE_INSTALL_LIBDIR to be lib. RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \ && cd ${ONNX_ROOT} \ + # # Require patching until upstreamed + # && sed -i -e 's/target_link_libraries(onnx PUBLIC onnx_proto)/target_link_libraries(onnx PUBLIC onnx_proto PUBLIC ${protobuf_ABSL_USED_TARGETS})/g' \ -e '/absl::log_initialize/a \ absl::log_internal_check_op\ absl::log_internal_message\ absl::log_internal_nullguard' CMakeLists.txt \ - # Required for pip install with `--no-build-isolation` flag. - # setuptools >= 70.x creates conflicts with pip packaging versions. - && python3 -m pip install --upgrade setuptools==68.2.2 \ + # + # To avoid the "externally managed environment" error on ubuntu, + # use --prefix=/usr to install into /usr/local/lib (default is + # /usr/lib) + # ubi9 doesn't have this problem and --prefix=/usr will install + # into /usr/lib64 (default is /usr/lib) + # + # Must specify --no-build-isolation to avoid protoc-gen-mypy.py + # segfault on s390x + # && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \ - python3 -m pip install . --no-build-isolation \ + python3 -m pip install . --prefix=/usr --no-build-isolation \ && rm -rf ${HOME}/.cache ARG NPROC=4 @@ -94,11 +103,10 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \ make ninja-build python3 && \ rm -rf /var/lib/apt/lists/*; \ elif [ "${distro}" = "rhel" ] || [ "${distro}" = "fedora" ]; then \ - pip3 uninstall -q -y Cython pybind11 pytest pytest-forked \ - pytest-xdist typing-extensions && \ + pip3 uninstall -q -y \ + Cython pytest pytest-xdist typing-extensions && \ yum remove -q -y \ - adwaita-icon-theme autoconf automake cmake file \ - git libtool ninja-build python39 && \ + autoconf automake cmake file git libtool ninja-build && \ rm -rf /var/cache/dnf/*; \ fi \ && rm -rf /tmp/* /usr/bin/python ${HOME}/.cache \ diff --git a/docker/Dockerfile.onnx-mlir-dev b/docker/Dockerfile.onnx-mlir-dev index 8d312ffb81..5aa2794719 100644 --- a/docker/Dockerfile.onnx-mlir-dev +++ b/docker/Dockerfile.onnx-mlir-dev @@ -16,17 +16,26 @@ COPY . onnx-mlir # Setup onnx RUN ONNX_ROOT=${WORK_DIR}/onnx-mlir/third_party/onnx \ && cd ${ONNX_ROOT} \ + # # Require patching until upstreamed + # && sed -i -e 's/target_link_libraries(onnx PUBLIC onnx_proto)/target_link_libraries(onnx PUBLIC onnx_proto PUBLIC ${protobuf_ABSL_USED_TARGETS})/g' \ -e '/absl::log_initialize/a \ absl::log_internal_check_op\ absl::log_internal_message\ absl::log_internal_nullguard' CMakeLists.txt \ - # Required for pip install with `--no-build-isolation` flag. - # setuptools >= 70.x creates conflicts with pip packaging versions. - && python3 -m pip install --upgrade setuptools==68.2.2 \ + # + # To avoid the "externally managed environment" error on ubuntu, + # use --prefix=/usr to install into /usr/local/lib (default is + # /usr/lib) + # ubi9 doesn't have this problem and --prefix=/usr will install + # into /usr/lib64 (default is /usr/lib) + # + # Must specify --no-build-isolation to avoid protoc-gen-mypy.py + # segfault on s390x + # && CC=clang CXX=clang++ CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib" \ - python3 -m pip install . --no-build-isolation + python3 -m pip install . --prefix=/usr --no-build-isolation ARG NPROC=4 ARG ACCEL=NNPA @@ -39,9 +48,8 @@ RUN if [ "$(uname -m)" = "s390x" ]; then \ cd ${WORK_DIR}/onnx-mlir && \ rm -rf build-light && mkdir -p build-light && cd build-light && \ cmake -DCMAKE_BUILD_TYPE=Debug \ - -DONNX_MLIR_TARGET_TO_BUILD=OMPyInfer \ - .. &&\ - cmake --build . && \ + -DONNX_MLIR_TARGET_TO_BUILD=OMPyInfer .. && \ + cmake --build . --parallel ${NPROC} && \ cd .. && rm -rf build-light; \ fi @@ -60,9 +68,9 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \ # Run the cppcheck scanner on s390x # onnx-{data|ml|operator-ml}.pb.cc are not generated by cmake so # run make onnx to generate them. It wasn't a problem with -# cppcheck v1.90 in Ubuntu focal since it apparently silently -# ignores nonexisting files. But cppcheck v2.7 in Ubuntu jammy -# will report error on nonexisting files. +# cppcheck v1.90 in Ubuntu Focal since it apparently silently +# ignores nonexisting files. But cppcheck v2.7 in Jammy and +# v2.13 in Noble will report error on nonexisting files. && if [ "$(uname -m)" = "s390x" ]; then \ make -j${NPROC} onnx && \ NPROC=${NPROC} ${ONNX_MLIR_ROOT}/utils/cppcheck_scan/cppcheck-scan.py; \ @@ -98,7 +106,7 @@ RUN LLVM_PROJECT_ROOT=${WORK_DIR}/llvm-project \ git remote rename origin upstream; \ fi \ # Install joblib for model zoo tests - && pip3 install -q joblib \ + && pip3 install -q joblib --prefix=/usr \ && rm -rf /tmp/* # Copy from onnx-mlir-dev-test to squash removed source layer and