diff --git a/.tekton/odh-workbench-jupyter-minimal-cpu-py312-ubi9-pull-request.yaml b/.tekton/odh-workbench-jupyter-minimal-cpu-py312-ubi9-pull-request.yaml index 23e85a2bf2..06532aebe7 100644 --- a/.tekton/odh-workbench-jupyter-minimal-cpu-py312-ubi9-pull-request.yaml +++ b/.tekton/odh-workbench-jupyter-minimal-cpu-py312-ubi9-pull-request.yaml @@ -36,6 +36,7 @@ spec: - name: build-platforms value: - linux/x86_64 + - linux/ppc64le - name: dockerfile value: jupyter/minimal/ubi9-python-3.12/Dockerfile.cpu - name: path-context diff --git a/ci/cached-builds/gen_gha_matrix_jobs.py b/ci/cached-builds/gen_gha_matrix_jobs.py index 3e233f1d9f..a24604020c 100755 --- a/ci/cached-builds/gen_gha_matrix_jobs.py +++ b/ci/cached-builds/gen_gha_matrix_jobs.py @@ -30,6 +30,10 @@ "runtime-cuda-tensorflow-ubi9-python-3.12", } +PPC64LE_COMPATIBLE = { + "jupyter-minimal-ubi9-python-3.12", +} + S390X_COMPATIBLE = { "runtime-minimal-ubi9-python-3.11", "runtime-minimal-ubi9-python-3.12", @@ -74,6 +78,9 @@ class Arm64Images(enum.Enum): INCLUDE = "include" ONLY = "only" +class Ppc64leImages(enum.Enum): + EXCLUDE = "exclude" + INCLUDE = "include" class S390xImages(enum.Enum): EXCLUDE = "exclude" @@ -109,6 +116,15 @@ def main() -> None: nargs="?", help="Whether to include, exclude, or only include arm64 images", ) + argparser.add_argument( + "--ppc64le-images", + type=Arm64Images, + choices=list(Ppc64leImages), + required=False, + default=Arm64Images.INCLUDE, + nargs="?", + help="Whether to include or exclude ppc64le images", + ) argparser.add_argument( "--s390x-images", type=S390xImages, @@ -145,6 +161,9 @@ def main() -> None: if args.arm64_images != Arm64Images.EXCLUDE and args.s390x_images != S390xImages.ONLY: if target in ARM64_COMPATIBLE: targets_with_platform.append((target, "linux/arm64")) + if args.ppc64le_images != Ppc64leImages.EXCLUDE: + if target in PPC64LE_COMPATIBLE: + targets_with_platform.append((target, "linux/ppc64le")) if args.s390x_images != S390xImages.EXCLUDE and args.arm64_images != Arm64Images.ONLY: # NOTE: hardcode the list of s390x-compatible Makefile targets in S390X_COMPATIBLE if target in S390X_COMPATIBLE: diff --git a/jupyter/minimal/ubi9-python-3.12/Dockerfile.cpu b/jupyter/minimal/ubi9-python-3.12/Dockerfile.cpu index ef1ebf8057..f7e7b070ff 100644 --- a/jupyter/minimal/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/minimal/ubi9-python-3.12/Dockerfile.cpu @@ -3,6 +3,25 @@ ######################### ARG BASE_IMAGE +############################ +# Stage 1: PDF Tool Build # +############################ +# e.g., registry.access.redhat.com/ubi9/python-312:latest +FROM ${BASE_IMAGE} AS pdf-builder + +WORKDIR /opt/app-root/bin + +# OS Packages needs to be installed as root +USER 0 + +# Copy scripts +COPY jupyter/utils/install_texlive.sh ./install_texlive.sh +COPY jupyter/utils/install_pandoc.sh ./install_pandoc.sh +RUN chmod +x install_texlive.sh install_pandoc.sh + +RUN ./install_texlive.sh +RUN ./install_pandoc.sh + #################### # cpu-base # #################### @@ -60,8 +79,14 @@ COPY ${JUPYTER_REUSABLE_UTILS} utils/ USER 0 # Dependencies for PDF export begin -RUN ./utils/install_pdf_deps.sh + +# Copy built TeXLive and Pandoc binaries only (no build deps) +COPY --from=pdf-builder /usr/local/texlive /usr/local/texlive +COPY --from=pdf-builder /usr/local/pandoc /usr/local/pandoc + ENV PATH="/usr/local/texlive/bin/linux:/usr/local/pandoc/bin:$PATH" +ENV PATH="/usr/local/texlive/bin/x86_64-linux:/usr/local/pandoc/bin:$PATH" +ENV PATH="/opt/texlive/2025/bin/powerpc64le-unknown-linux-gnu:/usr/local/pandoc/bin:$PATH" # Dependencies for PDF export end USER 1001 diff --git a/jupyter/utils/install_pandoc.sh b/jupyter/utils/install_pandoc.sh new file mode 100644 index 0000000000..123f378eaa --- /dev/null +++ b/jupyter/utils/install_pandoc.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -euxo pipefail + +# Mapping of `uname -m` values to equivalent GOARCH values +declare -A UNAME_TO_GOARCH +UNAME_TO_GOARCH["x86_64"]="amd64" +UNAME_TO_GOARCH["aarch64"]="arm64" +UNAME_TO_GOARCH["ppc64le"]="ppc64le" +UNAME_TO_GOARCH["s390x"]="s390x" + +ARCH="${UNAME_TO_GOARCH[$(uname -m)]}" +if [[ -z "${ARCH:-}" ]]; then + echo "Unsupported architecture: $(uname -m)" >&2 + exit 1 +fi + +# Skip PDF export installation for s390x architecture +if [[ "$(uname -m)" == "s390x" ]]; then + echo "PDF export functionality is not supported on s390x architecture. Skipping installation." + exit 0 +fi + +if [[ "$ARCH" == "ppc64le" ]]; then + # Install Pandoc from source + dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + dnf install -y cabal-install ghc gmp-devel + + # Set version + PANDOC_VERSION=3.7.0.2 + + cd /tmp + git clone --recurse-submodules https://github.com/jgm/pandoc.git + cd pandoc + git checkout ${PANDOC_VERSION} + git submodule update --init --recursive + + cabal update + + # Build the CLI tool (not the top-level library package) + cd pandoc-cli + + # Clean previous builds + cabal clean + + cabal build -j"$(nproc)" + mkdir -p /usr/local/pandoc/bin + cabal install \ + --installdir=/usr/local/pandoc/bin \ + --overwrite-policy=always \ + --install-method=copy + + # Clean up Haskell build system + rm -rf ~/.cabal ~/.ghc /tmp/pandoc + dnf remove -y cabal-install ghc gmp-devel + dnf clean all && rm -rf /var/cache/dnf + + # Verify installation + /usr/local/pandoc/bin/pandoc --version + +elif [[ "$ARCH" == "amd64" ]]; then + # pandoc installation + curl -fL "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-linux-${ARCH}.tar.gz" -o /tmp/pandoc.tar.gz + mkdir -p /usr/local/pandoc + tar xvzf /tmp/pandoc.tar.gz --strip-components 1 -C /usr/local/pandoc/ + rm -f /tmp/pandoc.tar.gz + +else + echo "Unsupported architecture: $ARCH" >&2 + exit 1 +fi diff --git a/jupyter/utils/install_pdf_deps.sh b/jupyter/utils/install_pdf_deps.sh index f07a54f836..bd4e5d2720 100755 --- a/jupyter/utils/install_pdf_deps.sh +++ b/jupyter/utils/install_pdf_deps.sh @@ -23,18 +23,89 @@ if [[ "$(uname -m)" == "s390x" ]]; then exit 0 fi -# tex live installation -echo "Installing TexLive to allow PDf export from Notebooks" -curl -fL https://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz -o install-tl-unx.tar.gz -zcat < install-tl-unx.tar.gz | tar xf - -cd install-tl-2* -perl ./install-tl --no-interaction --scheme=scheme-small --texdir=/usr/local/texlive -mv /usr/local/texlive/bin/"$(uname -m)-linux" /usr/local/texlive/bin/linux -cd /usr/local/texlive/bin/linux -./tlmgr install tcolorbox pdfcol adjustbox titling enumitem soul ucs collection-fontsrecommended - -# pandoc installation -curl -fL "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-linux-${ARCH}.tar.gz" -o /tmp/pandoc.tar.gz -mkdir -p /usr/local/pandoc -tar xvzf /tmp/pandoc.tar.gz --strip-components 1 -C /usr/local/pandoc/ -rm -f /tmp/pandoc.tar.gz +if [[ "$ARCH" == "ppc64le" ]]; then + echo "Installing TeX Live from source for $ARCH" + + # Download and extract source + wget https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2025/texlive-20250308-source.tar.xz + tar -xf texlive-20250308-source.tar.xz + cd texlive-20250308-source + + # Install build dependencies + dnf install -y gcc-toolset-13 perl make libX11-devel libXt-devel \ + zlib-devel freetype-devel libpng-devel ncurses-devel \ + gd-devel libtool wget tar xz bison flex libXaw-devel + + source /opt/rh/gcc-toolset-13/enable + + # Create build directory + mkdir ../texlive-build + cd ../texlive-build + + # Configure, build, install + ../texlive-20250308-source/configure --prefix=/usr/local/texlive + make -j$(nproc) + make install + + # Symlink for pdflatex + cd /usr/local/texlive/bin/powerpc64le-unknown-linux-gnu + ln -s pdftex pdflatex + + # Cleanup TeX source to reduce image size + rm -rf /texlive-20250308-source /texlive-build + + export PATH="/usr/local/texlive/bin/powerpc64le-unknown-linux-gnu:$PATH" + pdflatex --version + + # Install Pandoc from source + dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + dnf install -y cabal-install ghc gmp-devel + + # Set version + PANDOC_VERSION=3.7.0.2 + + cd /tmp + git clone --recurse-submodules https://github.com/jgm/pandoc.git + cd pandoc + git checkout ${PANDOC_VERSION} + git submodule update --init --recursive + + cabal update + + # Build the CLI tool (not the top-level library package) + cd pandoc-cli + + # Clean previous builds + cabal clean + + cabal build -j$(nproc) + cabal install --installdir=/usr/local/bin --overwrite-policy=always --install-method=copy + + # Clean up Haskell build system + rm -rf ~/.cabal ~/.ghc /tmp/pandoc + dnf remove -y cabal-install ghc gmp-devel + dnf clean all && rm -rf /var/cache/dnf + + # Verify installation + /usr/local/bin/pandoc --version + +fi + +if [[ "$ARCH" == "amd64" ]]; then + # tex live installation + echo "Installing TexLive to allow PDf export from Notebooks" + curl -fL https://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz -o install-tl-unx.tar.gz + zcat < install-tl-unx.tar.gz | tar xf - + cd install-tl-2* + perl ./install-tl --no-interaction --scheme=scheme-small --texdir=/usr/local/texlive + mv /usr/local/texlive/bin/"$(uname -m)-linux" /usr/local/texlive/bin/linux + cd /usr/local/texlive/bin/linux + ./tlmgr install tcolorbox pdfcol adjustbox titling enumitem soul ucs collection-fontsrecommended + + # pandoc installation + curl -fL "https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-linux-${ARCH}.tar.gz " -o /tmp/pandoc.tar.gz + mkdir -p /usr/local/pandoc + tar xvzf /tmp/pandoc.tar.gz --strip-components 1 -C /usr/local/pandoc/ + rm -f /tmp/pandoc.tar.gz + +fi diff --git a/jupyter/utils/install_texlive.sh b/jupyter/utils/install_texlive.sh new file mode 100644 index 0000000000..59808ea948 --- /dev/null +++ b/jupyter/utils/install_texlive.sh @@ -0,0 +1,99 @@ +#!/bin/bash +set -euxo pipefail + +# Mapping of `uname -m` values to equivalent GOARCH values +declare -A UNAME_TO_GOARCH +UNAME_TO_GOARCH["x86_64"]="amd64" +UNAME_TO_GOARCH["aarch64"]="arm64" +UNAME_TO_GOARCH["ppc64le"]="ppc64le" +UNAME_TO_GOARCH["s390x"]="s390x" + +ARCH="${UNAME_TO_GOARCH[$(uname -m)]}" +if [[ -z "${ARCH:-}" ]]; then + echo "Unsupported architecture: $(uname -m)" >&2 + exit 1 +fi + +# Skip PDF export installation for s390x architecture +if [[ "$(uname -m)" == "s390x" ]]; then + echo "PDF export functionality is not supported on s390x architecture. Skipping installation." + exit 0 +fi + +if [[ "$ARCH" == "ppc64le" ]]; then + echo "Installing TeX Live from source for $ARCH..." + + # Install build dependencies + dnf install -y gcc-toolset-13 perl make libX11-devel libXt-devel \ + zlib-devel freetype-devel libpng-devel ncurses-devel \ + gd-devel libtool wget tar xz bison flex libXaw-devel + + # Step 1: Download and extract the TeX Live source + wget https://ftp.math.utah.edu/pub/tex/historic/systems/texlive/2025/texlive-20250308-source.tar.xz + tar -xf texlive-20250308-source.tar.xz + cd texlive-20250308-source + + # Enable newer GCC toolchain + source /opt/rh/gcc-toolset-13/enable + + # Create build directory and build + mkdir ../texlive-build + cd ../texlive-build + + # Configure, build, install + ../texlive-20250308-source/configure --prefix=/usr/local/texlive + make -j"$(nproc)" + make install + + # Symlink for pdflatex + ln -sf pdftex /usr/local/texlive/bin/powerpc64le-unknown-linux-gnu/pdflatex + + # Cleanup sources to reduce image size + rm -rf /texlive-20250308-source /texlive-build + + # Step 2: Run TeX Live installer for runtime tree setup + cd / + wget http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz + tar -xzf install-tl-unx.tar.gz + cd install-tl-2*/ + + # Create a custom install profile + TEXLIVE_INSTALL_PREFIX="/usr/local/texlive" + cat < texlive.profile +selected_scheme scheme-small +TEXDIR $TEXLIVE_INSTALL_PREFIX +TEXMFCONFIG ~/.texlive2025/texmf-config +TEXMFVAR ~/.texlive2025/texmf-var +option_doc 0 +option_src 0 +EOF + + ./install-tl --profile=texlive.profile --custom-bin=$TEXLIVE_INSTALL_PREFIX/bin/powerpc64le-unknown-linux-gnu + + # TeX Live binary directory + TEX_BIN_DIR="/usr/local/texlive/bin/powerpc64le-unknown-linux-gnu" + + # Create standard symlink 'linux' → arch-specific folder + ln -sf "$TEX_BIN_DIR" /usr/local/texlive/bin/linux + + # Set up environment + export PATH="$TEXLIVE_INSTALL_PREFIX/bin/linux:$PATH" + pdflatex --version + tlmgr --version + +elif [[ "$ARCH" == "amd64" ]]; then + # tex live installation + echo "Installing TexLive to allow PDf export from Notebooks" + curl -fL https://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz -o install-tl-unx.tar.gz + zcat < install-tl-unx.tar.gz | tar xf - + cd install-tl-2* + perl ./install-tl --no-interaction --scheme=scheme-small --texdir=/usr/local/texlive + mv /usr/local/texlive/bin/"$(uname -m)-linux" /usr/local/texlive/bin/linux + cd /usr/local/texlive/bin/linux + ./tlmgr install tcolorbox pdfcol adjustbox titling enumitem soul ucs collection-fontsrecommended + +else + echo "Unsupported architecture: $ARCH" >&2 + exit 1 + +fi diff --git a/scripts/generate_pull_request_pipelineruns.py b/scripts/generate_pull_request_pipelineruns.py index ce43b12054..78c8d5dad1 100644 --- a/scripts/generate_pull_request_pipelineruns.py +++ b/scripts/generate_pull_request_pipelineruns.py @@ -132,7 +132,20 @@ def transform_build_pipeline_to_pr_pipeline(push_pipeline_path: pathlib.Path): component = push_pipeline["metadata"]["labels"]["appstudio.openshift.io/component"] build_platforms = ["linux/x86_64"] - if component in ["odh-pipeline-runtime-minimal-cpu-py311-ubi9", "odh-pipeline-runtime-minimal-cpu-py312-ubi9"]: + if component in [ + "odh-base-image-cuda-py311-c9s", + "odh-base-image-cuda-py312-c9s", + "odh-base-image-cuda-py312-ubi9", + ]: + build_platforms.extend(["linux/arm64"]) + if component in [ + "odh-workbench-jupyter-minimal-cpu-py312-ubi9", + ]: + build_platforms.extend(["linux/ppc64le"]) + if component in [ + "odh-pipeline-runtime-minimal-cpu-py311-ubi9", + "odh-pipeline-runtime-minimal-cpu-py312-ubi9", + ]: build_platforms.extend(["linux/arm64", "linux/s390x"]) # Collect params