Skip to content

ppc64le: implement datascience jupyter image build on ppc64le (#2406) #2460

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Sep 16, 2025
Merged

ppc64le: implement datascience jupyter image build on ppc64le (#2406) #2460

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f14f76c
ppc64le: implement datascience jupyter image build on ppc64le (#2406)
bhagyashrigai Sep 11, 2025
0554396
fixup, prefer TARGETARCH over uname -m for build-time conditionals (#…
jiridanek Sep 11, 2025
cdf0a34
refactor, use heredoc for RUN commands to improve readability and enf…
jiridanek Sep 11, 2025
acc75ab
enable ppc64le in github actions and in konflux, subject to further r…
jiridanek Sep 5, 2025
58dd6bb
add konflux pull-request pipeline s390x build support for odh-workben…
jiridanek Sep 15, 2025
677cdf8
ppc64le: run openblas installation as root
jiridanek Sep 15, 2025
eeffe86
ppc64le(datascience): build pyarrow from sources
jiridanek Sep 16, 2025
3d48c30
fix(datascience): pin pyarrow version to what our pylock.toml wants
jiridanek Sep 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .tekton/odh-workbench-jupyter-datascience-cpu-py312-ubi9-pull-request.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ spec:
- name: build-platforms
value:
- linux/x86_64
- linux/ppc64le
- linux/s390x
- name: dockerfile
value: jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu
- name: path-context
Expand Down
20 changes: 20 additions & 0 deletions ci/cached-builds/gen_gha_matrix_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
"runtime-cuda-tensorflow-ubi9-python-3.12",
}

PPC64LE_COMPATIBLE = {
"jupyter-minimal-ubi9-python-3.12",
"jupyter-datascience-ubi9-python-3.12",
}

S390X_COMPATIBLE = {
"runtime-minimal-ubi9-python-3.11",
"runtime-minimal-ubi9-python-3.12",
Expand Down Expand Up @@ -76,6 +81,9 @@ class Arm64Images(enum.Enum):
INCLUDE = "include"
ONLY = "only"

class Ppc64leImages(enum.Enum):
EXCLUDE = "exclude"
INCLUDE = "include"

class S390xImages(enum.Enum):
EXCLUDE = "exclude"
Expand Down Expand Up @@ -111,6 +119,15 @@ def main() -> None:
nargs="?",
help="Whether to include, exclude, or only include arm64 images",
)
argparser.add_argument(
"--ppc64le-images",
type=Ppc64leImages,
choices=list(Ppc64leImages),
required=False,
default=Ppc64leImages.INCLUDE,
nargs="?",
help="Whether to include or exclude ppc64le images",
)
argparser.add_argument(
"--s390x-images",
type=S390xImages,
Expand Down Expand Up @@ -147,6 +164,9 @@ def main() -> None:
if args.arm64_images != Arm64Images.EXCLUDE and args.s390x_images != S390xImages.ONLY:
if target in ARM64_COMPATIBLE:
targets_with_platform.append((target, "linux/arm64"))
if args.ppc64le_images != Ppc64leImages.EXCLUDE:
if target in PPC64LE_COMPATIBLE:
targets_with_platform.append((target, "linux/ppc64le"))
if args.s390x_images != S390xImages.EXCLUDE and args.arm64_images != Arm64Images.ONLY:
# NOTE: hardcode the list of s390x-compatible Makefile targets in S390X_COMPATIBLE
if target in S390X_COMPATIBLE:
Expand Down
126 changes: 113 additions & 13 deletions jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,23 +104,23 @@ RUN curl -L https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/oc

##############################
# wheel-builder stage #
# NOTE: Only used in s390x
# NOTE: Only used in ppc64le and s390x
##############################
FROM cpu-base AS s390x-builder
FROM cpu-base AS pyarrow-builder

ARG TARGETARCH
USER 0
WORKDIR /tmp/build-wheels

# Build pyarrow optimized for s390x
# Build pyarrow on ppc64le and s390x
RUN --mount=type=cache,target=/root/.cache/pip \
--mount=type=cache,target=/root/.cache/dnf \
if [ "$TARGETARCH" = "s390x" ]; then \
if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then \
# Install build dependencies (shared for pyarrow and onnx)
dnf install -y cmake make gcc-c++ pybind11-devel wget && \
dnf clean all && \
# Build and collect pyarrow wheel
git clone --depth 1 https://github.com/apache/arrow.git && \
git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git && \
cd arrow/cpp && \
mkdir release && cd release && \
cmake -DCMAKE_BUILD_TYPE=Release \
Expand Down Expand Up @@ -160,6 +160,65 @@ RUN --mount=type=cache,target=/root/.cache/pip \
mkdir -p /tmp/wheels; \
fi

#######################################################
# common-builder (for Power-only)
#######################################################
FROM cpu-base AS common-builder
ARG TARGETARCH
USER root
RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
dnf install -y gcc-toolset-13 cmake ninja-build git wget unzip
dnf clean all
else
echo "Skipping common-builder package install on non-Power"
fi
EOF

#######################################################
# onnx-builder (Power-only)
#######################################################
FROM common-builder AS onnx-builder
ARG TARGETARCH
ARG ONNX_VERSION=v1.19.0
WORKDIR /root
RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
source /opt/rh/gcc-toolset-13/enable
git clone --recursive https://github.com/onnx/onnx.git
cd onnx
git checkout ${ONNX_VERSION}
git submodule update --init --recursive
pip install -r requirements.txt
export CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)"
pip wheel . -w /root/onnx_wheel
else
echo "Skipping ONNX build on non-Power"
mkdir -p /root/onnx_wheel
fi
EOF

#######################################################
# openblas-builder (Power-only)
#######################################################
FROM common-builder AS openblas-builder
ARG TARGETARCH
ARG OPENBLAS_VERSION=0.3.30
WORKDIR /root
RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip
unzip OpenBLAS-${OPENBLAS_VERSION}.zip
cd OpenBLAS-${OPENBLAS_VERSION}
make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
else
mkdir -p OpenBLAS-${OPENBLAS_VERSION}
echo "Skipping OpenBLAS build on non-Power"
fi
EOF
Comment on lines +206 to +221
Copy link
Contributor

@coderabbitai coderabbitai bot Sep 16, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Install OpenBLAS in the builder with DESTDIR to avoid root/permission issues later.

Current flow installs into /usr/local in the final stage (previous logs showed Permission denied). Stage the install in the builder and just COPY the files.

Apply:

 FROM common-builder AS openblas-builder
@@
 RUN <<'EOF'
 set -Eeuxo pipefail
 if [ "${TARGETARCH}" = "ppc64le" ]; then
     wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip
     unzip OpenBLAS-${OPENBLAS_VERSION}.zip
     cd OpenBLAS-${OPENBLAS_VERSION}
-    make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
+    # Use gcc-toolset and stage install to /openblas-root
+    source /opt/rh/gcc-toolset-13/enable
+    make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
+    make -j$(nproc) PREFIX=/usr/local DESTDIR=/openblas-root install
 else
-    mkdir -p OpenBLAS-${OPENBLAS_VERSION}
+    mkdir -p /openblas-root
     echo "Skipping OpenBLAS build on non-Power"
 fi
 EOF
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
FROM common-builder AS openblas-builder
ARG TARGETARCH
ARG OPENBLAS_VERSION=0.3.30
WORKDIR /root
RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip
unzip OpenBLAS-${OPENBLAS_VERSION}.zip
cd OpenBLAS-${OPENBLAS_VERSION}
make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
else
mkdir -p OpenBLAS-${OPENBLAS_VERSION}
echo "Skipping OpenBLAS build on non-Power"
fi
EOF
FROM common-builder AS openblas-builder
ARG TARGETARCH
ARG OPENBLAS_VERSION=0.3.30
WORKDIR /root
RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip
unzip OpenBLAS-${OPENBLAS_VERSION}.zip
cd OpenBLAS-${OPENBLAS_VERSION}
# Use gcc-toolset and stage install to /openblas-root
source /opt/rh/gcc-toolset-13/enable
make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
make -j$(nproc) PREFIX=/usr/local DESTDIR=/openblas-root install
else
mkdir -p /openblas-root
echo "Skipping OpenBLAS build on non-Power"
fi
EOF
🤖 Prompt for AI Agents 
In jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu around lines 206-221, the
OpenBLAS build currently installs into system paths causing permission errors
later; change the builder stage to run make install with a DESTDIR (e.g.,
/root/openblas-install) so artifacts are staged inside the builder instead of
/usr/local, and then in the final stage COPY the staged files (preserve the
relative usr/local layout) from the openblas-builder's DESTDIR into the final
image; ensure ownership/permissions are set appropriately during COPY so no root
permission errors occur at runtime.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

create github issue "Stage the OpenBLAS install in the builder and just COPY the files." for this

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🧩 Analysis chain

@jiridanek I've created GitHub issue for staging the OpenBLAS installation in the builder to resolve the permission errors observed in the ppc64le builds. The issue includes comprehensive problem analysis, technical solution using the DESTDIR pattern, and detailed acceptance criteria covering build reliability and multi-architecture compatibility.

The proposed approach stages the complete OpenBLAS installation within the builder stage and copies the pre-installed artifacts to the final stage, eliminating the permission conflicts that caused build failures in your PR logs.

🏁 Script executed:

#!/bin/bash

# Create GitHub issue for OpenBLAS installation staging
gh issue create \
  --title "Stage the OpenBLAS install in the builder and just COPY the files" \
  --body "## Problem Description

During ppc64le builds of the jupyter-datascience image, OpenBLAS installation fails with permission errors when attempting to install directly into \`/usr/local\` during the final stage. Build logs from PR #2460 show:

- \`cc1: fatal error: can't open 'ctest1.s' for writing: Permission denied\`
- \`/usr/bin/ld: cannot open output file getarch: Permission denied\` 
- \`mkdir: cannot create directory '/usr/local/lib/cmake': Permission denied\`
- \`Writing /usr/local/include/openblas_config.h failed\`

The current implementation in \`jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu\` (lines 206-221) builds OpenBLAS in the \`openblas-builder\` stage but attempts installation in the final \`jupyter-datascience\` stage, causing permission conflicts.

## Root Cause Analysis

1. **Multi-stage permission mismatch**: OpenBLAS build artifacts are copied as source code, then make install runs in final stage
2. **Runtime permission conflicts**: Final stage USER 1001 cannot write to /usr/local system paths  
3. **Architecture-specific complexity**: Issue only affects ppc64le where OpenBLAS compilation is required

## Proposed Solution

**Recommended Approach**: Stage the complete installation within the builder stage using DESTDIR pattern:

\`\`\`dockerfile
# In openblas-builder stage
source /opt/rh/gcc-toolset-13/enable
make -j\$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
make -j\$(nproc) PREFIX=/usr/local DESTDIR=/openblas-root install

# In final stage  
COPY --from=openblas-builder /openblas-root/usr/local /usr/local
\`\`\`

### Alternative Approaches

1. **Custom prefix approach**: Install to builder-specific path and copy selectively
2. **Archive approach**: Create tar archive in builder, extract in final stage

## Acceptance Criteria

- [ ] OpenBLAS builds successfully in openblas-builder stage with DESTDIR staging
- [ ] Installation artifacts are copied cleanly to final stage without permission errors
- [ ] ppc64le builds complete successfully without OpenBLAS-related failures  
- [ ] Non-ppc64le architectures continue to work with placeholder/skip logic
- [ ] Build time remains reasonable (no significant performance regression)
- [ ] Runtime functionality verified - OpenBLAS libraries accessible to Python packages
- [ ] Multi-architecture CI builds pass consistently

## Implementation Guidance

1. Modify \`openblas-builder\` stage to use DESTDIR pattern for installation staging
2. Update final stage COPY commands to transfer staged artifacts from builder
3. Ensure proper file ownership/permissions during COPY operation
4. Verify PKG_CONFIG_PATH environment variable remains functional
5. Test integration with NumPy, SciPy and other OpenBLAS-dependent packages

## Context

- **PR**: #2460 (ppc64le datascience jupyter image implementation)  
- **Comment**: https://github.com/opendatahub-io/notebooks/pull/2460#discussion_r2351383598
- **Architecture**: ppc64le multi-architecture support
- **Component**: jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu

This issue addresses systematic infrastructure reliability improvements for Power architecture support in ODH notebook images." \
  --assignee jiridanek

echo "Issue created successfully"

Length of output: 82

Perfect! I've successfully created GitHub issue #2466 titled "Stage the OpenBLAS install in the builder and just COPY the files" for tracking the OpenBLAS permission issue resolution.

The issue comprehensively documents:

  • The permission denied errors from your ppc64le build logs
  • Root cause analysis of the multi-stage permission conflicts
  • Technical solution using DESTDIR staging pattern
  • Detailed acceptance criteria for build reliability verification
  • Implementation guidance with code examples
  • Proper context linking to PR ppc64le: implement datascience jupyter image build on ppc64le (#2406) #2460 and the review comment

This follows your established pattern of systematic infrastructure improvements through detailed issue tracking, and has been assigned to you for implementation.

✏️ Learnings added 
Learnt from: jiridanek
PR: opendatahub-io/notebooks#2460
File: jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu:206-221
Timestamp: 2025-09-16T10:39:23.268Z
Learning: jiridanek requested GitHub issue creation for OpenBLAS installation staging during ppc64le builds in jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu during PR #2460 review. Issue #2466 was created addressing permission errors where OpenBLAS make install fails when attempting to write to /usr/local system paths from USER 1001 context in final stage, proposing DESTDIR staging pattern to build and install OpenBLAS artifacts within openblas-builder stage then COPY pre-installed files to final stage, with comprehensive problem description covering specific permission denied errors, detailed technical solution with code examples, clear acceptance criteria for build reliability and multi-architecture compatibility, and proper context linking to PR #2460 review comment, continuing the systematic infrastructure improvement tracking methodology for Power architecture support.

Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure review instructions. You can manage existing learnings in the app.

🧠 Learnings used 
Learnt from: jiridanek
PR: opendatahub-io/notebooks#2432
File: jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu:232-249
Timestamp: 2025-09-12T08:27:00.439Z
Learning: jiridanek requested GitHub issue creation for Rust toolchain availability during s390x builds in jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu during PR #2432 review. The issue addresses PATH availability problems where Rust/cargo installed in cpu-base stage at /opt/.cargo/bin may not be accessible during uv pip install step in jupyter-datascience stage, proposing three solution approaches: immediate environment variable fix, builder stage pattern following codeserver approach, and ENV declaration fix, with comprehensive acceptance criteria covering build reliability, multi-architecture compatibility, and alignment with established patterns, continuing the systematic infrastructure improvement tracking methodology.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2215
File: runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu:0-0
Timestamp: 2025-09-05T12:10:50.856Z
Learning: jiridanek requested GitHub issue creation for Dockerfile environment variable refactoring during PR #2215 review. Issue addresses build-only variables (OPENBLAS_VERSION, ONNX_VERSION, GRPC_PYTHON_BUILD_SYSTEM_OPENSSL) being unnecessarily written to /etc/profile.d/ppc64le.sh in runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu, causing variable duplication across stages, unreliable sourcing in non-login build contexts, and violation of DRY principles. The issue includes comprehensive problem description covering affected lines 30-37, detailed impact analysis of build reliability and maintenance overhead, three solution options with centralized ARG/ENV approach as recommended, clear acceptance criteria for version centralization and build-only variable cleanup, and specific implementation guidance with code examples, continuing the established pattern of systematic infrastructure improvements through detailed issue tracking.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2432
File: jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu:232-249
Timestamp: 2025-09-12T08:27:00.439Z
Learning: jiridanek requested GitHub issue creation for Rust toolchain availability during s390x builds in jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu during PR #2432 review. Issue #2435 was created addressing PATH availability problems where Rust/cargo installed in cpu-base stage at /opt/.cargo/bin may not be accessible during uv pip install step in jupyter-datascience stage, proposing three solution approaches: immediate environment variable fix, builder stage pattern following codeserver approach, and ENV declaration fix, with comprehensive acceptance criteria covering build reliability, multi-architecture compatibility, and alignment with established patterns, continuing the systematic infrastructure improvement tracking methodology.

Learnt from: grdryn
PR: opendatahub-io/notebooks#1320
File: rstudio/rhel9-python-3.11/Dockerfile.cuda:40-42
Timestamp: 2025-07-04T17:08:02.399Z
Learning: In the opendatahub-io/notebooks repository, when using multi-architecture Dockerfiles with BuildKit, the implicit build argument TARGETARCH is automatically available in the global scope for FROM instructions without explicit declaration. However, if TARGETARCH is used within a build stage, it must be declared explicitly within that stage. The current placement pattern (declaring ARG TARGETARCH after FROM instructions that use it) is correct for modern Docker/Podman/Buildah environments and does not require compatibility with older Docker versions.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2215
File: runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu:0-0
Timestamp: 2025-09-05T12:10:50.856Z
Learning: jiridanek requested GitHub issue creation for Dockerfile environment variable refactoring during PR #2215 review. Issue #2311 was created addressing build-only variables (OPENBLAS_VERSION, ONNX_VERSION, GRPC_PYTHON_BUILD_SYSTEM_OPENSSL) being unnecessarily written to /etc/profile.d/ppc64le.sh in runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu, causing variable duplication across stages, unreliable sourcing in non-login build contexts, and violation of DRY principles. The issue includes comprehensive problem description covering affected lines 30-37, detailed impact analysis of build reliability and maintenance overhead, three solution options with centralized ARG/ENV approach as recommended, clear acceptance criteria for version centralization and build-only variable cleanup, and specific implementation guidance with code examples, assigned to jiridanek, continuing the established pattern of systematic infrastructure improvements through detailed issue tracking.

Learnt from: grdryn
PR: opendatahub-io/notebooks#1320
File: rstudio/rhel9-python-3.11/Dockerfile.cuda:34-35
Timestamp: 2025-07-04T10:41:13.061Z
Learning: In the opendatahub-io/notebooks repository, when adapting NVIDIA CUDA Dockerfiles, the project intentionally maintains consistency with upstream NVIDIA patterns even when it might involve potential risks like empty variable expansions in package installation commands. This is considered acceptable because the containers only run on RHEL 9 with known yum/dnf behavior, and upstream consistency is prioritized over defensive coding practices.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2227
File: codeserver/ubi9-python-3.12/Dockerfile.cpu:122-123
Timestamp: 2025-09-05T12:25:09.719Z
Learning: jiridanek requested GitHub issue creation for Docker multi-stage synchronization improvement in codeserver/ubi9-python-3.12/Dockerfile.cpu during PR #2227 review. The issue addresses sentinel file pattern using /tmp/control copied to /dev/null for stage coordination between rpm-base, whl-cache, and codeserver stages, proposing semantic improvements with descriptive file names, inline documentation, and elimination of /dev/null hack while maintaining multi-architecture build functionality for ppc64le support.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2215
File: runtimes/datascience/ubi9-python-3.12/Dockerfile.cpu:0-0
Timestamp: 2025-09-05T11:27:31.040Z
Learning: jiridanek requested GitHub issue creation for build toolchain optimization in datascience runtime during PR #2215 review. Issue #2308 was created addressing unnecessary build dependencies (gcc-toolset-13, cmake, ninja-build, rust, cargo) in final runtime image for ppc64le architecture, covering comprehensive problem analysis with specific line numbers, multiple solution options for builder-only toolchains, clear acceptance criteria for size reduction and security improvement, detailed implementation guidance for package segregation, and proper context linking to PR #2215 review comment, continuing the established pattern of systematic infrastructure improvements through detailed issue tracking.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2227
File: codeserver/ubi9-python-3.12/Dockerfile.cpu:218-218
Timestamp: 2025-09-05T12:29:07.819Z
Learning: jiridanek requested GitHub issue creation for uv multi-stage Docker build architectural investigation during PR #2227 review. The current implementation uses a three-stage build with whl-cache stage for wheel building/caching, base stage for OS setup, and final codeserver stage for offline installation using --offline flag and cache mounts. The pattern separates build phase (internet access, build tools) from install phase (offline, faster) while supporting multi-architecture builds (x86_64, ppc64le) with sentinel file coordination using /tmp/control files.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#1333
File: runtimes/pytorch/ubi9-python-3.12/Dockerfile.cuda:17-25
Timestamp: 2025-07-09T08:07:30.628Z
Learning: jiridanek requested GitHub issue creation for oc client installation permission problem in PyTorch CUDA runtime Dockerfile during PR #1333 review. Issue #1356 was created with comprehensive problem description covering USER 1001 permission conflicts with root-owned /opt/app-root/bin directory, detailed impact analysis of build failures and non-executable binaries, current problematic code snippet, complete solution with user switching approach, clear acceptance criteria, and proper context linking, continuing the established pattern of systematic code quality improvements through detailed issue tracking.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2317
File: codeserver/ubi9-python-3.12/get_code_server_rpm.sh:31-37
Timestamp: 2025-09-05T13:16:48.754Z
Learning: jiridanek requested GitHub issue creation for build tools installation unification across builder images during PR #2317 review. Issue #2322 was created addressing inconsistent build dependency management patterns across different builder images, proposing multiple solution approaches including Development Tools group installation, centralized configuration, and layered approaches, with comprehensive acceptance criteria covering auditing, standardization, regression prevention, and multi-architecture support (x86_64, ppc64le, aarch64, s390x), continuing the established pattern of systematic infrastructure improvements through detailed issue tracking.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#1909
File: runtimes/pytorch+llmcompressor/ubi9-python-3.11/Dockerfile.cuda:11-15
Timestamp: 2025-08-12T08:40:55.286Z
Learning: jiridanek requested GitHub issue creation for redundant CUDA upgrade optimization during PR #1909 review. Analysis revealed all 14 CUDA Dockerfiles contain redundant `yum upgrade -y` commands in cuda-base stages that execute after base stages already performed comprehensive `dnf upgrade` via pre-upgrade blocks, causing unnecessary CI latency and build inefficiency. Issue includes complete scope analysis with specific line numbers, investigation framework requiring NVIDIA upstream documentation review, multiple solution options, comprehensive acceptance criteria covering systematic testing and performance measurement, and proper context linking to PR #1909 review comment.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2432
File: jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu:102-159
Timestamp: 2025-09-12T08:20:45.369Z
Learning: jiridanek requested GitHub issue creation for pyarrow build reproducibility improvement during PR #2432 review. Issue #2433 was created addressing HEAD builds causing reproducibility issues in s390x wheel-builder stage, proposing pinned apache-arrow-20.0.0 tag to match pylock.toml specification, with comprehensive problem description covering version mismatch risks, detailed solution with implementation steps, clear acceptance criteria for build consistency verification, and proper context linking to PR #2432 review comment, continuing the established pattern of systematic infrastructure improvements through detailed issue tracking.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#0
File: :0-0
Timestamp: 2025-08-05T17:24:08.616Z
Learning: jiridanek requested PR review for #1521 covering s390x architecture support improvements, demonstrating continued focus on systematic multi-architecture compatibility enhancements in the opendatahub-io/notebooks repository through clean implementation with centralized configuration, proper CI integration, and architecture-aware testing patterns.

Learnt from: atheo89
PR: opendatahub-io/notebooks#1258
File: codeserver/ubi9-python-3.11/Dockerfile.cpu:32-32
Timestamp: 2025-07-07T11:08:48.524Z
Learning: atheo89 requested GitHub issue creation for multi-architecture Dockerfile improvements during PR #1258 review, specifically for enhancing structural consistency across Docker stages, replacing $(uname -m) with ${TARGETARCH} for cross-architecture builds, and adding OCI-compliant metadata labels. Issue #1332 was created with comprehensive problem description, phased implementation approach, detailed acceptance criteria, implementation guidance with code examples, and proper context linking, continuing the established pattern of systematic code quality improvements.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#2185
File: jupyter/pytorch/ubi9-python-3.12/Dockerfile.cuda:109-117
Timestamp: 2025-08-29T08:48:55.985Z
Learning: jiridanek prefers to implement systematic cleanup improvements through dedicated GitHub issues (like #2076) rather than applying individual point fixes during PR reviews, maintaining consistency with their established pattern of comprehensive code quality improvements.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#1179
File: jupyter/utils/install_pandoc.sh:1-1
Timestamp: 2025-09-05T07:46:50.781Z
Learning: jiridanek requested GitHub issue creation during PR #1179 review to explore installing Pandoc from EPEL repository for ppc64le architecture as an alternative to building from source, noting that EPEL packages are acceptable unlike CentOS Stream in red-hat-data-services/notebooks. Issue #2281 was successfully created with comprehensive problem description covering build complexity concerns, multiple solution options, clear acceptance criteria, and proper context linking, continuing the established pattern of systematic infrastructure improvements through detailed issue tracking.

Learnt from: jiridanek
PR: opendatahub-io/notebooks#1179
File: jupyter/utils/install_pandoc.sh:1-1
Timestamp: 2025-09-05T07:46:50.781Z
Learning: jiridanek requested GitHub issue creation during PR #1179 review to explore installing Pandoc from EPEL repository for ppc64le architecture as an alternative to building from source, noting that EPEL packages are acceptable unlike CentOS Stream in red-hat-data-services/notebooks, continuing the established pattern of systematic infrastructure improvements through comprehensive issue tracking.

####################
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Fix hadolint parse error (heredoc compatibility).

hadolint chokes on the heredoc just above this line. Replace heredocs with a single RUN using bash -lc (or unquote markers across the file). Minimal fix for the failing block shown below.

-FROM common-builder AS openblas-builder
-ARG TARGETARCH
-ARG OPENBLAS_VERSION=0.3.30
-WORKDIR /root
-RUN <<'EOF'
-set -Eeuxo pipefail
-if [ "${TARGETARCH}" = "ppc64le" ]; then
-    wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip
-    unzip OpenBLAS-${OPENBLAS_VERSION}.zip
-    cd OpenBLAS-${OPENBLAS_VERSION}
-    make -j$(nproc) TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0
-else
-    mkdir -p OpenBLAS-${OPENBLAS_VERSION}
-    echo "Skipping OpenBLAS build on non-Power"
-fi
-EOF
+FROM common-builder AS openblas-builder
+ARG TARGETARCH
+ARG OPENBLAS_VERSION=0.3.30
+WORKDIR /root
+RUN bash -lc 'set -Eeuxo pipefail; \
+  if [ "${TARGETARCH}" = "ppc64le" ]; then \
+    wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip; \
+    unzip OpenBLAS-${OPENBLAS_VERSION}.zip; \
+    cd OpenBLAS-${OPENBLAS_VERSION}; \
+    source /opt/rh/gcc-toolset-13/enable; \
+    make -j"$(nproc)" TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0; \
+  else \
+    mkdir -p OpenBLAS-${OPENBLAS_VERSION}; \
+    echo "Skipping OpenBLAS build on non-Power"; \
+  fi'
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
####################
FROM common-builder AS openblas-builder
ARG TARGETARCH
ARG OPENBLAS_VERSION=0.3.30
WORKDIR /root
RUN bash -lc 'set -Eeuxo pipefail; \
if [ "${TARGETARCH}" = "ppc64le" ]; then \
wget https://github.com/OpenMathLib/OpenBLAS/releases/download/v${OPENBLAS_VERSION}/OpenBLAS-${OPENBLAS_VERSION}.zip; \
unzip OpenBLAS-${OPENBLAS_VERSION}.zip; \
cd OpenBLAS-${OPENBLAS_VERSION}; \
source /opt/rh/gcc-toolset-13/enable; \
make -j"$(nproc)" TARGET=POWER9 BINARY=64 USE_OPENMP=1 USE_THREAD=1 NUM_THREADS=120 DYNAMIC_ARCH=1 INTERFACE64=0; \
else \
mkdir -p OpenBLAS-${OPENBLAS_VERSION}; \
echo "Skipping OpenBLAS build on non-Power"; \
fi'
🧰 Tools
🪛 GitHub Actions: Code static analysis

[error] 222-222: Parse error: unexpected '#' - hadolint: expected a new line followed by the next instruction

🤖 Prompt for AI Agents 
In jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu around line 222, hadolint
fails parsing the heredoc used just above this line; replace that heredoc block
with a single RUN that executes the same commands via bash -lc (or alternatively
unquote heredoc markers across the file). Edit the Dockerfile to remove the
multiline heredoc block and consolidate its commands into one RUN "bash -lc
'...'" invocation (preserving commands, environment variables and redirects),
ensuring any inner quotes are escaped or use a single outer quoting strategy so
the shell executes identically while hadolint can parse it.

# jupyter-minimal #
####################
Expand Down Expand Up @@ -191,8 +250,10 @@ ENTRYPOINT ["start-notebook.sh"]
# jupytyer-datascience #
########################
FROM jupyter-minimal AS jupyter-datascience
ARG TARGETARCH

ARG DATASCIENCE_SOURCE_CODE=jupyter/datascience/ubi9-python-3.12
ARG OPENBLAS_VERSION=0.3.30
ARG TARGETARCH

LABEL name="odh-notebook-jupyter-datascience-ubi9-python-3.12" \
Expand All @@ -211,33 +272,72 @@ WORKDIR /opt/app-root/bin
USER root

# Install useful OS packages
RUN dnf install -y jq unixODBC postgresql git-lfs libsndfile libxcrypt-compat && dnf clean all && rm -rf /var/cache/yum
RUN dnf install -y jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat && \
dnf clean all && rm -rf /var/cache/yum

# Copy dynamically-linked mongocli built in earlier build stage
COPY --from=mongocli-builder /tmp/mongocli /opt/app-root/bin/

# Other apps and tools installed as default user
USER 1001

# Copy wheels from build stage (s390x only)
COPY --from=s390x-builder /tmp/wheels /tmp/wheels
RUN if [ "$TARGETARCH" = "s390x" ]; then \
ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/

# Copy wheels from build stage (ppc64le and s390x only)
COPY --from=pyarrow-builder /tmp/wheels /tmp/wheels
RUN if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then \
pip install --no-cache-dir /tmp/wheels/*.whl; \
else \
echo "Skipping wheel install for $TARGETARCH"; \
fi

# Install Python packages and Jupyterlab extensions from requirements.txt
# Copy OpenBLAS,ONNX wheels for Power
COPY --from=openblas-builder /root/OpenBLAS-${OPENBLAS_VERSION} /openblas
COPY --from=onnx-builder /root/onnx_wheel/ /onnxwheels/

# Power-specific ONNX/OpenBLAS installation
RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
pip install /onnxwheels/*.whl
else
echo "Skipping ONNX/OpenBLAS install on non-Power"
fi
EOF

USER root
RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
rm -rf /onnxwheels
else
echo "Skipping ONNX/OpenBLAS install on non-Power"
fi
EOF

RUN <<'EOF'
set -Eeuxo pipefail
if [ "${TARGETARCH}" = "ppc64le" ]; then
PREFIX=/usr/local make -C /openblas install
rm -rf /openblas
else
echo "Skipping ONNX/OpenBLAS install on non-Power"
fi
EOF

USER 1001:0

# Install Python packages and Jupyterlab extensions from pylock.toml
COPY ${DATASCIENCE_SOURCE_CODE}/pylock.toml ./
# Copy Elyra setup to utils so that it's sourced at startup
COPY ${DATASCIENCE_SOURCE_CODE}/setup-elyra.sh ${DATASCIENCE_SOURCE_CODE}/utils ./utils/

RUN --mount=type=cache,target=/root/.cache/pip \
echo "Installing softwares and packages" && \
echo "Installing software and packages" && \
# This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,
# we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
if [ "$TARGETARCH" = "s390x" ]; then \
# For s390x, we need special flags and environment variables for building packages
if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then \
# We need special flags and environment variables when building packages
GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
CFLAGS="-O3" CXXFLAGS="-O3" \
uv pip install --strict --no-deps --no-cache --no-config --no-progress \
Expand Down
Loading
Loading