diff --git a/Makefile b/Makefile index d5ee9bae98..e74f66b309 100644 --- a/Makefile +++ b/Makefile @@ -481,6 +481,41 @@ refresh-pipfilelock-files: scan-image-vulnerabilities: python ci/security-scan/quay_security_analysis.py +ARCH := $(shell uname -m) +ifeq ($(ARCH),amd64) + ARCH := x86_64 +else ifeq ($(ARCH),arm64) + ARCH := aarch64 +endif + +ZIG_VERSION := 0.15.1 +ZIG_BINARY := zig-$(ZIG_VERSION) + +bin/zig-$(ZIG_VERSION): + @echo "Installing Zig $(ZIG_VERSION)..." + TMPDIR=$(shell mktemp -d) + wget --progress=dot:giga https://ziglang.org/download/$(ZIG_VERSION)/zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz + tar -xJf zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz -C $$TMPDIR --strip-components=1 + rm -rf zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz + + printf '%s\n' '#!/bin/sh' 'exec /mnt/zig cc -target s390x-linux-gnu.2.34 "$@"' > $$TMPDIR/zig-cc + printf '%s\n' '#!/bin/sh' 'exec /mnt/zig c++ -target s390x-linux-gnu.2.34 "$@"' > $$TMPDIR/zig-c++ + chmod +x $$TMPDIR/zig-cc + chmod +x $$TMPDIR/zig-c++ + + mv $$TMPDIR bin/zig-$(ZIG_VERSION) + @echo "Zig installed as bin/zig-$(ZIG_VERSION)" + +# This should be .PHONY because it's an alias/action +.PHONY: install-zig +install-zig: bin/zig-$(ZIG_VERSION) + @echo "Zig is ready to use!" + +# Another .PHONY target for cleanup +.PHONY: clean-zig +clean-zig: + rm -rf bin/zig-$(ZIG_VERSION) + # This is used primarily for gen_gha_matrix_jobs.py to we know the set of all possible images we may want to build .PHONY: all-images ifeq ($(RELEASE_PYTHON_VERSION), 3.11) diff --git a/runtimes/minimal/ubi9-python-3.12/Dockerfile.cpu b/runtimes/minimal/ubi9-python-3.12/Dockerfile.cpu index 18f56a95e2..c3a857c467 100644 --- a/runtimes/minimal/ubi9-python-3.12/Dockerfile.cpu +++ b/runtimes/minimal/ubi9-python-3.12/Dockerfile.cpu @@ -13,6 +13,8 @@ WORKDIR /opt/app-root/bin # OS Packages needs to be installed as root USER 0 +RUN ls /mnt && true + # Inject the official UBI 9 repository configuration into the AIPCC base image. # The Quay-based AIPCC image is "repo-less" by default (https://gitlab.com/redhat/rhel-ai/core/base-images/app#repositories), so dnf cannot upgrade or install packages. # By copying ubi.repo from the public UBI 9 image, we enable package management for upgrades and installations. @@ -31,7 +33,7 @@ RUN ARCH=$(uname -m) && \ echo "Detected architecture: $ARCH" && \ PACKAGES="perl mesa-libGL skopeo" && \ if [ "$ARCH" = "s390x" ] || [ "$ARCH" = "ppc64le" ]; then \ - PACKAGES="$PACKAGES gcc g++ make openssl-devel autoconf automake libtool cmake"; \ + PACKAGES="$PACKAGES gcc g++ ninja-build openssl-devel autoconf automake libtool cmake"; \ fi && \ dnf install -y $PACKAGES && \ dnf clean all && rm -rf /var/cache/yum diff --git a/scripts/sandbox.py b/scripts/sandbox.py index a8592b09d9..6df75c426a 100755 --- a/scripts/sandbox.py +++ b/scripts/sandbox.py @@ -47,7 +47,30 @@ def main() -> int: with tempfile.TemporaryDirectory(delete=True) as tmpdir: setup_sandbox(prereqs, pathlib.Path(tmpdir)) - command = [arg if arg != "{};" else tmpdir for arg in args.remaining[1:]] + # target glibc 2.28 or newer (supports FORTIFY_SOURCE) + target = "s390x-linux-gnu.2.34" + additional_arguments = [ + f"--volume={os.getcwd()}/bin/zig-0.15.1:/mnt", + # f"--env=CC=/mnt/zig cc -target {target}", + # f"--env=CXX=/mnt/zig c++ -target {target}", + # f"--env=CC=/mnt/zig-cc", + # f"--env=CXX=/mnt/zig-c++", + # -Wp,-D_FORTIFY_SOURCE=2 + # https://github.com/giampaolo/psutil/blob/master/setup.py#L254 + # defaults to using python's flags + # f"--env=CFLAGS=", + f"--env=bustcachez=", + "--env=CXXFLAGS=-Dundefined=64", + f"--unsetenv=CC", + f"--unsetenv=CXX", + tmpdir, + ] + command = [] + for arg in args.remaining[1:]: + if arg == "{};": + command.extend(additional_arguments) + else: + command.append(arg) print(f"running {command=}") try: subprocess.check_call(command) @@ -56,6 +79,64 @@ def main() -> int: return err.returncode return 0 +""" +Downloading jedi + × Failed to build `pyzmq==27.1.0` + ├─▶ The build backend returned an error + ╰─▶ Call to `scikit_build_core.build.build_wheel` failed (exit status: 1) + [stdout] + *** scikit-build-core 0.11.6 using CMake 3.26.5 (wheel) + *** Configuring CMake... + loading initial cache file /tmp/tmpf9bnfh5o/build/CMakeInit.txt + -- Configuring incomplete, errors occurred! + [stderr] + CMake Error at /usr/share/cmake/Modules/CMakeDetermineCCompiler.cmake:49 + (message): + Could not find compiler set in environment variable CC: + /mnt/zig-0.15.1/zig cc -target s390x-linux-gnu. + Call Stack (most recent call first): + CMakeLists.txt:2 (project) +""" + +""" +creating build/temp.linux-s390x-cpython-312/psutil/arch/linux + /mnt/zig cc -target s390x-linux-gnu -fno-strict-overflow + -Wsign-compare -DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG + -O2 -fexceptions -g -grecord-gcc-switches -pipe + -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 + -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong + -m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables + -fstack-clash-protection -O2 -fexceptions -g -grecord-gcc-switches + -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 + -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong + -m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables + -fstack-clash-protection -O2 -fexceptions -g -grecord-gcc-switches + -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 + -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong + -m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables + -fstack-clash-protection -fPIC -DPSUTIL_POSIX=1 -DPSUTIL_SIZEOF_PID_T=4 + -DPSUTIL_VERSION=700 -DPy_LIMITED_API=0x03060000 + -DPSUTIL_LINUX=1 -I/tmp/.tmpWlL4ZP/builds-v0/.tmpOwAhw2/include + -I/usr/include/python3.12 -c psutil/_psutil_common.c -o + build/temp.linux-s390x-cpython-312/psutil/_psutil_common.o + [stderr] + /tmp/.tmpWlL4ZP/builds-v0/.tmpOwAhw2/lib64/python3.12/site-packages/setuptools/dist.py:759: + SetuptoolsDeprecationWarning: License classifiers are deprecated. + !! + + ******************************************************************************** + Please consider removing the following classifiers in favor of a + SPDX license expression: + License :: OSI Approved :: BSD License + See + https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license + for details. + + ******************************************************************************** + !! + self._finalize_license_expression() + error: unsupported preprocessor arg: -D_FORTIFY_SOURCE +""" def buildinputs( dockerfile: pathlib.Path | str, @@ -63,6 +144,12 @@ def buildinputs( ) -> list[pathlib.Path]: if not (ROOT_DIR / "bin/buildinputs").exists(): subprocess.check_call([MAKE, "bin/buildinputs"], cwd=ROOT_DIR) + if not (ROOT_DIR / "bin/zig-0.15.1").exists(): + subprocess.check_call([MAKE, "bin/zig-0.15.1"], cwd=ROOT_DIR) + if not (ROOT_DIR / "bin/zig-0.15.1/zigcc").exists(): + subprocess.check_call([MAKE, "build"], cwd=ROOT_DIR / "scripts/zigcc") + shutil.copy(ROOT_DIR / "scripts/zigcc/bin/zigcc", ROOT_DIR / "bin/zig-0.15.1/zig-cc") + shutil.copy(ROOT_DIR / "scripts/zigcc/bin/zigcc", ROOT_DIR / "bin/zig-0.15.1/zig-c++") stdout = subprocess.check_output([ROOT_DIR / "bin/buildinputs", str(dockerfile)], text=True, cwd=ROOT_DIR, env={"TARGETPLATFORM": platform, **os.environ}) diff --git a/scripts/zigcc/Makefile b/scripts/zigcc/Makefile new file mode 100644 index 0000000000..23ff791e8a --- /dev/null +++ b/scripts/zigcc/Makefile @@ -0,0 +1,21 @@ +.PHONY: build test clean + +build: bin/zigcc + +# always build for linux and the machine's native architecture +bin/zigcc: *.go go.mod + GOOS=linux go build -o $@ -ldflags="-s -w" -v ./... + +test: + go test -v ./... + +fmt: + go fmt ./... + +vet: + go vet ./... + +clean: + go clean + rm -f bin/* + rmdir bin diff --git a/scripts/zigcc/go.mod b/scripts/zigcc/go.mod new file mode 100644 index 0000000000..e2b750b7bb --- /dev/null +++ b/scripts/zigcc/go.mod @@ -0,0 +1,3 @@ +module zigcc + +go 1.24 diff --git a/scripts/zigcc/zigcc.go b/scripts/zigcc/zigcc.go new file mode 100644 index 0000000000..cb7657570e --- /dev/null +++ b/scripts/zigcc/zigcc.go @@ -0,0 +1,56 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "syscall" +) + +func processArg0(arg0 string) (string, error) { + switch arg0 { + case "zig-cc": + return "cc", nil + case "zig-c++": + return "c++", nil + default: + return "", fmt.Errorf("unknown wrapper name: %s", arg0) + } +} + +func processArgs(args []string) []string { + newArgs := make([]string, 0, len(args)) + for _, arg := range args { + if strings.HasPrefix(arg, "-Wp,") { + newArgs = append(newArgs, strings.Split(arg, ",")[1:]...) + } else { + newArgs = append(newArgs, arg) + } + } + return newArgs +} + +func main() { + arg0 := filepath.Base(os.Args[0]) + subcommand, err := processArg0(arg0) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + + newArgs := make([]string, 0, len(os.Args)+4) + newArgs = append(newArgs, + "/mnt/zig", // Path to the real Zig executable. + subcommand, + "-target", + "s390x-linux-gnu.2.34", + ) + newArgs = append(newArgs, processArgs(os.Args[1:])...) + + env := os.Environ() + if err := syscall.Exec(newArgs[0], newArgs, env); err != nil { + fmt.Fprintf(os.Stderr, "Error executing zig: %v\n", err) + os.Exit(1) + } +} diff --git a/scripts/zigcc/zigcc_test.go b/scripts/zigcc/zigcc_test.go new file mode 100644 index 0000000000..1bea00d6e0 --- /dev/null +++ b/scripts/zigcc/zigcc_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + "reflect" + "testing" +) + +func TestProcessWp(t *testing.T) { + args := []string{"-Wp,-D_FORTIFY_SOURCE=2"} + newArgs := processArgs(args) + if !reflect.DeepEqual(newArgs, []string{"-D_FORTIFY_SOURCE=2"}) { + t.Fatalf("expected -DFOO=bar, got %v", newArgs) + } + for _, tc := range []struct { + args []string + expected []string + }{ + { + args: []string{"-Wp,-D_FORTIFY_SOURCE=2"}, + expected: []string{"-D_FORTIFY_SOURCE=2"}, + }, + { + args: []string{"-Wp,-DNDEBUG,-D_FORTIFY_SOURCE=2"}, + expected: []string{"-DNDEBUG", "-D_FORTIFY_SOURCE=2"}, + }, + } { + t.Run(fmt.Sprint(tc.args), func(t *testing.T) { + newArgs := processArgs(tc.args) + if !reflect.DeepEqual(newArgs, tc.expected) { + t.Fatalf("expected %#v, got %#v", tc.expected, newArgs) + } + }) + } +} diff --git a/wrapper.py b/wrapper.py new file mode 100644 index 0000000000..69d0201a28 --- /dev/null +++ b/wrapper.py @@ -0,0 +1,25 @@ +#! /usr/bin/env python3 +import os +import pathlib +import sys + +def main(): + arg0 = pathlib.Path(sys.argv[0]).name + args = [] + for arg in sys.argv[1:]: + if arg.startswith("-Wp,-D"): + args.append(arg.replace("-Wp,-D", "-D", 1)) + else: + args.append(arg) + + if arg0 == "zig-cc": + args = ["/mnt/zig", "cc", "-target", "s390x-linux-gnu.2.34"] + args + elif arg0 == "zig-c++": + args = ["/mnt/zig", "c++", "-target", "s390x-linux-gnu.2.34"] + args + else: + raise ValueError(f"Unknown argument {arg0}") + + os.execve(args[0], args, os.environ) + +if __name__ == "__main__": + sys.exit(main())