Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,41 @@ refresh-pipfilelock-files:
scan-image-vulnerabilities:
python ci/security-scan/quay_security_analysis.py

ARCH := $(shell uname -m)
ifeq ($(ARCH),amd64)
ARCH := x86_64
else ifeq ($(ARCH),arm64)
ARCH := aarch64
endif

ZIG_VERSION := 0.15.1
ZIG_BINARY := zig-$(ZIG_VERSION)

bin/zig-$(ZIG_VERSION):
@echo "Installing Zig $(ZIG_VERSION)..."
TMPDIR=$(shell mktemp -d)
wget --progress=dot:giga https://ziglang.org/download/$(ZIG_VERSION)/zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz
tar -xJf zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz -C $$TMPDIR --strip-components=1
rm -rf zig-$(ARCH)-linux-$(ZIG_VERSION).tar.xz

printf '%s\n' '#!/bin/sh' 'exec /mnt/zig cc -target s390x-linux-gnu.2.34 "$@"' > $$TMPDIR/zig-cc
printf '%s\n' '#!/bin/sh' 'exec /mnt/zig c++ -target s390x-linux-gnu.2.34 "$@"' > $$TMPDIR/zig-c++
chmod +x $$TMPDIR/zig-cc
chmod +x $$TMPDIR/zig-c++

mv $$TMPDIR bin/zig-$(ZIG_VERSION)
@echo "Zig installed as bin/zig-$(ZIG_VERSION)"

# This should be .PHONY because it's an alias/action
.PHONY: install-zig
install-zig: bin/zig-$(ZIG_VERSION)
@echo "Zig is ready to use!"

# Another .PHONY target for cleanup
.PHONY: clean-zig
clean-zig:
rm -rf bin/zig-$(ZIG_VERSION)

# This is used primarily for gen_gha_matrix_jobs.py to we know the set of all possible images we may want to build
.PHONY: all-images
ifeq ($(RELEASE_PYTHON_VERSION), 3.11)
Expand Down
4 changes: 3 additions & 1 deletion runtimes/minimal/ubi9-python-3.12/Dockerfile.cpu
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ WORKDIR /opt/app-root/bin
# OS Packages needs to be installed as root
USER 0

RUN ls /mnt && true

# Inject the official UBI 9 repository configuration into the AIPCC base image.
# The Quay-based AIPCC image is "repo-less" by default (https://gitlab.com/redhat/rhel-ai/core/base-images/app#repositories), so dnf cannot upgrade or install packages.
# By copying ubi.repo from the public UBI 9 image, we enable package management for upgrades and installations.
Expand All @@ -31,7 +33,7 @@ RUN ARCH=$(uname -m) && \
echo "Detected architecture: $ARCH" && \
PACKAGES="perl mesa-libGL skopeo" && \
if [ "$ARCH" = "s390x" ] || [ "$ARCH" = "ppc64le" ]; then \
PACKAGES="$PACKAGES gcc g++ make openssl-devel autoconf automake libtool cmake"; \
PACKAGES="$PACKAGES gcc g++ ninja-build openssl-devel autoconf automake libtool cmake"; \
fi && \
Comment on lines 35 to 37
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Restore make for non-ninja builds on s390x/ppc64le

Dropping make here leaves those arches without the classic build tool that many Python/C extensions still expect, so pip install of projects that call make will now fail. Please keep make while adding ninja-build.

-        PACKAGES="$PACKAGES gcc g++ ninja-build openssl-devel autoconf automake libtool cmake"; \
+        PACKAGES="$PACKAGES gcc g++ make ninja-build openssl-devel autoconf automake libtool cmake"; \
🤖 Prompt for AI Agents
In runtimes/minimal/ubi9-python-3.12/Dockerfile.cpu around lines 35 to 37, the
s390x/ppc64le PACKAGES list removed make which breaks builds that invoke make;
update the PACKAGES assignment to include make alongside ninja-build (e.g., add
"make" to the quoted package list) so both classic make and ninja are available
for those architectures.

dnf install -y $PACKAGES && \
dnf clean all && rm -rf /var/cache/yum
Expand Down
89 changes: 88 additions & 1 deletion scripts/sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,30 @@ def main() -> int:

with tempfile.TemporaryDirectory(delete=True) as tmpdir:
setup_sandbox(prereqs, pathlib.Path(tmpdir))
command = [arg if arg != "{};" else tmpdir for arg in args.remaining[1:]]
# target glibc 2.28 or newer (supports FORTIFY_SOURCE)
target = "s390x-linux-gnu.2.34"
additional_arguments = [
f"--volume={os.getcwd()}/bin/zig-0.15.1:/mnt",
# f"--env=CC=/mnt/zig cc -target {target}",
# f"--env=CXX=/mnt/zig c++ -target {target}",
# f"--env=CC=/mnt/zig-cc",
# f"--env=CXX=/mnt/zig-c++",
# -Wp,-D_FORTIFY_SOURCE=2
# https://github.com/giampaolo/psutil/blob/master/setup.py#L254
# defaults to using python's flags
# f"--env=CFLAGS=",
f"--env=bustcachez=",
"--env=CXXFLAGS=-Dundefined=64",
f"--unsetenv=CC",
f"--unsetenv=CXX",
tmpdir,
]
command = []
for arg in args.remaining[1:]:
if arg == "{};":
command.extend(additional_arguments)
else:
command.append(arg)
print(f"running {command=}")
try:
subprocess.check_call(command)
Expand All @@ -56,13 +79,77 @@ def main() -> int:
return err.returncode
return 0

"""
Downloading jedi
× Failed to build `pyzmq==27.1.0`
├─▶ The build backend returned an error
╰─▶ Call to `scikit_build_core.build.build_wheel` failed (exit status: 1)
[stdout]
*** scikit-build-core 0.11.6 using CMake 3.26.5 (wheel)
*** Configuring CMake...
loading initial cache file /tmp/tmpf9bnfh5o/build/CMakeInit.txt
-- Configuring incomplete, errors occurred!
[stderr]
CMake Error at /usr/share/cmake/Modules/CMakeDetermineCCompiler.cmake:49
(message):
Could not find compiler set in environment variable CC:
/mnt/zig-0.15.1/zig cc -target s390x-linux-gnu.
Call Stack (most recent call first):
CMakeLists.txt:2 (project)
"""

"""
creating build/temp.linux-s390x-cpython-312/psutil/arch/linux
/mnt/zig cc -target s390x-linux-gnu -fno-strict-overflow
-Wsign-compare -DDYNAMIC_ANNOTATIONS_ENABLED=1 -DNDEBUG
-O2 -fexceptions -g -grecord-gcc-switches -pipe
-Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2
-Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong
-m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables
-fstack-clash-protection -O2 -fexceptions -g -grecord-gcc-switches
-pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2
-Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong
-m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables
-fstack-clash-protection -O2 -fexceptions -g -grecord-gcc-switches
-pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2
-Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong
-m64 -march=z14 -mtune=z15 -fasynchronous-unwind-tables
-fstack-clash-protection -fPIC -DPSUTIL_POSIX=1 -DPSUTIL_SIZEOF_PID_T=4
-DPSUTIL_VERSION=700 -DPy_LIMITED_API=0x03060000
-DPSUTIL_LINUX=1 -I/tmp/.tmpWlL4ZP/builds-v0/.tmpOwAhw2/include
-I/usr/include/python3.12 -c psutil/_psutil_common.c -o
build/temp.linux-s390x-cpython-312/psutil/_psutil_common.o
[stderr]
/tmp/.tmpWlL4ZP/builds-v0/.tmpOwAhw2/lib64/python3.12/site-packages/setuptools/dist.py:759:
SetuptoolsDeprecationWarning: License classifiers are deprecated.
!!

********************************************************************************
Please consider removing the following classifiers in favor of a
SPDX license expression:
License :: OSI Approved :: BSD License
See
https://packaging.python.org/en/latest/guides/writing-pyproject-toml/#license
for details.

********************************************************************************
!!
self._finalize_license_expression()
error: unsupported preprocessor arg: -D_FORTIFY_SOURCE
"""

def buildinputs(
dockerfile: pathlib.Path | str,
platform: Literal["linux/amd64", "linux/arm64", "linux/s390x", "linux/ppc64le"] = "linux/amd64"
) -> list[pathlib.Path]:
if not (ROOT_DIR / "bin/buildinputs").exists():
subprocess.check_call([MAKE, "bin/buildinputs"], cwd=ROOT_DIR)
if not (ROOT_DIR / "bin/zig-0.15.1").exists():
subprocess.check_call([MAKE, "bin/zig-0.15.1"], cwd=ROOT_DIR)
if not (ROOT_DIR / "bin/zig-0.15.1/zigcc").exists():
subprocess.check_call([MAKE, "build"], cwd=ROOT_DIR / "scripts/zigcc")
shutil.copy(ROOT_DIR / "scripts/zigcc/bin/zigcc", ROOT_DIR / "bin/zig-0.15.1/zig-cc")
shutil.copy(ROOT_DIR / "scripts/zigcc/bin/zigcc", ROOT_DIR / "bin/zig-0.15.1/zig-c++")
stdout = subprocess.check_output([ROOT_DIR / "bin/buildinputs", str(dockerfile)],
text=True, cwd=ROOT_DIR,
env={"TARGETPLATFORM": platform, **os.environ})
Expand Down
21 changes: 21 additions & 0 deletions scripts/zigcc/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
.PHONY: build test clean

build: bin/zigcc

# always build for linux and the machine's native architecture
bin/zigcc: *.go go.mod
GOOS=linux go build -o $@ -ldflags="-s -w" -v ./...

test:
go test -v ./...

fmt:
go fmt ./...

vet:
go vet ./...

clean:
go clean
rm -f bin/*
rmdir bin
3 changes: 3 additions & 0 deletions scripts/zigcc/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module zigcc

go 1.24
Comment on lines +1 to +3
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Target a released Go toolchain

go 1.24 is not available yet, so current builders (Go ≤1.22/1.23) will abort with “requires go >=1.24”. Drop to the latest released version you actually need (e.g. 1.22).

-go 1.24
+go 1.22
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
module zigcc
go 1.24
module zigcc
go 1.22
🤖 Prompt for AI Agents
In scripts/zigcc/go.mod around lines 1 to 3, the module specifies "go 1.24"
which is not yet released and will abort builds on current Go toolchains; change
the go directive to an actual released version your project needs (for example
"go 1.22" or the minimum released version that supports the features you use),
update any CI/docker builders to match that version if necessary, and run go mod
tidy/build locally to verify compatibility.

56 changes: 56 additions & 0 deletions scripts/zigcc/zigcc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package main

import (
"fmt"
"os"
"path/filepath"
"strings"
"syscall"
)

func processArg0(arg0 string) (string, error) {
switch arg0 {
case "zig-cc":
return "cc", nil
case "zig-c++":
return "c++", nil
default:
return "", fmt.Errorf("unknown wrapper name: %s", arg0)
}
}

func processArgs(args []string) []string {
newArgs := make([]string, 0, len(args))
for _, arg := range args {
if strings.HasPrefix(arg, "-Wp,") {
newArgs = append(newArgs, strings.Split(arg, ",")[1:]...)
} else {
newArgs = append(newArgs, arg)
}
}
return newArgs
}

func main() {
arg0 := filepath.Base(os.Args[0])
subcommand, err := processArg0(arg0)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}

newArgs := make([]string, 0, len(os.Args)+4)
newArgs = append(newArgs,
"/mnt/zig", // Path to the real Zig executable.
subcommand,
"-target",
"s390x-linux-gnu.2.34",
)
newArgs = append(newArgs, processArgs(os.Args[1:])...)

env := os.Environ()
if err := syscall.Exec(newArgs[0], newArgs, env); err != nil {
fmt.Fprintf(os.Stderr, "Error executing zig: %v\n", err)
os.Exit(1)
}
}
35 changes: 35 additions & 0 deletions scripts/zigcc/zigcc_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package main

import (
"fmt"
"reflect"
"testing"
)

func TestProcessWp(t *testing.T) {
args := []string{"-Wp,-D_FORTIFY_SOURCE=2"}
newArgs := processArgs(args)
if !reflect.DeepEqual(newArgs, []string{"-D_FORTIFY_SOURCE=2"}) {
t.Fatalf("expected -DFOO=bar, got %v", newArgs)
}
for _, tc := range []struct {
args []string
expected []string
}{
{
args: []string{"-Wp,-D_FORTIFY_SOURCE=2"},
expected: []string{"-D_FORTIFY_SOURCE=2"},
},
{
args: []string{"-Wp,-DNDEBUG,-D_FORTIFY_SOURCE=2"},
expected: []string{"-DNDEBUG", "-D_FORTIFY_SOURCE=2"},
},
} {
t.Run(fmt.Sprint(tc.args), func(t *testing.T) {
newArgs := processArgs(tc.args)
if !reflect.DeepEqual(newArgs, tc.expected) {
t.Fatalf("expected %#v, got %#v", tc.expected, newArgs)
}
})
}
}
25 changes: 25 additions & 0 deletions wrapper.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#! /usr/bin/env python3
import os
import pathlib
import sys

def main():
arg0 = pathlib.Path(sys.argv[0]).name
args = []
for arg in sys.argv[1:]:
if arg.startswith("-Wp,-D"):
args.append(arg.replace("-Wp,-D", "-D", 1))
else:
args.append(arg)

if arg0 == "zig-cc":
args = ["/mnt/zig", "cc", "-target", "s390x-linux-gnu.2.34"] + args
elif arg0 == "zig-c++":
args = ["/mnt/zig", "c++", "-target", "s390x-linux-gnu.2.34"] + args
Comment on lines +9 to +18
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Fix -Wp flag rewriting

Replacing -Wp,-DNDEBUG,-D_FORTIFY_SOURCE=2 with a single -DNDEBUG,-D_FORTIFY_SOURCE=2 string prevents the compiler from seeing either define. Split the flag like the Go shim does.

-    for arg in sys.argv[1:]:
-        if arg.startswith("-Wp,-D"):
-            args.append(arg.replace("-Wp,-D", "-D", 1))
-        else:
-            args.append(arg)
+    for arg in sys.argv[1:]:
+        if arg.startswith("-Wp,"):
+            args.extend(part for part in arg.split(",")[1:] if part)
+        else:
+            args.append(arg)
🧰 Tools
🪛 Ruff (0.14.4)

16-16: Consider iterable unpacking instead of concatenation

Replace with iterable unpacking

(RUF005)


18-18: Consider iterable unpacking instead of concatenation

Replace with iterable unpacking

(RUF005)

else:
raise ValueError(f"Unknown argument {arg0}")

os.execve(args[0], args, os.environ)

if __name__ == "__main__":
sys.exit(main())
Loading