Merge pull request kata-containers#6586 from jongwu/cross_build

kata-deploy: enable cross build for non-x86

ci/install_libseccomp.sh

@@ -87,7 +87,8 @@ build_and_install_libseccomp() {
     curl -sLO "${libseccomp_tarball_url}"
     tar -xf "${libseccomp_tarball}"
     pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
+    [ "${arch}" == $(uname -m) ] && cc_name="" || cc_name="${arch}-linux-gnu-gcc"
+    CC=${cc_name} ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
     make
     make install
     popd

tools/osbuilder/image-builder/image_builder.sh

@@ -13,6 +13,16 @@ set -o pipefail
 DOCKER_RUNTIME=${DOCKER_RUNTIME:-runc}
 MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
 
+#For cross build
+CROSS_BUILD=${CROSS_BUILD:-false}
+BUILDX=""
+PLATFORM=""
+TARGET_ARCH=${TARGET_ARCH:-$(uname -m)}
+ARCH=${ARCH:-$(uname -m)}
+[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64
+TARGET_OS=${TARGET_OS:-linux}
+[ "${CROSS_BUILD}" == "true" ] && BUILDX=buildx && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}"
+
 readonly script_name="${0##*/}"
 readonly script_dir=$(dirname "$(readlink -f "$0")")
 readonly lib_file="${script_dir}/../scripts/lib.sh"
@@ -154,7 +164,7 @@ build_with_container() {
 		engine_build_args+=" --runtime ${DOCKER_RUNTIME}"
 	fi
 
-	"${container_engine}" build  \
+	"${container_engine}" ${BUILDX} build ${PLATFORM}  \
 		   ${engine_build_args} \
 		   --build-arg http_proxy="${http_proxy}" \
 		   --build-arg https_proxy="${https_proxy}" \
@@ -189,6 +199,8 @@ build_with_container() {
 		   --env MEASURED_ROOTFS="${MEASURED_ROOTFS}" \
 		   --env SELINUX="${SELINUX}" \
 		   --env DEBUG="${DEBUG}" \
+		   --env ARCH="${ARCH}" \
+		   --env TARGET_ARCH="${TARGET_ARCH}" \
 		   -v /dev:/dev \
 		   -v "${script_dir}":"/osbuilder" \
 		   -v "${script_dir}/../scripts":"/scripts" \

tools/osbuilder/rootfs-builder/rootfs.sh

@@ -31,6 +31,16 @@ SELINUX=${SELINUX:-"no"}
 lib_file="${script_dir}/../scripts/lib.sh"
 source "$lib_file"
 
+#For cross build
+CROSS_BUILD=${CROSS_BUILD:-false}
+BUILDX=""
+PLATFORM=""
+TARGET_ARCH=${TARGET_ARCH:-$(uname -m)}
+ARCH=${ARCH:-$(uname -m)}
+[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64
+TARGET_OS=${TARGET_OS:-linux}
+[ "${CROSS_BUILD}" == "true" ] && BUILDX=buildx && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}"
+
 handle_error() {
 	local exit_code="${?}"
 	local line_number="${1:-}"

tools/osbuilder/rootfs-builder/ubuntu/Dockerfile.in

@@ -8,6 +8,7 @@ FROM ${IMAGE_REGISTRY}/ubuntu:@OS_VERSION@
 
 # makedev tries to mknod from postinst
 RUN [ -x /usr/bin/systemd-detect-virt ] || ( echo "echo docker" >/usr/bin/systemd-detect-virt && chmod +x /usr/bin/systemd-detect-virt )
+# hadolint ignore=DL3009,SC2046
 RUN apt-get update && \
     DEBIAN_FRONTEND=noninteractive \
     apt-get --no-install-recommends -y install \
@@ -18,6 +19,7 @@ RUN apt-get update && \
          libc_arch="$gcc_arch" && \
          [ "$gcc_arch" = aarch64 ] && libc_arch=arm64; \
          [ "$gcc_arch" = ppc64le ] && gcc_arch=powerpc64le && libc_arch=ppc64el; \
+         [ "$gcc_arch" = s390x ] && gcc_arch=s390x && libc_arch=s390x; \
          [ "$gcc_arch" = x86_64 ] && gcc_arch=x86-64 && libc_arch=amd64; \
          echo "gcc-$gcc_arch-linux-gnu libc6-dev-$libc_arch-cross")) \
     git \

tools/packaging/guest-image/build_image.sh

@@ -21,7 +21,13 @@ readonly osbuilder_dir="$(cd "${repo_root_dir}/tools/osbuilder" && pwd)"
 
 export GOPATH=${GOPATH:-${HOME}/go}
 
-arch_target="$(uname -m)"
+ARCH=${ARCH:-$(uname -m)}
+if [ $(uname -m) == "${ARCH}" ]; then
+       arch_target="$(uname -m)"
+else
+       arch_target="${ARCH}"
+fi
+
 final_artifact_name="kata-containers"
 image_initrd_extension=".img"
 

tools/packaging/kata-deploy/local-build/kata-deploy-binaries-in-docker.sh

@@ -19,6 +19,29 @@ gid=$(id -g ${USER})
 http_proxy="${http_proxy:-}"
 https_proxy="${https_proxy:-}"
 
+ARCH=${ARCH:-$(uname -m)}
+CROSS_BUILD=
+BUILDX=""
+PLATFORM=""
+TARGET_ARCH=${TARGET_ARCH:-$(uname -m)}
+[ "$(uname -m)" != "${TARGET_ARCH}" ] && CROSS_BUILD=true
+
+[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64
+
+# used for cross build
+TARGET_OS=${TARGET_OS:-linux}
+TARGET_ARCH=${TARGET_ARCH:-$ARCH}
+
+[ "${CROSS_BUILD}" == "true" ] && BUILDX="buildx" && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}"
+if [ "${CROSS_BUILD}" == "true" ]; then
+       # check if the current docker support docker buildx
+       docker buildx ls > /dev/null 2>&1 || true
+       [ $? != 0 ] && echo "no docker buildx support, please upgrad your docker" && exit 1
+       # check if docker buildx support target_arch, if not install it
+       r=$(docker buildx ls | grep "${TARGET_ARCH}" || true)
+       [ -z "$r" ] && sudo docker run --privileged --rm tonistiigi/binfmt --install ${TARGET_ARCH}
+fi
+
 if [ "${script_dir}" != "${PWD}" ]; then
 	ln -sf "${script_dir}/build" "${PWD}/build"
 fi
@@ -66,6 +89,9 @@ docker run \
 	--env VIRTIOFSD_CONTAINER_BUILDER="${VIRTIOFSD_CONTAINER_BUILDER:-}" \
 	--env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \
 	--env USE_CACHE="${USE_CACHE:-}" \
+	--env CROSS_BUILD="${CROSS_BUILD}" \
+	--env TARGET_ARCH="${TARGET_ARCH}" \
+	--env ARCH="${ARCH}" \
 	--rm \
 	-w ${script_dir} \
 	build-kata-deploy "${kata_deploy_create}" $@

tools/packaging/kata-deploy/local-build/kata-deploy-binaries.sh

@@ -38,7 +38,7 @@ readonly rootfs_builder="${repo_root_dir}/tools/packaging/guest-image/build_imag
 readonly jenkins_url="http://jenkins.katacontainers.io"
 readonly cached_artifacts_path="lastSuccessfulBuild/artifact/artifacts"
 
-ARCH=$(uname -m)
+ARCH=${ARCH:-$(uname -m)}
 MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
 USE_CACHE="${USE_CACHE:-"yes"}"
 
@@ -150,7 +150,7 @@ install_image() {
 		image_type+="-${variant}"
 	fi
 
-	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-$(uname -m)/${cached_artifacts_path}"
+	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${image_type}-${ARCH}/${cached_artifacts_path}"
 	local component="rootfs-${image_type}"
 
 	local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")"
@@ -197,7 +197,7 @@ install_initrd() {
 		initrd_type+="-${variant}"
 	fi
 
-	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-$(uname -m)/${cached_artifacts_path}"
+	local jenkins="${jenkins_url}/job/kata-containers-main-rootfs-${initrd_type}-${ARCH}/${cached_artifacts_path}"
 	local component="rootfs-${initrd_type}"
 
 	local osbuilder_last_commit="$(get_last_modification "${repo_root_dir}/tools/osbuilder")"
@@ -208,6 +208,8 @@ install_initrd() {
 	local libseccomp_version="$(get_from_kata_deps "externals.libseccomp.version")"
 	local rust_version="$(get_from_kata_deps "languages.rust.meta.newest-version")"
 
+	[[ "${ARCH}" == "aarch64" && "${CROSS_BUILD}" == "true" ]] && echo "warning: Don't cross build initrd for aarch64 as it's too slow" && exit 0
+
 	install_cached_tarball_component \
 		"${component}" \
 		"${jenkins}" \
@@ -247,7 +249,7 @@ install_cached_kernel_tarball_component() {
 
 	install_cached_tarball_component \
 		"${kernel_name}" \
-		"${jenkins_url}/job/kata-containers-main-${kernel_name}-$(uname -m)/${cached_artifacts_path}" \
+		"${jenkins_url}/job/kata-containers-main-${kernel_name}-${ARCH}/${cached_artifacts_path}" \
 		"${kernel_version}-${kernel_kata_config_version}-$(get_last_modification $(dirname $kernel_builder))" \
 		"$(get_kernel_image_name)" \
 		"${final_tarball_name}" \
@@ -384,7 +386,7 @@ install_qemu_helper() {
 
 	install_cached_tarball_component \
 		"${qemu_name}" \
-		"${jenkins_url}/job/kata-containers-main-${qemu_name}-$(uname -m)/${cached_artifacts_path}" \
+		"${jenkins_url}/job/kata-containers-main-${qemu_name}-${ARCH}/${cached_artifacts_path}" \
 		"${qemu_version}-$(calc_qemu_files_sha256sum)" \
 		"$(get_qemu_image_name)" \
 		"${final_tarball_name}" \
@@ -495,7 +497,7 @@ install_clh_glibc() {
 install_virtiofsd() {
 	install_cached_tarball_component \
 		"virtiofsd" \
-		"${jenkins_url}/job/kata-containers-main-virtiofsd-$(uname -m)/${cached_artifacts_path}" \
+		"${jenkins_url}/job/kata-containers-main-virtiofsd-${ARCH}/${cached_artifacts_path}" \
 		"$(get_from_kata_deps "externals.virtiofsd.version")-$(get_from_kata_deps "externals.virtiofsd.toolchain")" \
 		"$(get_virtiofsd_image_name)" \
 		"${final_tarball_name}" \
@@ -542,7 +544,7 @@ install_shimv2() {
 
 	install_cached_tarball_component \
 		"shim-v2" \
-		"${jenkins_url}/job/kata-containers-main-shim-v2-$(uname -m)/${cached_artifacts_path}" \
+		"${jenkins_url}/job/kata-containers-main-shim-v2-${ARCH}/${cached_artifacts_path}" \
 		"${shim_v2_version}" \
 		"$(get_shim_v2_image_name)" \
 		"${final_tarball_name}" \

tools/packaging/kernel/build-kernel.sh

@@ -65,6 +65,8 @@ kernel_url=""
 #Linux headers for GPU guest fs module building
 linux_headers=""
 
+CROSS_BUILD_ARG=""
+
 MEASURED_ROOTFS=${MEASURED_ROOTFS:-no}
 
 packaging_scripts_dir="${script_dir}/../scripts"
@@ -436,7 +438,7 @@ setup_kernel() {
 
 	info "Copying config file from: ${kernel_config_path}"
 	cp "${kernel_config_path}" ./.config
-	make oldconfig
+	ARCH=${arch_target}  make oldconfig ${CROSS_BUILD_ARG}
 	)
 }
 
@@ -447,7 +449,7 @@ build_kernel() {
 	[ -n "${arch_target}" ] || arch_target="$(uname -m)"
 	arch_target=$(arch_to_kernel "${arch_target}")
 	pushd "${kernel_path}" >>/dev/null
-	make -j $(nproc ${CI:+--ignore 1}) ARCH="${arch_target}"
+	make -j $(nproc ${CI:+--ignore 1}) ARCH="${arch_target}" ${CROSS_BUILD_ARG}
 	if [ "${conf_guest}" == "sev" ]; then
 		make -j $(nproc ${CI:+--ignore 1}) INSTALL_MOD_STRIP=1 INSTALL_MOD_PATH=${kernel_path} modules_install
 	fi
@@ -658,6 +660,8 @@ main() {
 
 	info "Kernel version: ${kernel_version}"
 
+	[ "${arch_target}" != "" -a "${arch_target}" != $(uname -m) ] && CROSS_BUILD_ARG="CROSS_COMPILE=${arch_target}-linux-gnu-"
+
 	case "${subcmd}" in
 		build)
 			build_kernel "${kernel_path}"

tools/packaging/kernel/kata_config_version

@@ -1 +1 @@
-111
+112

tools/packaging/scripts/lib.sh

@@ -19,6 +19,16 @@ short_commit_length=10
 
 hub_bin="hub-bin"
 
+#for cross build
+CROSS_BUILD=${CROSS_BUILD-:}
+BUILDX=""
+PLATFORM=""
+TARGET_ARCH=${TARGET_ARCH:-$(uname -m)}
+ARCH=${ARCH:-$(uname -m)}
+[ "${TARGET_ARCH}" == "aarch64" ] && TARGET_ARCH=arm64
+TARGET_OS=${TARGET_OS:-linux}
+[ "${CROSS_BUILD}" == "true" ] && BUILDX=buildx && PLATFORM="--platform=${TARGET_OS}/${TARGET_ARCH}"
+
 clone_tests_repo() {
 	# KATA_CI_NO_NETWORK is (has to be) ignored if there is
 	# no existing clone.
@@ -189,7 +199,7 @@ get_ovmf_image_name() {
 }
 
 get_virtiofsd_image_name() {
-	ARCH=$(uname -m)
+	ARCH=${ARCH:-$(uname -m)}
 	case ${ARCH} in
 	        "aarch64")
 	                libc="musl"

tools/packaging/static-build/kernel/Dockerfile

@@ -5,6 +5,8 @@
 FROM ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive
 
+ARG ARCH
+
 # kernel deps
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -23,4 +25,5 @@ RUN apt-get update && \
 	    rsync \
 	    cpio \
 	    patch && \
-    apt-get clean && apt-get autoclean
+    if [ "${ARCH}" != "$(uname -m)" ]; then apt-get install --no-install-recommends -y gcc-"${ARCH}"-linux-gnu binutils-"${ARCH}"-linux-gnu; fi && \
+    apt-get clean && apt-get autoclean && rm -rf /var/lib/apt/lists/*

tools/packaging/static-build/kernel/build.sh

@@ -14,34 +14,48 @@ source "${script_dir}/../../scripts/lib.sh"
 
 readonly kernel_builder="${repo_root_dir}/tools/packaging/kernel/build-kernel.sh"
 
+BUILDX=
+PLATFORM=
+
 DESTDIR=${DESTDIR:-${PWD}}
 PREFIX=${PREFIX:-/opt/kata}
 container_image="${KERNEL_CONTAINER_BUILDER:-$(get_kernel_image_name)}"
 
+if [ "${CROSS_BUILD}" == "true" ]; then
+       container_image="${container_image}-${ARCH}-cross-build"
+       # Need to build a s390x image due to an issue at
+       # https://github.com/kata-containers/kata-containers/pull/6586#issuecomment-1603189242
+       if [ ${ARCH} == "s390x" ]; then
+               BUILDX="buildx"
+               PLATFORM="--platform=linux/s390x"
+       fi
+fi
+
 sudo docker pull ${container_image} || \
-	(sudo docker build -t "${container_image}" "${script_dir}" && \
+	(sudo docker ${BUILDX} build ${PLATFORM} \
+	--build-arg ARCH=${ARCH} -t "${container_image}" "${script_dir}" && \
 	 # No-op unless PUSH_TO_REGISTRY is exported as "yes"
 	 push_to_registry "${container_image}")
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env MEASURED_ROOTFS="${MEASURED_ROOTFS:-}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} $* setup"
+	bash -c "${kernel_builder} -a ${ARCH} $* setup"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} $* build"
+	bash -c "${kernel_builder} -a ${ARCH} $* build"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} $* install"
+	bash -c "${kernel_builder} -a ${ARCH} $* install"
 
 sudo docker run --rm -i -v "${repo_root_dir}:${repo_root_dir}" \
 	-w "${PWD}" \
 	--env DESTDIR="${DESTDIR}" --env PREFIX="${PREFIX}" \
 	"${container_image}" \
-	bash -c "${kernel_builder} $* build-headers"
+	bash -c "${kernel_builder} -a ${ARCH} $* build-headers"