Port redpanda to s390x (big-endian)

MODULE.bazel

@@ -9,6 +9,10 @@ archive_override(
     integrity = "sha256-wQsxOxZh1tluc6AXj+ncsYhBSl03uC+hPhBrU6JkupI=",
     strip_prefix = "rules_boost-f5b0f8c904f2487d8f5a9a956d4388724e627210",
     urls = ["https://github.com/nelhage/rules_boost/archive/f5b0f8c904f2487d8f5a9a956d4388724e627210.tar.gz"],
+    patch_strip = 1,
+    patches = [
+        "//bazel/thirdparty:rules-boost-s390x-context.patch",
+    ],
 )
 
 non_module_boost_repositories = use_extension("@com_github_nelhage_rules_boost//:boost/repositories.bzl", "non_module_dependencies")
@@ -21,6 +25,13 @@ bazel_dep(name = "gazelle", version = "0.45.0")
 bazel_dep(name = "abseil-cpp", version = "20250814.1")
 bazel_dep(name = "bazel_skylib", version = "1.8.2")
 bazel_dep(name = "crc32c", version = "1.1.0")
+single_version_override(
+    module_name = "crc32c",
+    patch_strip = 1,
+    patches = [
+        "//bazel/thirdparty:crc32c-big-endian-config.patch",
+    ],
+)
 bazel_dep(name = "fmt", version = "12.1.0")
 bazel_dep(name = "googletest", version = "1.17.0")
 bazel_dep(name = "lexy", version = "2025.05.0")

bazel/BUILD

@@ -94,3 +94,8 @@ config_setting(
         ":antithesis": "true",
     },
 )
+
+config_setting(
+    name = "s390x",
+    constraint_values = ["@platforms//cpu:s390x"],
+)

bazel/test.bzl

@@ -75,7 +75,6 @@ def _test_options():
     data = [
         "//:ubsan_suppressions",
         "//:lsan_suppressions",
-        "@current_llvm_toolchain//:llvm-symbolizer",
     ]
     env = {
         "BOOST_TEST_LOG_LEVEL": "test_suite",
@@ -84,7 +83,6 @@ def _test_options():
         "BOOST_TEST_REPORT_LEVEL": "no",
         "BOOST_LOGGER": "HRF,test_suite",
         "ASAN_OPTIONS": "disable_coredump=0:abort_on_error=1",
-        "ASAN_SYMBOLIZER_PATH": "$(rootpath @current_llvm_toolchain//:llvm-symbolizer)",
         "LSAN_OPTIONS": "suppressions=$(rootpath //:lsan_suppressions)",
         "UBSAN_OPTIONS": "symbolize=1:print_stacktrace=1:halt_on_error=1:abort_on_error=1:report_error_type=1:suppressions=$(rootpath //:ubsan_suppressions)",
         # see https://redpandadata.atlassian.net/wiki/x/BwDSUw
@@ -93,6 +91,23 @@ def _test_options():
     deps = antithesis_deps()
     return data, env, deps
 
+# The hermetic LLVM toolchain has no s390x distribution, so the
+# llvm-symbolizer target cannot be fetched there. Sanitizers are not used
+# on s390x, so drop the symbolizer data dep and ASAN_SYMBOLIZER_PATH on that
+# arch while keeping them everywhere else.
+_SYMBOLIZER_DATA = select({
+    "//bazel:s390x": [],
+    "//conditions:default": ["@current_llvm_toolchain//:llvm-symbolizer"],
+})
+
+def _symbolizer_env(base_env):
+    return select({
+        "//bazel:s390x": base_env,
+        "//conditions:default": base_env | {
+            "ASAN_SYMBOLIZER_PATH": "$(rootpath @current_llvm_toolchain//:llvm-symbolizer)",
+        },
+    })
+
 def _redpanda_cc_test(
         name,
         timeout,
@@ -171,9 +186,9 @@ def _redpanda_cc_test(
             "layering_check",
         ],
         tags = resource_tags + tags,
-        env = {"RP_FIXTURE_ENV": "1"} | test_env | env,
+        env = _symbolizer_env({"RP_FIXTURE_ENV": "1"} | test_env | env),
         target_compatible_with = target_compatible_with,
-        data = data + test_data,
+        data = data + test_data + _SYMBOLIZER_DATA,
         local_defines = local_defines,
         flaky = flaky,
     )
@@ -215,8 +230,8 @@ def _redpanda_cc_fuzz_test(
         tags = [
             "fuzz",
         ],
-        env = test_env | env,
-        data = data + test_data,
+        env = _symbolizer_env(test_env | env),
+        data = data + test_data + _SYMBOLIZER_DATA,
         linkopts = [
             "-fsanitize=fuzzer",
         ],
@@ -355,8 +370,8 @@ def redpanda_cc_btest_no_seastar(
             "//src/v/test_utils:boost_test_hooks",
             "@boost//:test.so",
         ] + deps + test_deps,
-        data = test_data,
-        env = test_env,
+        data = test_data + _SYMBOLIZER_DATA,
+        env = _symbolizer_env(test_env),
     )
 
 def redpanda_test_cc_library(
@@ -518,7 +533,7 @@ def redpanda_cc_bench(
         main = "bench_wrapper.py",
         tags = resource_tags + tags,
         srcs = ["//bazel:bench_wrapper"],
-        env = test_env | env,
+        env = _symbolizer_env(test_env | env),
         args = test_args,
-        data = [":" + binary_name] + data + test_data,
+        data = [":" + binary_name] + data + test_data + _SYMBOLIZER_DATA,
     )

bazel/thirdparty/crc32c-big-endian-config.patch

@@ -0,0 +1,19 @@
+Set BYTE_ORDER_BIG_ENDIAN from the compiler's byte-order macros instead of
+hardcoding it to 0. The crc32c portable implementation reads input words via
+ReadUint32LE, whose big-endian branch is only compiled when
+BYTE_ORDER_BIG_ENDIAN is non-zero. Hardcoding 0 makes the LE memcpy path run on
+big-endian hosts (e.g. s390x), producing incorrect CRC32C values and breaking
+Kafka record-batch validation. Deriving the value from __BYTE_ORDER__ keeps the
+config correct on every target arch.
+
+--- a/BUILD.bazel
++++ b/BUILD.bazel
+@@ -76,7 +76,7 @@
+     outs = ["crc32c/crc32c_config.h"],
+     cmd = """
+ sed -e 's/#cmakedefine01/#define/' \
+-    -e 's/ BYTE_ORDER_BIG_ENDIAN/ BYTE_ORDER_BIG_ENDIAN 0/' \
++    -e 's/ BYTE_ORDER_BIG_ENDIAN/ BYTE_ORDER_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)/' \
+     -e 's/ HAVE_BUILTIN_PREFETCH/ HAVE_BUILTIN_PREFETCH 0/' \
+     -e 's/ HAVE_MM_PREFETCH/ HAVE_MM_PREFETCH 0/' \
+     -e 's/ HAVE_SSE42/ HAVE_SSE42 1/' \

bazel/thirdparty/hwloc.BUILD

@@ -23,6 +23,13 @@ configure_make(
     configure_options = [
         "--disable-libudev",
 
+        # Don't let autoconf pick up a system libxml2 for optional XML
+        # topology import/export. We don't use it, and an auto-detected
+        # libxml2 leaks undefined xml* symbols into every hwloc consumer
+        # (the dep isn't wired up), breaking links on hosts that happen to
+        # have libxml2 headers installed.
+        "--disable-libxml2",
+
         # Disable graphics and the many kinds of display driver discovery
         "--disable-gl",
         "--disable-opencl",

bazel/thirdparty/rules-boost-s390x-context.patch

@@ -0,0 +1,29 @@
+--- a/boost.BUILD
++++ b/boost.BUILD
+@@ -24,6 +24,14 @@
+ )
+
+ config_setting(
++    name = "linux_s390x",
++    constraint_values = [
++        "@platforms//os:linux",
++        "@platforms//cpu:s390x",
++    ],
++)
++
++config_setting(
+     name = "linux_aarch64",
+     constraint_values = [
+         "@platforms//os:linux",
+@@ -106,6 +114,11 @@
+         "libs/context/src/asm/make_ppc64_sysv_elf_gas.S",
+         "libs/context/src/asm/ontop_ppc64_sysv_elf_gas.S",
+     ],
++    ":linux_s390x": [
++        "libs/context/src/asm/jump_s390x_sysv_elf_gas.S",
++        "libs/context/src/asm/make_s390x_sysv_elf_gas.S",
++        "libs/context/src/asm/ontop_s390x_sysv_elf_gas.S",
++    ],
+     ":linux_x86_64": [
+         "libs/context/src/asm/jump_x86_64_sysv_elf_gas.S",
+         "libs/context/src/asm/make_x86_64_sysv_elf_gas.S",

src/v/bytes/ioarray.cc

@@ -269,7 +269,9 @@ uint32_t ioarray::read_fixed32(size_t i) const {
     v |= read(++i) << 8u;
     v |= read(++i) << 16u;
     v |= read(++i) << 24u;
-    return ss::le_to_cpu(v);
+    // The shifts above already assemble the bytes into host byte order
+    // (least-significant byte first), so no le_to_cpu is needed here.
+    return v;
 }
 
 void ioarray::trim_back(size_t n) {

src/v/cloud_topics/level_one/common/object.cc

@@ -296,8 +296,9 @@ ss::future<std::variant<footer, size_t>> footer::read(iobuf buf) {
               buf.size_bytes())));
     }
     iobuf_const_parser parser(buf);
-    auto footer_size
-      = iobuf_parser(buf.tail(sizeof(uint32_t))).consume_type<uint32_t>();
+    // footer_size is written little-endian via as_bytes/cpu_to_le.
+    auto footer_size = ss::le_to_cpu(
+      iobuf_parser(buf.tail(sizeof(uint32_t))).consume_type<uint32_t>());
     if (buf.size_bytes() >= (footer_size + sizeof(uint32_t))) {
         iobuf_parser p(buf.share(
           buf.size_bytes() - sizeof(uint32_t) - footer_size, footer_size));
@@ -310,7 +311,8 @@ ss::future<std::variant<footer, size_t>> footer::read(iobuf buf) {
                   "expected footer data type, got: {}",
                   std::to_underlying(dt))));
         }
-        auto size = p.consume_type<uint32_t>();
+        // size is written little-endian via as_bytes/cpu_to_le.
+        auto size = ss::le_to_cpu(p.consume_type<uint32_t>());
         if (size != p.bytes_left()) {
             co_await ss::coroutine::return_exception(
               std::runtime_error(

src/v/hashing/murmur.cc

@@ -9,10 +9,22 @@
 
 #include "hashing/murmur.h"
 
+#include <bit>
+
 // adapted from original:
 // https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.h
 
 namespace {
+
+// MurmurHash3 reads each block as a little-endian word. Convert from
+// little-endian to host order so the hash matches on big-endian platforms.
+template<typename T>
+inline __attribute__((always_inline)) T le_block_to_host(T k) {
+    if constexpr (std::endian::native == std::endian::big) {
+        return std::byteswap(k);
+    }
+    return k;
+}
 inline uint32_t rotl32(uint32_t x, int8_t r) {
     return (x << r) | (x >> (32 - r));
 }
@@ -46,13 +58,13 @@ inline __attribute__((always_inline)) uint32_t
 getblock32(const uint32_t* p, int i) {
     uint32_t k;
     std::memcpy(&k, reinterpret_cast<const char*>(p + i), sizeof(k));
-    return k;
+    return le_block_to_host(k);
 }
 inline __attribute__((always_inline)) uint64_t
 getblock64(const uint64_t* p, int i) {
     uint64_t k;
     std::memcpy(&k, reinterpret_cast<const char*>(p + i), sizeof(k));
-    return k;
+    return le_block_to_host(k);
 }
 namespace x86_32 {
 const uint32_t c1 = 0xcc9e2d51;
@@ -138,7 +150,9 @@ uint32_t murmurhash3_x86_32(const iobuf& data, uint32_t seed) {
           frag_begin,
           frag_begin + torn_remaining_capacity,
           torn_block_data_end);
-        x86_32::consume_block(h1, torn_block);
+        // torn_block is assembled byte-wise; read it as a little-endian word to
+        // match getblock32.
+        x86_32::consume_block(h1, le_block_to_host(torn_block));
 
         // rest of full blocks in fragment
         auto blocks_begin = frag_begin + torn_remaining_capacity;

src/v/iceberg/bucket_transform_hashing_visitor.cc

@@ -44,22 +44,24 @@ bucket_transform_hashing_visitor::operator()(const timestamptz_value& value) {
 }
 uint32_t
 bucket_transform_hashing_visitor::operator()(const decimal_value& value) {
-    const auto low_val = ss::cpu_to_be(Int128Low64(value.val));
-    const auto high_val = ss::cpu_to_be(Int128High64(value.val));
+    const auto low_val = Int128Low64(value.val);
+    const auto high_val = Int128High64(value.val);
 
     std::array<uint8_t, 16> value_bytes{0};
 
     /**
      * Both Java and PyIceberg implementations encode the decimal in big endian
      * format and then they hash an array with the smallest size required to
-     * represent the decimal
+     * represent the decimal.
+     *
+     * Extract the bytes most-significant first via arithmetic shifts, which is
+     * independent of host byte order (don't byteswap into a native value and
+     * then shift, which only yields big-endian bytes on little-endian hosts).
      */
     for (uint8_t i = 0; i < 8; i++) {
-        const auto h_byte = (high_val >> (i * 8)) & 0xFF;
-        const auto l_byte = (low_val >> (i * 8)) & 0xFF;
-
-        value_bytes[i] = h_byte;
-        value_bytes[i + 8] = l_byte;
+        const auto shift = (7 - i) * 8;
+        value_bytes[i] = (high_val >> shift) & 0xFF;
+        value_bytes[i + 8] = (low_val >> shift) & 0xFF;
     }
     /**
      * Limit the size of the array to the smallest size required to represent

src/v/kafka/protocol/tests/batch_reader_test.cc

@@ -123,8 +123,10 @@ SEASTAR_THREAD_TEST_CASE(consumer_records_consume_batch) {
 
 SEASTAR_THREAD_TEST_CASE(consumer_records_consume_batch_fail_magic) {
     auto ctx = make_context(base_offset, few_batches);
-    corrupt_offset<int32_t>(
-      ctx.record_set, mag_offset, [](int32_t& t) { --t; });
+    // The magic is a single byte; corrupt it as int8 so exactly that byte
+    // changes. Corrupting a 4-byte int here only alters the magic byte on
+    // little-endian hosts (where it is the least-significant byte).
+    corrupt_offset<int8_t>(ctx.record_set, mag_offset, [](int8_t& t) { --t; });
 
     auto crs = kafka::batch_reader(std::move(ctx.record_set));
 
@@ -209,8 +211,10 @@ SEASTAR_THREAD_TEST_CASE(batch_reader_record_batch_reader_impl_fail_lod) {
 
 SEASTAR_THREAD_TEST_CASE(batch_reader_record_batch_reader_impl_fail_magic) {
     auto ctx = make_context(base_offset, few_batches);
-    corrupt_offset<int32_t>(
-      ctx.record_set, mag_offset, [](int32_t& t) { --t; });
+    // The magic is a single byte; corrupt it as int8 so exactly that byte
+    // changes. Corrupting a 4-byte int here only alters the magic byte on
+    // little-endian hosts (where it is the least-significant byte).
+    corrupt_offset<int8_t>(ctx.record_set, mag_offset, [](int8_t& t) { --t; });
 
     auto rdr = model::make_record_batch_reader<kafka::batch_reader>(
       std::move(ctx.record_set));

src/v/lsm/block/filter.cc

@@ -126,12 +126,14 @@ iobuf filter_builder::finish() {
     if (!_keys.empty()) {
         generate_filter();
     }
-    // Append array of per-filter offsets
+    // Append array of per-filter offsets. These fixed32s are read back through
+    // block::contents::read_fixed32, which round-trips host byte order, so they
+    // must be written in native byte order (matching the block builder) rather
+    // than little-endian.
     uint32_t offsets_start = _filter.size_bytes();
     for (const auto& offset : std::exchange(_filter_offsets, {})) {
         _filter.append(
-          std::bit_cast<std::array<uint8_t, sizeof(offset)>>(
-            ss::cpu_to_le(offset)));
+          std::bit_cast<std::array<uint8_t, sizeof(offset)>>(offset));
     }
     _filter.append(
       std::bit_cast<std::array<uint8_t, sizeof(uint32_t)>>(offsets_start));

src/v/lsm/sst/builder.cc

@@ -11,6 +11,7 @@
 #include "hashing/crc32c.h"
 #include "lsm/sst/footer.h"
 
+#include <seastar/core/byteorder.hh>
 #include <seastar/core/coroutine.hh>
 
 namespace lsm::sst {
@@ -81,8 +82,11 @@ builder::write_raw_block(iobuf buf, compression_type comp_type) {
     buf.append(std::to_array({std::to_underlying(comp_type)}));
     crc::crc32c crc;
     crc_extend_iobuf(crc, buf);
+    // The reader decodes this CRC with ioarray::read_fixed32 (little-endian),
+    // so store it little-endian to round-trip on big-endian hosts too.
     buf.append(
-      std::bit_cast<std::array<uint8_t, sizeof(crc.value())>>(crc.value()));
+      std::bit_cast<std::array<uint8_t, sizeof(crc.value())>>(
+        ss::cpu_to_le(crc.value())));
     _written_bytes += buf.size_bytes();
     co_await _writer->append(std::move(buf));
     co_return h;