Library Forwarding: Add support for 32-bit Vulkan

ThunkLibs/Generator/analysis.cpp

@@ -301,14 +301,8 @@ void AnalysisAction::ParseInterface(clang::ASTContext& context) {
                 if (auto emitted_function = llvm::dyn_cast<clang::FunctionDecl>(template_args[0].getAsDecl())) {
                     // Process later
                 } else if (auto annotated_member = llvm::dyn_cast<clang::FieldDecl>(template_args[0].getAsDecl())) {
-                    {
-                        if (decl->getNumBases() != 1 || decl->bases_begin()->getType().getAsString() != "fexgen::custom_repack") {
-                            throw report_error(template_arg_loc, "Unsupported member annotation(s)");
-                        }
-
-                        if (!annotated_member->getType()->isPointerType() && !annotated_member->getType()->isArrayType()) {
-                            throw report_error(template_arg_loc, "custom_repack annotation requires pointer member");
-                        }
+                    if (decl->getNumBases() != 1 || decl->bases_begin()->getType().getAsString() != "fexgen::custom_repack") {
+                        throw report_error(template_arg_loc, "Unsupported member annotation(s)");
                     }
 
                     // Get or add parent type to list of structure types

ThunkLibs/Generator/data_layout.cpp

@@ -305,15 +305,15 @@ TypeCompatibility DataLayoutCompareAction::GetTypeCompatibility(
                 host_member_type = context.getCanonicalType(array_type->getElementType().getTypePtr());
             }
 
-            if (host_member_type->isPointerType()) {
+            if (types.at(type).UsesCustomRepackFor(host_member_field)) {
+                member_compat.push_back(TypeCompatibility::Repackable);
+                continue;
+            } else if (host_member_type->isPointerType()) {
                 // Automatic repacking of pointers to non-compatible types is only possible if:
                 // * Pointee is fully compatible, or
                 // * Pointer member is annotated
-                // TODO: Don't restrict this to structure types. it applies to pointers to builtin types too!
                 auto host_member_pointee_type = context.getCanonicalType(host_member_type->getPointeeType().getTypePtr());
-                if (types.at(type).UsesCustomRepackFor(host_member_field)) {
-                    member_compat.push_back(TypeCompatibility::Repackable);
-                } else if (types.contains(host_member_pointee_type) && types.at(host_member_pointee_type).assumed_compatible) {
+                if (types.contains(host_member_pointee_type) && types.at(host_member_pointee_type).assumed_compatible) {
                     // Pointee doesn't need repacking, but pointer needs extending on 32-bit
                     member_compat.push_back(is_32bit ? TypeCompatibility::Repackable : TypeCompatibility::Full);
                 } else if (host_member_pointee_type->isPointerType()) {

ThunkLibs/Generator/gen.cpp

@@ -308,7 +308,7 @@ void GenerateThunkLibsAction::EmitLayoutWrappers(
         if (type_repack_info.custom_repacked_members.empty()) {
             fmt::print(file, "void fex_apply_custom_repacking_entry(host_layout<{}>& source, const guest_layout<{}>& from) {{\n", struct_name, struct_name);
             fmt::print(file, "}}\n");
-            fmt::print(file, "bool fex_apply_custom_repacking_exit(guest_layout<{}>& into, host_layout<{}>& from) {{\n", struct_name, struct_name);
+            fmt::print(file, "bool fex_apply_custom_repacking_exit(guest_layout<{}>& into, const host_layout<{}>& from) {{\n", struct_name, struct_name);
             fmt::print(file, "  return false;\n");
             fmt::print(file, "}}\n");
         } else {
@@ -321,7 +321,7 @@ void GenerateThunkLibsAction::EmitLayoutWrappers(
             fmt::print(file, "  fex_custom_repack_entry(source, from);\n");
             fmt::print(file, "}}\n");
 
-            fmt::print(file, "bool fex_apply_custom_repacking_exit(guest_layout<{}>& into, host_layout<{}>& from) {{\n", struct_name, struct_name);
+            fmt::print(file, "bool fex_apply_custom_repacking_exit(guest_layout<{}>& into, const host_layout<{}>& from) {{\n", struct_name, struct_name);
             fmt::print(file, "  return fex_custom_repack_exit(into, from);\n");
             fmt::print(file, "}}\n");
         }
@@ -339,7 +339,12 @@ void GenerateThunkLibsAction::OnAnalysisComplete(clang::ASTContext& context) {
         std::unordered_map<const clang::Type*, TypeCompatibility> ret;
         const auto host_abi = ComputeDataLayout(context, types);
         for (const auto& [type, type_repack_info] : types) {
-            if (!type_repack_info.pointers_only) {
+            if (type_repack_info.emit_layout_wrappers) {
+                // Assume incompatible, since this annotation is set when
+                // compatibility checks would otherwise fail (e.g. due to
+                // circular references)
+                ret.emplace(type, TypeCompatibility::None);
+            } else if (!type_repack_info.pointers_only) {
                 GetTypeCompatibility(context, type, host_abi, ret);
             }
         }

ThunkLibs/GuestLibs/CMakeLists.txt

@@ -217,10 +217,6 @@ if (BITNESS EQUAL 64)
   generate(libXfixes ${CMAKE_CURRENT_SOURCE_DIR}/../libXfixes/libXfixes_interface.cpp)
   add_guest_lib(Xfixes "libXfixes.so.3")
 
-  generate(libvulkan ${CMAKE_CURRENT_SOURCE_DIR}/../libvulkan/libvulkan_interface.cpp)
-  target_include_directories(libvulkan-guest-deps INTERFACE ${FEX_PROJECT_SOURCE_DIR}/External/Vulkan-Headers/include/)
-  add_guest_lib(vulkan "libvulkan.so.1")
-
   find_package(PkgConfig)
   pkg_search_module(XCB REQUIRED xcb)
   version_to_variables(${XCB_VERSION} XCB)
@@ -265,6 +261,10 @@ if (BITNESS EQUAL 64)
   add_guest_lib(drm "libdrm.so.2")
 endif()
 
+generate(libvulkan ${CMAKE_CURRENT_SOURCE_DIR}/../libvulkan/libvulkan_interface.cpp)
+target_include_directories(libvulkan-guest-deps INTERFACE ${FEX_PROJECT_SOURCE_DIR}/External/Vulkan-Headers/include/)
+add_guest_lib(vulkan "libvulkan.so.1")
+
 generate(libwayland-client ${CMAKE_CURRENT_SOURCE_DIR}/../libwayland-client/libwayland-client_interface.cpp)
 add_guest_lib(wayland-client "libwayland-client.so.0.20.0")
 

ThunkLibs/HostLibs/CMakeLists.txt

@@ -148,10 +148,6 @@ foreach(GUEST_BITNESS IN LISTS BITNESS_LIST)
   generate(libXfixes ${CMAKE_CURRENT_SOURCE_DIR}/../libXfixes/libXfixes_interface.cpp ${GUEST_BITNESS})
   add_host_lib(Xfixes ${GUEST_BITNESS})
 
-  generate(libvulkan ${CMAKE_CURRENT_SOURCE_DIR}/../libvulkan/libvulkan_interface.cpp ${GUEST_BITNESS})
-  target_include_directories(libvulkan-${GUEST_BITNESS}-deps INTERFACE ${FEX_PROJECT_SOURCE_DIR}/External/Vulkan-Headers/include/)
-  add_host_lib(vulkan ${GUEST_BITNESS})
-
   find_package(PkgConfig)
   pkg_search_module(XCB REQUIRED xcb)
   version_to_variables(${XCB_VERSION} XCB)
@@ -201,6 +197,10 @@ foreach(GUEST_BITNESS IN LISTS BITNESS_LIST)
   generate(libfex_thunk_test ${CMAKE_CURRENT_SOURCE_DIR}/../libfex_thunk_test/libfex_thunk_test_interface.cpp ${GUEST_BITNESS})
   add_host_lib(fex_thunk_test ${GUEST_BITNESS})
 
+  generate(libvulkan ${CMAKE_CURRENT_SOURCE_DIR}/../libvulkan/libvulkan_interface.cpp ${GUEST_BITNESS})
+  target_include_directories(libvulkan-${GUEST_BITNESS}-deps INTERFACE ${FEX_PROJECT_SOURCE_DIR}/External/Vulkan-Headers/include/)
+  add_host_lib(vulkan ${GUEST_BITNESS})
+
   generate(libwayland-client ${CMAKE_CURRENT_SOURCE_DIR}/../libwayland-client/libwayland-client_interface.cpp ${GUEST_BITNESS})
   add_host_lib(wayland-client ${GUEST_BITNESS})
 endforeach()

ThunkLibs/include/common/Host.h

@@ -144,6 +144,12 @@ struct __attribute__((packed)) guest_layout {
   }
 };
 
+template<typename T, std::size_t N>
+struct __attribute__((packed)) guest_layout<T[N]> {
+  using type = std::enable_if_t<!std::is_pointer_v<T>, T>;
+  std::array<guest_layout<type>, N> data;
+};
+
 template<typename T>
 struct guest_layout<T*> {
 #ifdef IS_32BIT_THUNK
@@ -241,6 +247,17 @@ const host_layout<T>& to_host_layout(const T& t) {
   return reinterpret_cast<const host_layout<T>&>(t);
 }
 
+template<typename T, size_t N>
+struct host_layout<T[N]> {
+  std::array<T, N> data;
+
+  explicit host_layout(const guest_layout<T[N]>& from) {
+    for (size_t i = 0; i < N; ++i) {
+      data[i] = host_layout<T> { from.data[i] }.data;
+    }
+  }
+};
+
 template<typename T>
 constexpr bool is_long_or_longlong =
     std::is_same_v<T, long> ||

ThunkLibs/libfex_thunk_test/Host.cpp

@@ -4,9 +4,10 @@ tags: thunklibs|fex_thunk_test
 $end_info$
 */
 
-#include <cstddef>
 #include <dlfcn.h>
 
+#include <unordered_map>
+
 #include "common/Host.h"
 
 #include "api.h"
@@ -29,4 +30,84 @@ bool fex_custom_repack_exit(guest_layout<CustomRepackedType>& to, host_layout<Cu
   return false;
 }
 
+template<StructType TypeIndex, typename Type>
+static const TestBaseStruct* convert(const TestBaseStruct* source) {
+    // Using malloc here since no easily available type information is available at the time of destruction.
+    auto guest_next = reinterpret_cast<guest_layout<Type>*>((void*)source);
+    auto child_mem = (char*)aligned_alloc(alignof(host_layout<Type>), sizeof(host_layout<Type>));
+    auto child = new (child_mem) host_layout<Type> { *guest_next };
+
+    fex_custom_repack_entry(*child, *reinterpret_cast<guest_layout<Type>*>((void*)(source)));
+
+    return (const TestBaseStruct*)child;
+}
+
+template<StructType TypeIndex, typename Type>
+static void convert_to_guest(void* into, const TestBaseStruct* from) {
+    auto typed_into = (guest_layout<Type>*)into;
+    auto oldNext = typed_into->data.Next;
+    *typed_into = to_guest(to_host_layout(*(Type*)from));
+    typed_into->data.Next = oldNext;
+
+    fex_custom_repack_exit(*typed_into, to_host_layout(*(Type*)from));
+}
+
+template<StructType TypeIndex, typename Type>
+inline constexpr std::pair<StructType, std::pair<const TestBaseStruct*(*)(const TestBaseStruct*), void(*)(void*, const TestBaseStruct*)>> converters =
+  { TypeIndex, { convert<TypeIndex, Type>, convert_to_guest<TypeIndex, Type> } };
+
+static std::unordered_map<StructType, std::pair<const TestBaseStruct*(*)(const TestBaseStruct*), void(*)(void*, const TestBaseStruct*)>> next_handlers {
+    converters<StructType::Struct1, TestStruct1>,
+    converters<StructType::Struct2, TestStruct2>,
+};
+
+static void default_fex_custom_repack_entry(TestBaseStruct& into, const guest_layout<TestBaseStruct>* from) {
+  if (!from->data.Next.get_pointer()) {
+      into.Next = nullptr;
+      return;
+  }
+  auto typed_source = reinterpret_cast<const guest_layout<TestBaseStruct>*>(from->data.Next.get_pointer());
+
+  auto next_handler = next_handlers.at(StructType { typed_source->data.Type.data });
+
+  into.Next = (TestBaseStruct*)next_handler.first((const TestBaseStruct*)typed_source);
+}
+
+static void default_fex_custom_repack_reverse(guest_layout<TestBaseStruct>& into, const TestBaseStruct* from) {
+  auto NextHost = from->Next;
+  if (!NextHost) {
+    return;
+  }
+
+  auto next_handler = next_handlers.at(static_cast<StructType>(into.data.Next.get_pointer()->data.Type.data));
+  next_handler.second((void*)into.data.Next.get_pointer(), from->Next);
+
+  free((void*)NextHost);
+}
+
+#define CREATE_INFO_DEFAULT_CUSTOM_REPACK(name) \
+  void fex_custom_repack_entry(host_layout<name>& into, const guest_layout<name>& from) { \
+    default_fex_custom_repack_entry(*(TestBaseStruct*)&into.data, reinterpret_cast<const guest_layout<TestBaseStruct>*>(&from)); \
+  } \
+   \
+  bool fex_custom_repack_exit(guest_layout<name>& into, const host_layout<name>& from) { \
+    auto prev_next = into.data.Next; \
+    default_fex_custom_repack_reverse(*reinterpret_cast<guest_layout<TestBaseStruct>*>(&into), &reinterpret_cast<const TestBaseStruct&>(from.data)); \
+    into = to_guest(from); \
+    into.data.Next = prev_next; \
+    return true; \
+  }
+
+CREATE_INFO_DEFAULT_CUSTOM_REPACK(TestStruct1)
+CREATE_INFO_DEFAULT_CUSTOM_REPACK(TestStruct2)
+
+void fex_custom_repack_entry(host_layout<TestBaseStruct>&, const guest_layout<TestBaseStruct>&) {
+  std::abort();
+}
+
+bool fex_custom_repack_exit(guest_layout<TestBaseStruct>&, const host_layout<TestBaseStruct>&) {
+  std::abort();
+  return false;
+}
+
 EXPORTS(libfex_thunk_test)

ThunkLibs/libfex_thunk_test/api.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include <cstdint>
+#include <limits>
 
 extern "C" {
 
@@ -74,4 +75,36 @@ enum DivType : uint32_t {};
 #endif
 int FunctionWithDivergentSignature(DivType, DivType, DivType, DivType);
 
+
+/// Interfaces used to test Vulkan-like APIs
+
+// Equivalent of VkStructureType
+enum class StructType {
+    Struct1,
+    Struct2,
+};
+
+// Equivalent of VkBaseInStructure
+struct TestBaseStruct {
+    TestBaseStruct* Next;
+    StructType Type;
+};
+
+// Equivalent of e.g. VkImageCreateInfo
+struct TestStruct1 {
+    const void* Next;
+    StructType Type; // StructType::Struct1
+    uint8_t Data2;
+    uint8_t pad0[3];
+    int Data1;
+};
+
+struct TestStruct2 {
+    const void* Next;
+    StructType Type;  // StructType::Struct2
+    int Data1;
+};
+
+int ReadData1(TestStruct1*, int depth);
+
 }

ThunkLibs/libfex_thunk_test/lib.cpp

@@ -1,5 +1,8 @@
 #include "api.h"
 
+#include <cstdio>
+#include <cstddef>
+
 extern "C" {
 
 uint32_t GetDoubledValue(uint32_t input) {
@@ -59,4 +62,28 @@ int FunctionWithDivergentSignature(DivType a, DivType b, DivType c, DivType d) {
   return ((uint8_t)a << 24) | ((uint8_t)b << 16) | ((uint8_t)c << 8) | (uint8_t)d;
 }
 
+int ReadData1(TestStruct1* data, int depth) {
+  auto* base = (TestBaseStruct*)data;
+  for (int i = 0; i != depth; ++i) {
+    if (!base) {
+      return -1;
+    }
+    base = base->Next;
+  }
+  if (!base) {
+    return -1;
+  }
+
+  switch (base->Type) {
+  case StructType::Struct1:
+    return ((TestStruct1*)base)->Data1;
+
+  case StructType::Struct2:
+    return ((TestStruct2*)base)->Data1;
+
+  default:
+    return -2;
+  }
+}
+
 } // extern "C"

ThunkLibs/libfex_thunk_test/libfex_thunk_test_interface.cpp

@@ -35,3 +35,9 @@ template<> struct fex_gen_config<&CustomRepackedType::data> : fexgen::custom_rep
 template<> struct fex_gen_config<RanCustomRepack> {};
 
 template<> struct fex_gen_config<FunctionWithDivergentSignature> {};
+
+template<> struct fex_gen_config<&TestBaseStruct::Next> : fexgen::custom_repack {};
+template<> struct fex_gen_config<&TestStruct1::Next> : fexgen::custom_repack {};
+template<> struct fex_gen_config<&TestStruct2::Next> : fexgen::custom_repack {};
+
+template<> struct fex_gen_config<ReadData1> {};