feat(oss): Support thrift-py3/thrift-rust and improve python support for c++ inclusion

.github/workflows/edenfs_linux.yml

@@ -26,8 +26,10 @@ jobs:
     - id: paths
       name: Query paths
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --no-tests --recursive --src-dir=. eden  >> "$GITHUB_OUTPUT"
-    - name: Install Rust Stable
-      uses: dtolnay/rust-toolchain@stable
+    - name: Install Rust Nightly
+      uses: dtolnay/rust-toolchain@nightly
+    - name: Install rustfmt
+      run: rustup component add rustfmt
     - name: Fetch lmdb
       if: ${{ steps.paths.outputs.lmdb_SOURCE }}
       run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests lmdb

build/fbcode_builder/CMake/FBPythonBinary.cmake

@@ -105,7 +105,7 @@ function(add_fb_python_executable TARGET)
 
   # Parse the arguments
   set(one_value_args BASE_DIR NAMESPACE MAIN_MODULE TYPE)
-  set(multi_value_args SOURCES DEPENDS)
+  set(multi_value_args SOURCES DEPENDS NORMAL_DEPENDS)
   fb_cmake_parse_args(
     ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
   )
@@ -118,11 +118,13 @@ function(add_fb_python_executable TARGET)
     NAMESPACE "${ARG_NAMESPACE}"
     SOURCES ${ARG_SOURCES}
     DEPENDS ${ARG_DEPENDS}
+    NORMAL_DEPENDS ${ARG_NORMAL_DEPENDS}
   )
 
+  # Tightly coupled to target_include_directories() below in add_fb_python_library()
   set(
     manifest_files
-    "$<TARGET_PROPERTY:${TARGET}.main_lib.py_lib,INTERFACE_INCLUDE_DIRECTORIES>"
+    "$<FILTER:$<TARGET_PROPERTY:${TARGET}.main_lib.py_lib,INTERFACE_INCLUDE_DIRECTORIES>,EXCLUDE,/.*/include>"
   )
   set(
     source_files
@@ -232,7 +234,7 @@ endfunction()
 # passing additional configuration to the test.
 function(add_fb_python_unittest TARGET)
   # Parse the arguments
-  set(multi_value_args SOURCES DEPENDS ENV PROPERTIES)
+  set(multi_value_args SOURCES DEPENDS ENV NORMAL_DEPENDS PROPERTIES)
   set(
     one_value_args
     WORKING_DIRECTORY BASE_DIR NAMESPACE TEST_LIST DISCOVERY_TIMEOUT
@@ -298,8 +300,33 @@ function(add_fb_python_unittest TARGET)
     BASE_DIR "${ARG_BASE_DIR}"
     SOURCES ${ARG_SOURCES}
     DEPENDS ${ARG_DEPENDS}
+    NORMAL_DEPENDS ${ARG_NORMAL_DEPENDS}
   )
 
+  set(_python_dirs "")
+  set(_runtime_dirs "")
+
+  foreach( _dep IN LISTS ARG_DEPENDS)
+    list(APPEND _python_dirs "$<TARGET_PROPERTY:${_dep}.py_lib,INTERFACE_INCLUDE_DIRECTORIES>")
+    list(APPEND _runtime_dirs "$<INSTALL_INTERFACE:${_dep}.py_lib>")
+  endforeach()
+
+  foreach( _dep IN LISTS ARG_NORMAL_DEPENDS)
+    list(APPEND _runtime_dirs "$<TARGET_FILE_DIR:${_dep}>")
+  endforeach()
+
+  foreach(_dep IN LISTS "$ENV{PYTHONPATH}")
+    list(APPEND _python_dirs "${_dep}")
+  endforeach()
+
+  set(_runtime_env_var "$<IF:$<PLATFORM_ID:Darwin>,DYLD_LIBRARY_PATH,LD_LIBRARY_PATH>")
+  foreach(_dep IN LISTS "$ENV{_runtime_env_var}")
+    list(APPEND _runtime_dirs "${_dep}")
+  endforeach()
+
+  cmake_path(CONVERT "${_python_dirs}" TO_NATIVE_PATH_LIST _python_path)
+  cmake_path(CONVERT "${_runtime_dirs}" TO_NATIVE_PATH_LIST _runtime_path)
+
   # Run test discovery after the test executable is built.
   # This logic is based on the code for gtest_discover_tests()
   set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}")
@@ -309,6 +336,9 @@ function(add_fb_python_unittest TARGET)
     TARGET "${TARGET}.GEN_PY_EXE" POST_BUILD
     BYPRODUCTS "${ctest_tests_file}"
     COMMAND
+      "${CMAKE_COMMAND}" -E env
+      "${_runtime_env_var}=${_runtime_path}"
+      "PYTHONPATH=${_python_path}"
       "${CMAKE_COMMAND}"
       -D "TEST_TARGET=${TARGET}"
       -D "TEST_INTERPRETER=${Python3_EXECUTABLE}"
@@ -401,7 +431,7 @@ function(add_fb_python_library LIB_NAME)
   # cmake_parse_arguments() does not handle empty arguments, and it is common
   # for callers to want to specify an empty NAMESPACE parameter.
   set(one_value_args BASE_DIR NAMESPACE INSTALL_DIR)
-  set(multi_value_args SOURCES DEPENDS)
+  set(multi_value_args SOURCES DEPENDS NORMAL_DEPENDS)
   fb_cmake_parse_args(
     ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
   )
@@ -468,6 +498,7 @@ function(add_fb_python_library LIB_NAME)
   endforeach()
   configure_file("${tmp_manifest}" "${manifest_path}" COPYONLY)
 
+  # Tightly coupled to manifest_files in add_fbpython_executable()
   target_include_directories(
     "${LIB_NAME}.py_lib" INTERFACE
     "$<BUILD_INTERFACE:${manifest_path}>"
@@ -500,6 +531,11 @@ function(add_fb_python_library LIB_NAME)
     add_dependencies("${LIB_NAME}.py_sources_built" "${dep}.py_lib")
   endforeach()
 
+  foreach(dep IN LISTS ARG_NORMAL_DEPENDS)
+    target_link_libraries("${LIB_NAME}.py_lib" INTERFACE "${dep}")
+    add_dependencies("${LIB_NAME}.py_sources_built" "${dep}")
+  endforeach()
+
   # Add a custom command to help with library installation, in case
   # install_fb_python_library() is called later for this library.
   # add_custom_command() only works with file dependencies defined in the same

build/fbcode_builder/CMake/FBThriftLibrary.cmake

@@ -1,8 +1,10 @@
 # Copyright (c) Facebook, Inc. and its affiliates.
 
 include(FBCMakeParseArgs)
-include(FBThriftPyLibrary)
 include(FBThriftCppLibrary)
+include(FBThriftPyLibrary)
+include(FBThriftPy3Library)
+include(FBThriftRustLibrary)
 
 #
 # add_fbthrift_library()
@@ -24,8 +26,8 @@ include(FBThriftCppLibrary)
 #
 function(add_fbthrift_library LIB_NAME THRIFT_FILE)
   # Parse the arguments
-  set(one_value_args PY_NAMESPACE INCLUDE_DIR THRIFT_INCLUDE_DIR)
-  set(multi_value_args SERVICES DEPENDS LANGUAGES CPP_OPTIONS PY_OPTIONS)
+  set(one_value_args PY_NAMESPACE PY3_NAMESPACE INCLUDE_DIR THRIFT_INCLUDE_DIR)
+  set(multi_value_args SERVICES DEPENDS LANGUAGES CPP_OPTIONS PY_OPTIONS RUST_OPTIONS)
   fb_cmake_parse_args(
     ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
   )
@@ -41,9 +43,11 @@ function(add_fbthrift_library LIB_NAME THRIFT_FILE)
   # now we still want to support older versions of CMake.
   set(CPP_DEPENDS)
   set(PY_DEPENDS)
+  set(RUST_DEPENDS)
   foreach(dep IN LISTS ARG_DEPENDS)
     list(APPEND CPP_DEPENDS "${dep}_cpp")
     list(APPEND PY_DEPENDS "${dep}_py")
+    list(APPEND RUST_DEPENDS "${dep}_rs")
   endforeach()
 
   foreach(lang IN LISTS ARG_LANGUAGES)
@@ -68,6 +72,26 @@ function(add_fbthrift_library LIB_NAME THRIFT_FILE)
         OPTIONS ${ARG_PY_OPTIONS}
         THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}"
       )
+    elseif ("${lang}" STREQUAL "py3")
+      if (DEFINED ARG_PY3_NAMESPACE)
+        set(namespace_args NAMESPACE "${ARG_PY3_NAMESPACE}")
+      endif()
+      add_fbthrift_py3_library(
+        "${LIB_NAME}_py3" "${THRIFT_FILE}"
+        SERVICES ${ARG_SERVICES}
+        ${namespace_args}
+        DEPENDS ${PY_DEPENDS}
+        OPTIONS ${ARG_PY_OPTIONS}
+        THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}"
+      )
+    elseif ("${lang}" STREQUAL "rust")
+      add_fbthrift_rust_library(
+        "${LIB_NAME}_rs" "${THRIFT_FILE}"
+        SERVICES ${ARG_SERVICES}
+        DEPENDS ${RUST_DEPENDS}
+        OPTIONS ${ARG_RUST_OPTIONS}
+        THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}"
+      )
     else()
       message(
         FATAL_ERROR "unknown language for thrift library ${LIB_NAME}: ${lang}"

build/fbcode_builder/CMake/FBThriftPy3Library.cmake

@@ -0,0 +1,124 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+
+include(FBCMakeParseArgs)
+include(FBPythonBinary)
+
+find_package(FBThrift REQUIRED)
+
+# Generate a Python library from a thrift file
+function(add_fbthrift_py3_library LIB_NAME THRIFT_FILE)
+  # Parse the arguments
+  set(one_value_args NAMESPACE THRIFT_INCLUDE_DIR)
+  set(multi_value_args SERVICES DEPENDS OPTIONS)
+  fb_cmake_parse_args(
+    ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
+  )
+
+  if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR)
+    set(ARG_THRIFT_INCLUDE_DIR "include/thrift-files")
+  endif()
+
+  get_filename_component(base ${THRIFT_FILE} NAME_WE)
+  set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE}-py3")
+
+  # Parse the namespace value
+  if (NOT DEFINED ARG_NAMESPACE)
+    set(ARG_NAMESPACE "${base}")
+  endif()
+
+  string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}")
+  set(py_output_dir "${output_dir}/gen-python/${namespace_dir}")
+  list(APPEND generated_sources
+    "${py_output_dir}/thrift_abstract_types.py"
+    "${py_output_dir}/thrift_enums.py"
+    "${py_output_dir}/thrift_metadata.py"
+    # thrift_mutable_types.py
+    # thrift_mutable_types.pyi
+    "${py_output_dir}/thrift_types.py"
+    "${py_output_dir}/thrift_types.pyi"
+  )
+  if(ARG_SERVICES)
+    list(APPEND generated_sources
+      "${py_output_dir}/thrift_clients.py"
+      "${py_output_dir}/thrift_mutable_clients.py"
+      # thrift_services.py
+      # thrift_mutable_services.py
+    )
+  endif()
+
+  # Define a dummy interface library to help propagate the thrift include
+  # directories between dependencies.
+  add_library("${LIB_NAME}.thrift_includes" INTERFACE)
+  target_include_directories(
+    "${LIB_NAME}.thrift_includes"
+    INTERFACE
+      "$<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}>"
+      "$<INSTALL_INTERFACE:${ARG_THRIFT_INCLUDE_DIR}>"
+  )
+  foreach(dep IN LISTS ARG_DEPENDS)
+    target_link_libraries(
+      "${LIB_NAME}.thrift_includes"
+      INTERFACE "${dep}.thrift_includes"
+    )
+  endforeach()
+
+  # This generator expression gets the list of include directories required
+  # for all of our dependencies.
+  # It requires using COMMAND_EXPAND_LISTS in the add_custom_command() call
+  # below.  COMMAND_EXPAND_LISTS is only available in CMake 3.8+
+  # If we really had to support older versions of CMake we would probably need
+  # to use a wrapper script around the thrift compiler that could take the
+  # include list as a single argument and split it up before invoking the
+  # thrift compiler.
+  if (NOT POLICY CMP0067)
+    message(FATAL_ERROR "add_fbthrift_py3_library() requires CMake 3.8+")
+  endif()
+  set(
+    thrift_include_options
+    "-I;$<JOIN:$<TARGET_PROPERTY:${LIB_NAME}.thrift_includes,INTERFACE_INCLUDE_DIRECTORIES>,;-I;>"
+  )
+
+  # Always force generation of "new-style" python classes for Python 2
+  # list(APPEND ARG_OPTIONS "new_style")
+  # CMake 3.12 is finally getting a list(JOIN) function, but until then
+  # treating the list as a string and replacing the semicolons is good enough.
+  string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}")
+
+  # Emit the rule to run the thrift compiler
+  add_custom_command(
+    OUTPUT
+      ${generated_sources}
+    COMMAND_EXPAND_LISTS
+    COMMAND
+      "${CMAKE_COMMAND}" -E make_directory "${output_dir}"
+    COMMAND
+      "${FBTHRIFT_COMPILER}"
+      --legacy-strict
+      --recurse
+      --gen "mstch_python:${GEN_ARG_STR}"
+      "${thrift_include_options}"
+      -o "${output_dir}"
+      "${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}"
+    WORKING_DIRECTORY
+      "${CMAKE_BINARY_DIR}"
+    MAIN_DEPENDENCY
+      "${THRIFT_FILE}"
+    DEPENDS
+      "${FBTHRIFT_COMPILER}"
+  )
+
+  # We always want to pass the namespace as "" to this call:
+  # thrift will already emit the files with the desired namespace prefix under
+  # gen-py3.  We don't want add_fb_python_library() to prepend the namespace a
+  # second time.
+  add_fb_python_library(
+    "${LIB_NAME}"
+    BASE_DIR "${output_dir}/gen-python"
+    NAMESPACE ""
+    SOURCES ${generated_sources}
+    DEPENDS ${ARG_DEPENDS} Folly::folly_python_cpp
+    NORMAL_DEPENDS
+      Folly::folly_python_cpp
+      FBThrift::thrift_python_cpp
+  )
+endfunction()