Unable to wrap a function that's called with an enum

[smurfix]

I find myself in need of wrapping a heap of functions in a generic wrapper (think "take a lock before calling this API").

This works quite nicely, except when a parameter of the wrapped function is an enum.

Hints how to fix this (I'm not that good a C++ programmer) would be appreciated.

#include <iostream>

#include <pybind11/pybind11.h>
#include <pybind11/stl.h>

template <class R, class... Args>
struct wrap
{
    using funct_type = R(*)(Args...);
    funct_type func;
    wrap(funct_type f): func(f) {};
    R operator()(Args&&... args)
    {
          //before code block
          std::cout << "before calling\n";
          R ret=func(std::forward<Args>(args)...);
          //after code block
          std::cout << "After calling\n";
          return ret;
    }
};

class bar {};
class foo : bar {
    public:
    enum xfoo {
        FOO=1,
        BAR=2,
        BAZ=3,
    };
    static int add(int a, int b, enum xfoo c) {
        return a+b;
    }
};

namespace py = pybind11;

PYBIND11_MODULE(example, m) {
    m.doc() = "pybind11 example plugin"; // optional module docstring

    // this works
    std::cout << "Result" << wrap{foo::add}(1,2, foo::xfoo::FOO) << std::endl;

    // so does this
    m.def("add", foo::add, "A function that adds two numbers and no enum");

    // but this doesn't
    m.def("add", wrap{foo::add}, "A function that adds two numbers and no enum");
}

Using the command line

c++ -O3 -Wall -shared -std=c++17 -fPIC $(python3 -m pybind11 --includes) /tmp/example.cpp -o /tmp/example$(python3-config --extension-suffix)

the result is this error:

In file included from /usr/lib/python3/dist-packages/pybind11/include/pybind11/attr.h:14,
                 from /usr/lib/python3/dist-packages/pybind11/include/pybind11/detail/class.h:12,
                 from /usr/lib/python3/dist-packages/pybind11/include/pybind11/pybind11.h:12,
                 from /tmp/example.cpp:3:
/usr/lib/python3/dist-packages/pybind11/include/pybind11/cast.h: In instantiation of ‘Return pybind11::detail::argument_loader<Args>::call_impl(Func&&, std::index_sequence<Is ...>, Guard&&)&& [with Return = int; Func = wrap<int, int, int, foo::xfoo>&; long unsigned int ...Is = {0, 1, 2}; Guard = pybind11::detail::void_type; Args = {int&&, int&&, foo::xfoo&&}; std::index_sequence<Is ...> = std::integer_sequence<long unsigned int, 0, 1, 2>]’:
/usr/lib/python3/dist-packages/pybind11/include/pybind11/cast.h:1599:72:   required from ‘std::enable_if_t<((bool)(! std::is_void<_Dummy>::value)), Return> pybind11::detail::argument_loader<Args>::call(Func&&) && [with Return = int; Guard = pybind11::detail::void_type; Func = wrap<int, int, int, foo::xfoo>&; Args = {int&&, int&&, foo::xfoo&&}; std::enable_if_t<((bool)(! std::is_void<_Dummy>::value)), Return> = int]’
 1599 |         return std::move(*this).template call_impl<remove_cv_t<Return>>(
      |                ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
 1600 |             std::forward<Func>(f), indices{}, Guard{});
      |             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/usr/lib/python3/dist-packages/pybind11/include/pybind11/pybind11.h:275:78:   required from ‘void pybind11::cpp_function::initialize(Func&&, Return (*)(Args ...), const Extra& ...) [with Func = wrap<int, int, int, foo::xfoo>; Return = int; Args = {int&&, int&&, foo::xfoo&&}; Extra = {pybind11::name, pybind11::scope, pybind11::sibling, char [45]}]’
  275 |                 (void) std::move(args_converter).template call<Return, Guard>(cap->f);
      |                        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~~~~~~
/usr/lib/python3/dist-packages/pybind11/include/pybind11/pybind11.h:127:19:   required from ‘pybind11::cpp_function::cpp_function(Func&&, const Extra& ...) [with Func = wrap<int, int, int, foo::xfoo>; Extra = {pybind11::name, pybind11::scope, pybind11::sibling, char [45]}; <template-parameter-1-3> = void]’
  127 |         initialize(
      |         ~~~~~~~~~~^
  128 |             std::forward<Func>(f), (detail::function_signature_t<Func> *) nullptr, extra...);
      |             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/usr/lib/python3/dist-packages/pybind11/include/pybind11/pybind11.h:1188:22:   required from ‘pybind11::module_& pybind11::module_::def(const char*, Func&&, const Extra& ...) [with Func = wrap<int, int, int, foo::xfoo>; Extra = {char [45]}]’
 1188 |         cpp_function func(std::forward<Func>(f),
      |                      ^~~~
/tmp/example.cpp:50:10:   required from here
   50 |     m.def("add", wrap{foo::add}, "A function that adds two numbers and no enum");
      |     ~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/usr/lib/python3/dist-packages/pybind11/include/pybind11/cast.h:1631:37: error: no match for call to ‘(wrap<int, int, int, foo::xfoo>) (int, int, foo::xfoo&)’
 1631 |         return std::forward<Func>(f)(cast_op<Args>(std::move(std::get<Is>(argcasters)))...);
      |                ~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/tmp/example.cpp:14:7: note: candidate: ‘R wrap<R, Args>::operator()(Args&& ...) [with R = int; Args = {int, int, foo::xfoo}]’ (near match)
   14 |     R operator()(Args&&... args)
      |       ^~~~~~~~
/tmp/example.cpp:14:7: note:   conversion of argument 3 would be ill-formed:
/tmp/example.cpp:14:7: error: cannot bind rvalue reference of type ‘foo::xfoo&&’ to lvalue of type ‘foo::xfoo’

[luigibrancati]

Hi @smurfix,

I have been looking at this issue. This seems to be caused by the fact that when compiling the wrapped function, the compiler only generates the template instance wrap<int, int, int, xfoo>(int&&, int&&, xfoo&&) which is what you would expect in C++ by the resolution of the universal reference. However, pybind11 tries to pass the enums as lvalue so it looks for wrap<int, int, int, xfoo>(int, int, xfoo&), which hasn't been instantiated by the compiler.

I've attempted multiple solutions (see below), but most of them ultimately require to provide pybind11 with a non-templated function or operator() so that it can correctly infer its signature with decltype(&F::operator()), which pybind11 needs to correctly bind the callable.

My main takeaway here is that there isn't really any point in forcing a pure C++ behaviour (perfect forwarding) over pybind11. As mentioned here

[...] a copy of the data must be made on every Python ↔ C++ transition: this is needed since the C++ and Python versions of the same type generally won’t have the same memory layout.

Data is copied at the boundary between Python and C++. So, the natural solution would be to define a non-templated wrapper function to be used by pybind11, while keeping the templated wrapper class wrap as-is to be used in C++

#include <pybind11/pybind11.h>
#include <iostream>

namespace py = pybind11;

template <class R, class... Args>
struct wrap {
    using funct_type = R (*)(Args...);
    funct_type func;

    wrap(funct_type f) : func(f) {}

    // Templated operator() to be used in C++ code
    R operator()(Args&&... args) {
        // Perfect forwarding
        std::cout << "before calling\n";
        R ret = func(std::forward<Args>(args)...);
        std::cout << "after calling\n";
        return ret;
    }
};

enum xfoo {
    FOO = 1,
    BAR = 2,
    BAZ = 3,
};

// Example function we want to wrap
int add(int a, int b, enum xfoo c) {
    return a + b + c;
}

// Non-templated wrapper function for pybind11
int add_wrapped(int a, int b, enum xfoo c) {
    wrap w{&add};
    return w(a, b, c);
}

PYBIND11_MODULE(example, m) {
    py::enum_<xfoo>(m, "xfoo")
        .value("FOO", xfoo::FOO)
        .value("BAR", xfoo::BAR)
        .value("BAZ", xfoo::BAZ)
        .export_values();

    m.def("add", &add_wrapped);
    /*
    Alternative: use a lambda
    m.def("add", [](int a, int b, enum xfoo c) {
        wrap w{&add};
        return w(a, b, c);
    });
    */
}

You'll have to define an helper function like add_wrapped (or a corresponding lambda) for each function that you want wrap and use in Python.

I'll provide a few alternatives anyway (among the ones I've attempted), though I'm not sure if any of them is better than the one above.

Possible Alternatives

1. Modify the add function

One quick solution would be to modify the add function to expect the enum as an lvalue-reference instead of just an lvalue

int add(int a, int b, enum xfoo& c) {
    return a + b + c;
}

This, due to the fact that T& && -> T&, would force the compiler to generate exactly the template instance that pybind11 uses, i.e. wrap<int, int, int, xfoo>(int&&, int&&, xfoo&). This solution might or might not be feasible for your case depending on the number of functions you'd have to modify.

2. Drop the universal reference when encountering enums

One solution would be to replace the R operator()(Args&&... args) with R operator()(Args... args), which would of course pass all arguments as lvalues. This would defeat the purpose of using an universal references and deep-copy even complex objects, though, so we can try to mitigate this issue by using this overload only if Args... contains at least one enum

#include <iostream>
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <type_traits>

namespace py = pybind11;

// Helper types to detect enums in parameter pack
template <typename T>
constexpr bool is_enum_v = std::is_enum_v<std::remove_reference_t<T>>;

template <typename... Args>
constexpr bool has_enum_v = (is_enum_v<Args> || ...);

enum xfoo {
    FOO = 1,
    BAR = 2,
    BAZ = 3,
};

// Generic template
template <class R, class... Args>
struct wrap
{
    using funct_type = R (*)(Args...);
    funct_type func;

    wrap(funct_type f) : func(f) {}

    R operator()(Args&&... args)
    {
        // Before calling
        std::cout << "before calling\n";
        R ret = func(std::forward<Args>(args)...);
        // After calling
        std::cout << "After calling\n";
        return ret;
    }
};

// Specialization for when Args contains enums
template <class R, class... Args>
struct wrap_with_enum
{
    using funct_type = R (*)(Args...);
    funct_type func;

    wrap_with_enum(funct_type f) : func(f) {}

    R operator()(Args... args)
    {
        // Before calling
        std::cout << "before calling (specialized for enums)\n";
        R ret = func(args...); // Call without forwarding
        // After calling
        std::cout << "After calling (specialized for enums)\n";
        return ret;
    }
};

// Factory to choose the correct wrap implementation
template <class R, class... Args>
using wrap_selector = std::conditional_t<
    has_enum_v<Args...>, 
    wrap_with_enum<R, Args...>, 
    wrap<R, Args...>>;

int add(int a, int b) {
    return a + b;
}

int add_with_enum(int a, int b, enum xfoo c) {
    return a + b + c;
}

PYBIND11_MODULE(example, m) {
    m.doc() = "pybind11 example plugin"; // Optional module docstring

    py::enum_<xfoo>(m, "xfoo")
        .value("FOO", xfoo::FOO)
        .value("BAR", xfoo::BAR)
        .value("BAZ", xfoo::BAZ)
        .export_values();

    // Use wrap_selector to choose the appropriate implementation
    m.def("add", wrap_selector<int, int, int>{add}, "A function that adds two numbers");
    m.def("add_with_enum", wrap_selector<int, int, int, xfoo>{add_with_enum}, "A function that adds two numbers and an enum");
}

This solution uses Args... only when it finds at least one enum in the parameter pack, otherwise it uses the universal reference you wrote.

import example

print(example.add(1, 2, 3))
print(example.add_with_enum(1, 2, example.xfoo.FOO))

# Output:
# before calling
# After calling
# 3
# before calling (specialized for enums)
# After calling (specialized for enums)
# 4

3. Partial specialization/partial forwarding

This is a bit redundant, but I'll include it here for completeness.

Another route I attempted was to rewrite the wrapper class so that it only forwards non enum types while enums are passed by value. This would allow to use the wrapper class in C++ and Python code as-is. I was able to came up with the following code:

template <typename R, typename... Ts>
struct partial_forward_wrapper {
    using funct_type = R (*)(Ts...);
    funct_type func;

    explicit partial_forward_wrapper(funct_type f) : func(f) {}

    template <typename... UArgs>
    R operator()(UArgs&&... uargs) {
        static_assert(sizeof...(UArgs) == sizeof...(Ts),
            "Mismatched argument count");
        
        std::cout << "[before calling wrapped function]\n";
        R ret = call_impl(std::index_sequence_for<Ts...>{}, std::forward<UArgs>(uargs)...);
        std::cout << "[after calling wrapped function]\n";
        return ret;
    }

private:
    // Expand each argument with a helper that either perfect-forwards
    // or passes-by-value if it's an enum.
    template <std::size_t... I, typename... UArgs>
    R call_impl(std::index_sequence<I...>, UArgs&&... uargs) {
        // Call 'func' with each argument processed by 'select_arg'
        return func(
            select_arg<std::tuple_element_t<I, std::tuple<Ts...>>>(
                std::forward<UArgs>(uargs)
            )...
        );
    }

    // If T is an enum, cast & pass by value
    template <typename T, typename U>
    auto select_arg(U&& u) 
        -> std::enable_if_t<std::is_enum_v<T>, T>
    {
        return static_cast<T>(u);
    }

    // Otherwise, perfect-forward
    template <typename T, typename U>
    auto select_arg(U&& u)
        -> std::enable_if_t<!std::is_enum_v<T>, T&&>
    {
        return std::forward<U>(u);
    }
};

This works in C++, but unfortunately doesn't work with pybind11: the issue here is that operator() is still a template function when instantiating partial_forward_wrapper. This means it works in C++, but not in Pybind11 since it requires a concrete operator() to correctly infer its signature with decltype(&F::operator()).

A possible solution here would be to provide a non-templated function for pybind11 use, as done in the very first solution with add_wrapped, thus making all of the above code redundant.