Initial Numba support (#626)

* add FFI as a requirement for dynamic calls
* add basic JIT (Numba) test
* initial Numba/JIT support
* proper dyncall return type
* basic jit test
* apparently, numba has Float and float32 and they aren't the same thing
* test all supported JIT types
* move adding of properties to the end to ensure test is defined first
* add support for Numba-jited observers
* properly NULL pointers in callback move operators
* use cfunc from the top-level module instead of the decorators one
* move setting of py:phlex properties until after it has been created
* check for missing output suffixes on transform and report a proper error if it does
* pass transform concurrency on to the converter nodes
* verify conversion result in python providers
* delete py_callback_base move constructor/assignment
* fix shadowing of ffi_type
* add missing decref
* improve reporting of conversion errors
* temporary "fix" to make the AI happy; to be removed after release
* guarantee that a path that is supposed to fail actually does

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: wlav

plugins/python/CMakeLists.txt

@@ -7,6 +7,9 @@ if(Python_NumPy_VERSION VERSION_LESS "2.0.0")
   )
 endif()
 
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(FFI REQUIRED IMPORTED_TARGET libffi)
+
 # Phlex module to run Python algorithms
 add_library(
   pymodule
@@ -17,8 +20,13 @@ add_library(
   src/dciwrap.cpp
   src/lifelinewrap.cpp
   src/errorwrap.cpp
+  src/dyncall.cpp
+)
+
+target_link_libraries(
+  pymodule
+  PRIVATE phlex::module phlex::source PkgConfig::FFI Python::Python Python::NumPy
 )
-target_link_libraries(pymodule PRIVATE phlex::module phlex::source Python::Python Python::NumPy)
 target_compile_definitions(pymodule PRIVATE NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION)
 
 install(TARGETS pymodule LIBRARY DESTINATION lib)

plugins/python/python/phlex/_typing.py

@@ -14,11 +14,18 @@
 
 import numpy as np
 
+try:
+    import numba.core.types as nb_types
+    has_numba = True
+except ImportError:
+    has_numba = False
+
 __all__ = [
     "normalize_type",
 ]
 
-# ctypes and numpy types are likely candidates for use in annotations
+# ctypes and numpy types are likely candidates for use in annotations; Numba
+# types may appear from callback signatures
 # TODO: should users be allowed to add to these?
 _PY2CPP: dict[type, str] = {
     # numpy types
@@ -40,6 +47,23 @@
     # np.uintp:             "size_t",
 }
 
+if has_numba:
+    _PY2CPP.update({
+        nb_types.bool: "bool",
+        nb_types.int8: "int8_t",
+        nb_types.int16: "int16_t",
+        nb_types.int32: "int32_t",
+        nb_types.int64: "int64_t",
+        nb_types.uint8: "uint8_t",
+        nb_types.uint16: "uint16_t",
+        nb_types.uint32: "uint32_t",
+        nb_types.uint64: "uint64_t",
+        nb_types.Float: "float",
+        nb_types.float32: "float",
+        nb_types.double: "double",
+        nb_types.void: "None",
+    })
+
 # ctypes types that don't map cleanly to intN_t / uintN_t
 _CTYPES_SPECIAL: dict[type, str] = {}
 for _attr, _cpp in [
@@ -96,8 +120,11 @@ def _build_ctypes_map() -> dict[type, str]:
     "unsigned int": _PY2CPP[ctypes.c_uint],
     "long": _PY2CPP[ctypes.c_long],
     "unsigned long": _PY2CPP[ctypes.c_ulong],
+    # special cases; not necessarily correct but as expected on major platforms
     "long long": "int64_t",
     "unsigned long long": "uint64_t",
+    "float32": "float",
+    "float64": "double",
 }
 
 

plugins/python/src/dyncall.cpp

@@ -0,0 +1,129 @@
+// Dynamic dispatcher from generically packaged args to any C or Python function.
+//
+// Note: this particular implementation is based on libffi, presumed to be for
+// now the minimal dependency,  but an alternative could be based on JITing
+// using Cling or even Numba's llvmlite.
+
+#include "dyncall.hpp"
+#include <stdexcept>
+
+#include <ffi.h>
+
+using namespace phlex::experimental;
+
+phlex::experimental::dcarg phlex::experimental::dcarg::from_str(std::string const& stype)
+{
+  // only types currently used in modulewrap are added, not all ffi types
+  if (stype == "bool")
+    return dcarg(false);
+  else if (stype == "int32_t")
+    return dcarg(static_cast<std::int32_t>(0));
+  else if (stype == "uint32_t")
+    return dcarg(static_cast<std::uint32_t>(0));
+  else if (stype == "int64_t")
+    return dcarg(static_cast<ph_long_t>(0));
+  else if (stype == "uint64_t")
+    return dcarg(static_cast<ph_ulong_t>(0));
+  else if (stype == "float")
+    return dcarg(0.0f);
+  else if (stype == "double")
+    return dcarg(0.0);
+  else if (stype == "void")
+    return dcarg{};
+
+  throw std::invalid_argument("unknown type string: " + stype);
+}
+
+void* phlex::experimental::dcarg::value_ptr()
+{
+  return std::visit(
+    [](auto& val) -> void* {
+      using T = std::decay_t<decltype(val)>;
+      if constexpr (std::is_same_v<T, std::monostate>) {
+        return nullptr;
+      } else {
+        return static_cast<void*>(&val);
+      }
+    },
+    m_value);
+}
+
+namespace {
+  static ffi_type* get_ffi_type(dcarg const& d)
+  {
+    return std::visit(
+      [](auto&& val) -> ffi_type* {
+        using T = std::decay_t<decltype(val)>;
+
+        // there are duplicate bodies here b/c bool is represented by uint8,
+        // just as uint8 is, there being no bool in C; the code is cleaner
+        // with each type on its own line, however, rather than combining the
+        // two in a single predicate as a special case
+        // NOLINTBEGIN(bugprone-branch-clone)
+        if constexpr (std::is_same_v<T, std::monostate>)
+          return &ffi_type_void;
+        else if constexpr (std::is_same_v<T, void*>)
+          return &ffi_type_pointer;
+        else if constexpr (std::is_same_v<T, bool>)
+          return &ffi_type_uint8;
+        else if constexpr (std::is_same_v<T, std::int8_t>)
+          return &ffi_type_sint8;
+        else if constexpr (std::is_same_v<T, std::uint8_t>)
+          return &ffi_type_uint8;
+        else if constexpr (std::is_same_v<T, std::int16_t>)
+          return &ffi_type_sint16;
+        else if constexpr (std::is_same_v<T, std::uint16_t>)
+          return &ffi_type_uint16;
+        else if constexpr (std::is_same_v<T, std::int32_t>)
+          return &ffi_type_sint32;
+        else if constexpr (std::is_same_v<T, std::uint32_t>)
+          return &ffi_type_uint32;
+        else if constexpr (std::is_same_v<T, ph_long_t>)
+          return &ffi_type_sint64;
+        else if constexpr (std::is_same_v<T, ph_ulong_t>)
+          return &ffi_type_uint64;
+        else if constexpr (std::is_same_v<T, float>)
+          return &ffi_type_float;
+        else if constexpr (std::is_same_v<T, double>)
+          return &ffi_type_double;
+        // NOLINTEND(bugprone-branch-clone)
+      },
+      d.m_value);
+  }
+}
+
+void phlex::experimental::dyncall(void* fn, dcarg& result, dcargs_t& args, int var_offset)
+{
+  // Perform a dynamic call of function fn, taking arguments `args` and returning
+  // `result`. Set `var_offset` to the appropriate number of positional arguments
+  // if the other arguments are variational.
+
+  // Except for the memory management unique_ptrs, this code is essentially C,
+  // because libffi is, and that yields a plethora of warnings from clang-tidy,
+  // none of which warrant actual changes.
+  // NOLINTBEGIN
+  std::size_t nargs = (std::size_t)args.size();
+
+  auto t = std::make_unique<ffi_type*[]>(nargs);
+  auto p = std::make_unique<void*[]>(nargs);
+
+  for (dcargs_t::size_type i = 0; i < nargs; ++i) {
+    auto& a = args[i];
+    t[i] = get_ffi_type(a);
+    p[i] = a.value_ptr();
+  }
+
+  ffi_cif cif;
+  ffi_status status;
+  if (0 < var_offset)
+    status =
+      ffi_prep_cif_var(&cif, FFI_DEFAULT_ABI, var_offset, nargs, get_ffi_type(result), t.get());
+  else
+    status = ffi_prep_cif(&cif, FFI_DEFAULT_ABI, nargs, get_ffi_type(result), t.get());
+
+  if (status)
+    throw std::runtime_error("ffi prep failed");
+
+  ffi_call(&cif, (void (*)())fn, result.value_ptr(), p.get());
+  // NOLINTEND
+}

plugins/python/src/dyncall.hpp

@@ -0,0 +1,98 @@
+#ifndef PLUGINS_PYTHON_SRC_DYNCALL_HPP
+#define PLUGINS_PYTHON_SRC_DYNCALL_HPP
+
+// =======================================================================================
+//
+// Dynamic dispatcher from generically packaged args to any C or Python function.
+//
+// Design rationale
+// ================
+//
+// Python code is inserted in the Phlex execution graph using generic types to avoid a
+// combinatorial explosion of types. This way, all template instantiations can be done at
+// compile time. Callback wrappers are then needed to either pack from generic to Python
+// or to unpack from generic to C/C++ and perform the call. This dynamic dispatcher
+// provides that functionality.
+//
+// =======================================================================================
+
+#include "Python.h" // for PyObject* get<> specialization only
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <variant>
+#include <vector>
+
+#if defined(__APPLE__) && defined(__MACH__)
+// This is a temporary workaround until we have a solution for handling translation of types
+// between C++ and Python.
+typedef long ph_long_t;
+typedef unsigned long ph_ulong_t;
+#else
+typedef std::int64_t ph_long_t;
+typedef std::uint64_t ph_ulong_t;
+#endif
+
+namespace phlex::experimental {
+
+  struct dcarg {
+    using ffi_variant_type = std::variant<std::monostate, // void (default)
+                                          void*,
+                                          bool,
+                                          std::int8_t,
+                                          std::uint8_t,
+                                          std::int16_t,
+                                          std::uint16_t,
+                                          std::int32_t,
+                                          std::uint32_t,
+                                          ph_long_t,
+                                          ph_ulong_t,
+                                          float,
+                                          double>;
+
+    ffi_variant_type m_value;
+
+    // convenience mapper of human-readable string to dcarg
+    static dcarg from_str(std::string const& stype);
+
+    // factory-style constructors to guarantee value/type match
+    dcarg() : m_value(std::monostate{}) {}
+    explicit dcarg(void* v) : m_value(v) {}
+    explicit dcarg(bool v) : m_value(v) {}
+    explicit dcarg(std::int8_t v) : m_value(v) {}
+    explicit dcarg(std::uint8_t v) : m_value(v) {}
+    explicit dcarg(std::int16_t v) : m_value(v) {}
+    explicit dcarg(std::uint16_t v) : m_value(v) {}
+    explicit dcarg(std::int32_t v) : m_value(v) {}
+    explicit dcarg(std::uint32_t v) : m_value(v) {}
+    explicit dcarg(ph_long_t v) : m_value(v) {}
+    explicit dcarg(ph_ulong_t v) : m_value(v) {}
+    explicit dcarg(float v) : m_value(v) {}
+    explicit dcarg(double v) : m_value(v) {}
+
+    // pointer access to payload
+    void* value_ptr();
+
+    // value access to payload
+    template <typename T>
+    T get() const
+    {
+      return std::get<T>(m_value);
+    }
+  };
+
+  // specialization to simplify a very common case
+  template <>
+  inline PyObject* dcarg::get<PyObject*>() const
+  {
+    return reinterpret_cast<PyObject*>(std::get<void*>(m_value));
+  }
+
+  typedef std::vector<dcarg> dcargs_t;
+
+  void dyncall(void* fn, dcarg& result, dcargs_t& args, int var_offset = -1);
+
+} // phlex::experimental
+
+#endif // PLUGINS_PYTHON_SRC_DYNCALL_HPP