PyData London 2017

Foreign Function Interface Generator: Generating Python bindings from C++

Jonathan B Coe

jbcoe@ffig.org

https://github.com/ffig/ffig

We want to run C++ code in Python without doing any extra work.

Gathering Input

Write a C++ class out to a file in the current working directory



In [1]:

    
outputfile = "Shape.h"



In [2]:

    
%%file $outputfile

#include "ffig/attributes.h"
#include <stdexcept>
#include <string>

struct FFIG_EXPORT Shape
{
  virtual ~Shape() = default;
  virtual double area() const = 0;
  virtual double perimeter() const = 0;
  virtual const char* name() const = 0;
};

static const double pi = 3.14159;

class Circle : public Shape
{
  const double radius_;

public:
  double area() const override
  {
    return pi * radius_ * radius_;
  }

  double perimeter() const override
  {
    return 2 * pi * radius_;
  }

  const char* name() const override
  {
    return "Circle";
  }

  Circle(double radius) : radius_(radius)
  {
    if ( radius < 0 ) 
    { 
      std::string s = "Circle radius \"" + std::to_string(radius_) + "\" must be non-negative.";
      throw std::runtime_error(s);
    }
  }
};









    



Overwriting Shape.h

Compile our header to check it's valid C++



In [3]:

    
%%sh
clang++ -x c++ -fsyntax-only -std=c++14 -I../ffig/include Shape.h

Read the code using libclang



In [4]:

    
import sys
sys.path.insert(0,'../ffig')
sys.path.insert(0,'..')



In [5]:

    
import ffig.clang.cindex

index = ffig.clang.cindex.Index.create()
translation_unit = index.parse(outputfile, ['-x', 'c++', '-std=c++14', '-I../ffig/include'])



In [6]:

    
import asciitree

def node_children(node):
    return (c for c in node.get_children() if c.location.file.name == outputfile)

print(asciitree.draw_tree(translation_unit.cursor,
  lambda n: [c for c in node_children(n)],
  lambda n: "%s (%s)" % (n.spelling or n.displayname, str(n.kind).split(".")[1])))









    



Shape.h (TRANSLATION_UNIT)
  +--Shape (STRUCT_DECL)
  |  +--FFIG:EXPORT (ANNOTATE_ATTR)
  |  +--~Shape (DESTRUCTOR)
  |  |  +-- (COMPOUND_STMT)
  |  +--area (CXX_METHOD)
  |  +--perimeter (CXX_METHOD)
  |  +--name (CXX_METHOD)
  +--pi (VAR_DECL)
  |  +-- (FLOATING_LITERAL)
  +--Circle (CLASS_DECL)
     +--struct Shape (CXX_BASE_SPECIFIER)
     |  +--struct Shape (TYPE_REF)
     +--radius_ (FIELD_DECL)
     +-- (CXX_ACCESS_SPEC_DECL)
     +--area (CXX_METHOD)
     |  +-- (CXX_OVERRIDE_ATTR)
     |  +-- (COMPOUND_STMT)
     |     +-- (RETURN_STMT)
     |        +-- (BINARY_OPERATOR)
     |           +-- (BINARY_OPERATOR)
     |           |  +--pi (UNEXPOSED_EXPR)
     |           |  |  +--pi (DECL_REF_EXPR)
     |           |  +--radius_ (UNEXPOSED_EXPR)
     |           |     +--radius_ (MEMBER_REF_EXPR)
     |           +--radius_ (UNEXPOSED_EXPR)
     |              +--radius_ (MEMBER_REF_EXPR)
     +--perimeter (CXX_METHOD)
     |  +-- (CXX_OVERRIDE_ATTR)
     |  +-- (COMPOUND_STMT)
     |     +-- (RETURN_STMT)
     |        +-- (BINARY_OPERATOR)
     |           +-- (BINARY_OPERATOR)
     |           |  +-- (UNEXPOSED_EXPR)
     |           |  |  +-- (INTEGER_LITERAL)
     |           |  +--pi (UNEXPOSED_EXPR)
     |           |     +--pi (DECL_REF_EXPR)
     |           +--radius_ (UNEXPOSED_EXPR)
     |              +--radius_ (MEMBER_REF_EXPR)
     +--name (CXX_METHOD)
     |  +-- (CXX_OVERRIDE_ATTR)
     |  +-- (COMPOUND_STMT)
     |     +-- (RETURN_STMT)
     |        +-- (UNEXPOSED_EXPR)
     |           +--"Circle" (STRING_LITERAL)
     +--Circle (CONSTRUCTOR)
        +--radius (PARM_DECL)
        +--radius_ (MEMBER_REF)
        +--radius (UNEXPOSED_EXPR)
        |  +--radius (DECL_REF_EXPR)
        +-- (COMPOUND_STMT)
           +-- (IF_STMT)
              +-- (BINARY_OPERATOR)
              |  +--radius (UNEXPOSED_EXPR)
              |  |  +--radius (DECL_REF_EXPR)
              |  +-- (UNEXPOSED_EXPR)
              |     +-- (INTEGER_LITERAL)
              +-- (COMPOUND_STMT)
                 +-- (DECL_STMT)
                 |  +--s (VAR_DECL)
                 |     +--std (NAMESPACE_REF)
                 |     +--string (TYPE_REF)
                 |     +-- (UNEXPOSED_EXPR)
                 |        +-- (CALL_EXPR)
                 |           +-- (UNEXPOSED_EXPR)
                 |              +-- (UNEXPOSED_EXPR)
                 |                 +--operator+ (CALL_EXPR)
                 |                    +-- (UNEXPOSED_EXPR)
                 |                    |  +-- (UNEXPOSED_EXPR)
                 |                    |     +--operator+ (CALL_EXPR)
                 |                    |        +-- (UNEXPOSED_EXPR)
                 |                    |        |  +--"Circle radius \"" (STRING_LITERAL)
                 |                    |        +--operator+ (UNEXPOSED_EXPR)
                 |                    |        |  +--operator+ (DECL_REF_EXPR)
                 |                    |        +-- (UNEXPOSED_EXPR)
                 |                    |           +-- (UNEXPOSED_EXPR)
                 |                    |              +--to_string (CALL_EXPR)
                 |                    |                 +--to_string (UNEXPOSED_EXPR)
                 |                    |                 |  +--to_string (DECL_REF_EXPR)
                 |                    |                 |     +--std (NAMESPACE_REF)
                 |                    |                 +--radius_ (UNEXPOSED_EXPR)
                 |                    |                    +--radius_ (MEMBER_REF_EXPR)
                 |                    +--operator+ (UNEXPOSED_EXPR)
                 |                    |  +--operator+ (DECL_REF_EXPR)
                 |                    +-- (UNEXPOSED_EXPR)
                 |                       +--"\" must be non-negative." (STRING_LITERAL)
                 +-- (UNEXPOSED_EXPR)
                    +-- (CXX_THROW_EXPR)
                       +-- (CALL_EXPR)
                          +-- (UNEXPOSED_EXPR)
                             +-- (UNEXPOSED_EXPR)
                                +-- (CXX_FUNCTIONAL_CAST_EXPR)
                                   +--std (NAMESPACE_REF)
                                   +--class std::runtime_error (TYPE_REF)
                                   +-- (UNEXPOSED_EXPR)
                                      +--runtime_error (CALL_EXPR)
                                         +--s (UNEXPOSED_EXPR)
                                            +--s (DECL_REF_EXPR)

Turn the AST into some easy to manipulate Python classes



In [7]:

    
import ffig.cppmodel
import ffig.clang.cindex



In [8]:

    
model = ffig.cppmodel.Model(translation_unit=translation_unit, force_noexcept=False)



In [9]:

    
[f.name for f in model.functions][-5:]









    Out[9]:





['to_wstring', 'operator""s', 'operator""s', 'operator""s', 'operator""s']



In [10]:

    
[c.name for c in model.classes][-5:]









    Out[10]:





['char_traits', 'basic_string', 'basic_string', 'Shape', 'Circle']



In [11]:

    
shape_class = [c for c in model.classes if c.name=='Shape'][0]



In [12]:

    
["{}::{}".format(shape_class.name,m.name) for m in shape_class.methods]









    Out[12]:





['Shape::area', 'Shape::perimeter', 'Shape::name']

Code Generation

We now have some input to use in a code generator.

Look at the templates the generator uses



In [13]:

    
%cat ../ffig/templates/json.tmpl









    



[{% for class in classes %}
{
  "name" : "{{class.name}}"{% if class.methods %},
  "methods" : [{% for method in class.methods %}
    {
      "name" : "{{method.name}}",
      "return_type" : "{{method.return_type}}"
    }{% if not loop.last %},{% endif %}{% endfor %}
  ]{% endif %}
}{% if not loop.last %},{% endif %}{% endfor %}
]

Run the code generator



In [14]:

    
%%sh
cd ../
python -m ffig -b _c.h.tmpl _c.cpp.tmpl json.tmpl python -m Shape -i demos/Shape.h -o demos

See what it created



In [15]:

    
%ls









    



CMakeCache.txt              Shape/
CMakeFiles/                 Shape.h
CMakeLists.txt              Shape.json
CppLondon_Aug-2017.ipynb    Shape_c.cpp
LLVM-Cauldron.ipynb.broken  Shape_c.h
Makefile                    cmake_install.cmake
PyDataLondon-2017.ipynb     libShape_c.dylib*



In [16]:

    
%cat Shape.json









    



[
{
  "name" : "Shape",
  "methods" : [
    {
      "name" : "area",
      "return_type" : "double"
    },
    {
      "name" : "perimeter",
      "return_type" : "double"
    },
    {
      "name" : "name",
      "return_type" : "const char *"
    }
  ]
}
]

Build some bindings with the generated code.



In [17]:

    
%%file CMakeLists.txt

cmake_minimum_required(VERSION 3.0)
set(CMAKE_CXX_STANDARD 14)

include_directories(../ffig/include)

add_library(Shape_c SHARED Shape_c.cpp)









    



Overwriting CMakeLists.txt



In [18]:

    
%%sh
cmake . 
cmake --build .









    



-- Configuring done
-- Generating done
-- Build files have been written to: /Users/jon/DEV/FFIG/demos
Scanning dependencies of target Shape_c
[ 50%] Building CXX object CMakeFiles/Shape_c.dir/Shape_c.cpp.o
[100%] Linking CXX shared library libShape_c.dylib
[100%] Built target Shape_c



In [19]:

    
%%sh
nm -U libShape_c.dylib | c++filt









    



0000000000003cf0 short GCC_except_table10
0000000000003d7c short GCC_except_table15
0000000000003ddc short GCC_except_table26
0000000000003bd0 short GCC_except_table6
0000000000003c30 short GCC_except_table8
0000000000003c90 short GCC_except_table9
0000000000002580 T _Shape_Circle_create
0000000000002220 T _Shape_Shape_area
00000000000021c0 T _Shape_Shape_dispose
0000000000002470 T _Shape_Shape_name
0000000000002350 T _Shape_Shape_perimeter
0000000000001f70 T _Shape_clear_error
00000000000020f0 T _Shape_error
0000000000004108 short Shape_error_
0000000000004248 short __ZL12Shape_error_$tlv$init
0000000000002f30 unsigned short Shape::Shape()
0000000000003030 unsigned short Shape::~Shape()
0000000000003010 unsigned short Shape::~Shape()
0000000000002f50 unsigned short Shape::~Shape()
0000000000002b60 unsigned short Circle::Circle(double)
0000000000002b90 unsigned short Circle::Circle(double)
0000000000002f80 unsigned short Circle::~Circle()
0000000000002f60 unsigned short Circle::~Circle()
0000000000003010 unsigned short Circle::~Circle()
0000000000002fb0 unsigned short Circle::area() const
0000000000002ff0 unsigned short Circle::name() const
0000000000002fd0 unsigned short Circle::perimeter() const
0000000000003810 unsigned short std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >::__get_deleter(std::type_info const&) const
0000000000002200 unsigned short std::__1::shared_ptr<Shape>::~shared_ptr()
0000000000002b20 unsigned short std::__1::shared_ptr<Shape>::~shared_ptr()
0000000000002ad0 unsigned short std::__1::shared_ptr<Shape const>::shared_ptr<Circle const>(Circle const*, std::__1::enable_if<is_convertible<Circle const*, Shape const*>::value, std::__1::shared_ptr<Shape const>::__nat>::type)
0000000000003060 unsigned short std::__1::shared_ptr<Shape const>::shared_ptr<Circle const>(Circle const*, std::__1::enable_if<is_convertible<Circle const*, Shape const*>::value, std::__1::shared_ptr<Shape const>::__nat>::type)
0000000000002b00 T std::__1::char_traits<char>::assign(char&, char const&)
0000000000003740 unsigned short std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >::__on_zero_shared()
00000000000038a0 unsigned short std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >::__on_zero_shared_weak()
0000000000003710 unsigned short std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >::~__shared_ptr_pointer()
00000000000036f0 unsigned short std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >::~__shared_ptr_pointer()
0000000000003940 unsigned short std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >::~__shared_ptr_pointer()
0000000000003960 unsigned short thread-local initialization routine for Shape_error_
0000000000004178 D typeinfo for Shape
0000000000004190 D typeinfo for Circle
0000000000004238 D typeinfo for std::__1::default_delete<Circle const>
0000000000004220 D typeinfo for std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >
0000000000003e18 S typeinfo name for Shape
0000000000003e10 S typeinfo name for Circle
0000000000003e80 S typeinfo name for std::__1::default_delete<Circle const>
0000000000003e20 S typeinfo name for std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >
00000000000041a8 double vtable for Shape
0000000000004140 double vtable for Circle
00000000000041e0 double vtable for std::__1::__shared_ptr_pointer<Circle const*, std::__1::default_delete<Circle const>, std::__1::allocator<Circle const> >
00000000000020d0 unsigned short thread-local wrapper routine for Shape_error_
0000000000002340 unsigned short ___clang_call_terminate
00000000000039b0 unsigned short ___cxx_global_var_init
0000000000004120 short ___tls_guard
0000000000004260 short ___tls_guard$tlv$init
0000000000003960 unsigned short ___tls_init



In [20]:

    
cat Shape/_py3.py









    



# This code was generated by FFIG <http://ffig.org>.
# Manual edits will be lost.

import os
from ctypes import *
c_object_p = POINTER(c_void_p)

class c_interop_string(c_char_p):

  def __init__(self, p=None):
    if p is None:
      p = ""
    if isinstance(p, str):
      p = p.encode("utf8")
    super(c_char_p, self).__init__(p)

  def __str__(self):
    return self.value

  @property
  def value(self):
    if super(c_char_p, self).value is None:
      return None
    return super(c_char_p, self).value.decode("utf8")

  @classmethod
  def from_param(cls, param):
    if isinstance(param, str):
      return cls(param)
    if isinstance(param, bytes):
      return cls(param)
    raise TypeError("Cannot convert '{}' to '{}'".format(type(param).__name__, cls.__name__))

  @staticmethod
  def to_python_string(x, *args):
    return x.value


class Shape_error(Exception):
    def __init__(self):
        self.value = conf.lib.Shape_error()
        conf.lib.Shape_clear_error()

    def __str__(self):
        return self.value


class Shape:
  def area(self) -> 'float':
    rv = c_double()
    rc = conf.lib.Shape_Shape_area(self , byref(rv))
    if rc == 0:
      return rv.value
    raise Shape_error()
  def perimeter(self) -> 'float':
    rv = c_double()
    rc = conf.lib.Shape_Shape_perimeter(self , byref(rv))
    if rc == 0:
      return rv.value
    raise Shape_error()
  def name(self) -> 'str':
    rv = c_interop_string()
    rc = conf.lib.Shape_Shape_name(self , byref(rv))
    if rc == 0:
      return rv.value
    raise Shape_error()
  
  @classmethod
  def from_param(k, x):
    assert isinstance(x,k)
    return x.ptr

  def __del__(self):
    conf.lib.Shape_Shape_dispose(self)

class Circle(Shape):
  @classmethod
  def from_capi(cls, ptr):
    assert(isinstance(ptr, c_object_p))
    if not bool(ptr): 
      return None
    return cls(_p=ptr)

  def __init__(self, radius: 'float'=None,  _p=None) -> 'Circle':
    if _p:
      self.ptr = _p
    else:
      self.ptr = c_object_p()
      rc = conf.lib.Shape_Circle_create(radius,  byref(self.ptr))
      if rc != 0:
        raise Shape_error()



methodList = [
  ("Shape_error",
  [],
  c_interop_string,
  c_interop_string.to_python_string),

  ("Shape_clear_error",
  [],
  None),
  

  ("Shape_Shape_dispose",
  [Shape],
  None),
  ("Shape_Circle_create",
  [c_double,  POINTER(c_object_p)],
  c_int),
  ("Shape_Shape_area",
  [Shape, POINTER(c_double)],
  c_int),
  ("Shape_Shape_perimeter",
  [Shape, POINTER(c_double)],
  c_int),
  ("Shape_Shape_name",
  [Shape, POINTER(c_interop_string)],
  c_int),
]

# library loading and method registrations
# based on clang python bindings approach

def register_method(lib, item):
  func = getattr(lib, item[0])

  if len(item) >= 2:
    func.argtypes = item[1]

  if len(item) >= 3:
    func.restype = item[2]

  if len(item) == 4:
    func.errcheck = item[3]

class CachedProperty(object):

    def __init__(self, wrapped):
        self.wrapped = wrapped
        try:
            self.__doc__ = wrapped.__doc__
        except:
            pass

    def __get__(self, instance, instance_type=None):
        if instance is None:
            return self

        value = self.wrapped(instance)
        setattr(instance, self.wrapped.__name__, value)

        return value

class Config:
    library_path = None
    loaded = False

    @staticmethod
    def set_library_path(path):
        if Config.loaded:
            raise Exception("library path is already set.")
        Config.library_path = path

    @CachedProperty
    def lib(self):
        lib = self._get_library()
        for m in methodList:
          register_method(lib,m)
        Config.loaded = True
        return lib

    def _get_filename(self):
        import platform
        name = platform.system()

        if name == 'Darwin':
            file = 'libShape_c.dylib'
        elif name == 'Windows':
            file = 'Shape_c.dll'
        else:
            file = 'libShape_c.so'
        return file

    def _get_filepath(self):
        filename = self._get_filename()
        if not Config.library_path:
            return filename
        return os.path.join(Config.library_path, filename)

    def _get_library(self):
        try:
            # Use user-specified library path.
            if Config.library_path:
                library = cdll.LoadLibrary(self._get_filepath())
            else:
                # Use local file
                try:
                    this_file_dir = os.path.abspath(os.path.dirname(__file__))
                    library = cdll.LoadLibrary(os.path.join(this_file_dir, self._get_filename()))
                except:
                    # Use system library path (last).
                    library = cdll.LoadLibrary(self._get_filename())
        except OSError as e:
            msg = str(e) + ". To provide a path to {} use Config.set_library_path()".format(self._get_filename())
            raise Exception(msg)

        return library

conf = Config()



In [21]:

    
%%python2
import Shape
Shape.Config.set_library_path(".")
c = Shape.Circle(8)

print("A {} with radius {} has area {}".format(c.name(), 8, c.area()))









    



A Circle with radius 8 has area 201.06176



In [22]:

    
%%python3
import Shape
Shape.Config.set_library_path(".")
c = Shape.Circle(8)

print("A {} with radius {} has area {}".format(c.name(), 8, c.area()))









    



A Circle with radius 8 has area 201.06176



In [23]:

    
%%script pypy
import Shape
Shape.Config.set_library_path(".")
c = Shape.Circle(8)

print("A {} with radius {} has area {}".format(c.name(), 8, c.area()))









    



A Circle with radius 8 has area 201.06176



In [24]:

    
%%script /opt/intel/intelpython35/bin/python
import Shape
Shape.Config.set_library_path(".")
c = Shape.Circle(8)

print("A {} with radius {} has area {}".format(c.name(), 8, c.area()))









    



A Circle with radius 8 has area 201.06176



In [25]:

    
from Shape import *



In [26]:

    
try:
    c = Circle(-8)
except Exception as e:
    print(e)









    



Circle radius "-8.000000" must be non-negative.

FFIG needs you!

FFIG is MIT-licensed and hosted on GitHub.

Contributions, issues and feedback are very welcome.