Skip to content

Commit c5ba697

Browse files
committed
Add capi backend (GOPY_BACKEND=capi)
1 parent c047d50 commit c5ba697

9 files changed

Lines changed: 302 additions & 8 deletions

File tree

‎.github/requirements-capi.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Python packages for the GOPY_BACKEND=capi job in workflows/ci.yml.
2+
# pybindgen is deliberately absent: the capi backend must not need it.
3+
# used by the memory-leak checks on Windows, where the resource module is missing
4+
psutil

‎.github/workflows/ci.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,50 @@ jobs:
239239
- name: Test
240240
run: go test -v -timeout=30m ./...
241241

242+
# Builds and tests the opt-in capi backend (GOPY_BACKEND=capi). Runs beside
243+
# the main matrix, on one Go version, and stops at the first failure. Builds
244+
# exactly as the default backend does, except that build.py writes the C
245+
# module itself (see bind/capi_build.py), so pybindgen isn't installed here.
246+
capi:
247+
name: capi backend (${{ matrix.platform }}, Python ${{ matrix.python-version }})
248+
strategy:
249+
fail-fast: true
250+
matrix:
251+
platform: [ubuntu-latest, windows-latest, macos-15]
252+
python-version: ['3.11', '3.12']
253+
runs-on: ${{ matrix.platform }}
254+
env:
255+
GOPY_BACKEND: capi
256+
# print the python stack if the process crashes, e.g. at exit
257+
PYTHONFAULTHANDLER: 1
258+
steps:
259+
- name: Checkout code
260+
uses: actions/checkout@v4
261+
262+
- name: Set up Python
263+
uses: actions/setup-python@v5
264+
with:
265+
python-version: ${{ matrix.python-version }}
266+
cache: pip
267+
cache-dependency-path: .github/requirements-capi.txt
268+
269+
- name: Install Go
270+
uses: actions/setup-go@v5
271+
with:
272+
go-version: 1.25.x
273+
cache: true
274+
275+
- name: Install packages
276+
run: |
277+
python -m pip install -r .github/requirements-capi.txt
278+
go install golang.org/x/tools/cmd/goimports@v0.29.0
279+
280+
- name: Build
281+
run: go build -v ./...
282+
283+
- name: Test
284+
run: go test -v ./...
285+
242286
# Compares per-call overhead across backends (see _examples/bench/run.sh):
243287
# not a pass/fail check, just a table uploaded as a build artifact. Runs
244288
# on ubuntu-latest only -- the comparison is between backends, not OSes.

‎_examples/bench/run.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ GOPY="$WORK/gopy"
2020

2121
printf '%-10s %8s %14s %14s %10s\n' backend calls "add (s)" "concat (s)" "peak (KB)"
2222

23-
for backend in pybindgen cffi pybind11 nanobind; do
23+
for backend in pybindgen capi cffi pybind11 nanobind; do
2424
out="$WORK/$backend"
2525
mkdir -p "$out"
2626
# gopy build cds into -output and runs go build there; give it a module

‎bind/backend.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ var backends = []struct {
3535
{BackendCFFI, true},
3636
{BackendPyBind11, true},
3737
{BackendNanobind, true},
38-
{BackendCAPI, false},
38+
{BackendCAPI, true},
3939
{BackendCGO, false},
4040
}
4141

‎bind/backend_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ func TestParseBackend(t *testing.T) {
2121
{in: "cffi", want: BackendCFFI},
2222
{in: "pybind11", want: BackendPyBind11},
2323
{in: "nanobind", want: BackendNanobind},
24-
{in: "capi", errPart: "not implemented yet"},
24+
{in: "capi", want: BackendCAPI},
25+
{in: "cgo", errPart: "not implemented yet"},
2526
{in: "bogus", errPart: "unknown GOPY_BACKEND"},
2627
} {
2728
got, err := parseBackend(tc.in)

‎bind/capi.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// Copyright 2026 The go-python Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package bind
6+
7+
import (
8+
_ "embed"
9+
"strings"
10+
)
11+
12+
// The capi backend (GOPY_BACKEND=capi) is the default backend without
13+
// pybindgen: the same cgo shim, which calls the CPython C API itself, and the
14+
// same build (the generated <name>.c compiled into the Go shared library).
15+
// Only build.py differs: capi_build.py records the pybindgen calls gopy
16+
// writes, and writes <name>.c from them itself.
17+
18+
//go:embed capi_build.py
19+
var capiBuildPy string
20+
21+
func (g *pyGen) isCAPI() bool {
22+
return g.cfg.Backend == BackendCAPI
23+
}
24+
25+
// capiBuildPreamble returns the start of build.py: the capi recorder.
26+
func (g *pyGen) capiBuildPreamble() string {
27+
return strings.NewReplacer(
28+
"@NAME@", g.cfg.Name,
29+
"@CMD@", g.cfg.Cmd,
30+
"@VERSION@", g.cfg.Version,
31+
).Replace(capiBuildPy)
32+
}

‎bind/capi_build.py‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
# python build stubs for package @NAME@ (capi backend)
2+
# File is generated by gopy version @VERSION@. Do not edit.
3+
# @CMD@
4+
#
5+
# The generated build code below is written against the pybindgen API. Here
6+
# the same calls are only recorded, and Module.generate() then writes
7+
# @NAME@.c itself: the same CPython C-API wrappers pybindgen would write, for
8+
# the same cgo shim, so nothing beyond the interpreter is needed to build.
9+
# Every conversion below follows pybindgen's own generated code, so that a
10+
# wrongly-typed argument raises the same exception under either backend.
11+
12+
13+
class Retval(object):
14+
def __init__(self, ctype, caller_owns_return=False, *a, **kw):
15+
self.ctype = ctype
16+
17+
18+
class Param(object):
19+
def __init__(self, ctype, name, transfer_ownership=True, *a, **kw):
20+
self.ctype = ctype
21+
self.name = name
22+
23+
24+
retval = Retval
25+
param = Param
26+
27+
28+
class Function(object):
29+
def __init__(self, name, ret, params, checked=False, frees_string=False):
30+
self.name = name
31+
self.ret = ret.ctype if ret is not None else None
32+
self.params = params
33+
self.checked = checked
34+
self.frees_string = frees_string
35+
36+
37+
class Module(object):
38+
def __init__(self, name, *a, **kw):
39+
self.name = name # the compiled extension's import name, e.g. "_hi"
40+
self.includes = []
41+
self.funcs = []
42+
43+
def add_include(self, inc):
44+
self.includes.append(inc)
45+
46+
def add_function(self, name, ret, params, *a, **kw):
47+
self.funcs.append(Function(name, ret, params))
48+
49+
def generate(self, out):
50+
out.write(MODULE_HEAD.replace("@INCLUDES@", "\n".join("#include " + i for i in self.includes)))
51+
for fn in self.funcs:
52+
out.write(wrapper(self.name, fn))
53+
out.write("static PyMethodDef %s_functions[] = {\n" % self.name)
54+
for fn in self.funcs:
55+
flags = "METH_VARARGS|METH_KEYWORDS" if fn.params else "METH_NOARGS"
56+
out.write(' {"%s", (PyCFunction)%s, %s, "%s"},\n' % (fn.name, wrap_name(self.name, fn), flags, doc(fn)))
57+
out.write(" {NULL, NULL, 0, NULL}\n};\n")
58+
out.write(MODULE_TAIL.replace("@MOD@", self.name))
59+
out.close()
60+
61+
62+
# failure_expression is never anything but '' from gopy, so a checked
63+
# function's only check is PyErr_Occurred(), as with pybindgen.
64+
def add_checked_function(mod, name, retval, params, failure_expression="", *a, **kw):
65+
mod.funcs.append(Function(name, retval, params, checked=True))
66+
67+
68+
# As above, and the char* the function returns is the caller's to free.
69+
def add_checked_string_function(mod, name, retval, params, failure_expression="", *a, **kw):
70+
mod.funcs.append(Function(name, retval, params, checked=True, frees_string=True))
71+
72+
73+
# ctype -> (PyArg_Parse format, C type parsed into, maximum value or None).
74+
# Narrow integers parse as int, and only their maximum is checked.
75+
PARSE = {
76+
"int64_t": ("L", "int64_t", None),
77+
"uint64_t": ("K", "uint64_t", None),
78+
"int": ("i", "int", None),
79+
"int32_t": ("i", "int", None),
80+
"uint32_t": ("I", "unsigned int", None),
81+
"int16_t": ("i", "int", "0x7fff"),
82+
"uint16_t": ("i", "int", "0xffff"),
83+
"int8_t": ("i", "int", "0x7f"),
84+
"uint8_t": ("i", "int", "0xff"),
85+
"double": ("d", "double", None),
86+
"float": ("f", "float", None),
87+
"bool": ("O", "PyObject *", None),
88+
"char*": ("s", "const char *", None),
89+
"PyObject*": ("O", "PyObject *", None),
90+
}
91+
92+
# ctype -> Py_BuildValue arguments for the result in "retval". Py_BuildValue
93+
# turns a NULL char* into None, and passes on a NULL PyObject* as an error.
94+
BUILD = {
95+
"int64_t": '"L", retval',
96+
"uint64_t": '"K", retval',
97+
"int": '"i", retval',
98+
"int32_t": '"i", retval',
99+
"int16_t": '"i", retval',
100+
"uint16_t": '"i", retval',
101+
"int8_t": '"i", retval',
102+
"uint8_t": '"i", (int)retval',
103+
"uint32_t": '"N", PyLong_FromUnsignedLong(retval)',
104+
"double": '"d", retval',
105+
"float": '"f", retval',
106+
"bool": '"N", PyBool_FromLong(retval)',
107+
"char*": '"s", retval',
108+
"PyObject*": '"N", retval',
109+
}
110+
111+
112+
def wrap_name(mod, fn):
113+
return "_wrap_%s_%s" % (mod, fn.name)
114+
115+
116+
def doc(fn):
117+
sig = "%s(%s)\\n\\n" % (fn.name, ", ".join(p.name for p in fn.params))
118+
return sig + "\\n".join("type: %s: %s" % (p.name, p.ctype.replace("*", " *")) for p in fn.params)
119+
120+
121+
def wrapper(mod, fn):
122+
for ctype in [p.ctype for p in fn.params] + ([fn.ret] if fn.ret else []):
123+
if ctype not in PARSE:
124+
raise ValueError("capi backend: unsupported C type %r in %s" % (ctype, fn.name))
125+
if not fn.params:
126+
lines = ["static PyObject *\n%s(PyObject *self, PyObject *unused)\n{" % wrap_name(mod, fn)]
127+
else:
128+
lines = ["static PyObject *\n%s(PyObject *self, PyObject *args, PyObject *kwargs)\n{" % wrap_name(mod, fn)]
129+
if fn.ret:
130+
lines.append(" %s retval;" % fn.ret)
131+
# locals are prefixed, so that no parameter name can collide with them
132+
call_args = []
133+
for p in fn.params:
134+
lines.append(" %s a_%s;" % (PARSE[p.ctype][1], p.name))
135+
if p.ctype == "bool":
136+
call_args.append("(bool)PyObject_IsTrue(a_%s)" % p.name)
137+
elif p.ctype == "char*":
138+
call_args.append("(char *)a_" + p.name)
139+
else:
140+
call_args.append("a_" + p.name)
141+
if fn.params:
142+
lines.append(
143+
" static const char *keywords[] = {%s, NULL};" % ", ".join('"%s"' % p.name for p in fn.params)
144+
)
145+
lines.append(
146+
' if (!PyArg_ParseTupleAndKeywords(args, kwargs, "%s", (char **)keywords, %s)) {\n'
147+
" return NULL;\n }"
148+
% ("".join(PARSE[p.ctype][0] for p in fn.params), ", ".join("&a_" + p.name for p in fn.params))
149+
)
150+
for p in fn.params:
151+
limit = PARSE[p.ctype][2]
152+
if limit:
153+
lines.append(
154+
" if (a_%s > %s) {\n"
155+
' PyErr_SetString(PyExc_ValueError, "Out of range");\n'
156+
" return NULL;\n }" % (p.name, limit)
157+
)
158+
call = "%s(%s);" % (fn.name, ", ".join(call_args))
159+
lines.append(" " + ("retval = " + call if fn.ret else call))
160+
if fn.checked:
161+
cleanup = " if (retval != NULL) free(retval);\n" if fn.frees_string else ""
162+
lines.append(" if (PyErr_Occurred()) {\n%s return NULL;\n }" % cleanup)
163+
if not fn.ret:
164+
lines.append(" Py_RETURN_NONE;")
165+
elif fn.frees_string:
166+
lines.append(" PyObject *py_retval = Py_BuildValue(%s);" % BUILD[fn.ret])
167+
lines.append(" free(retval);")
168+
lines.append(" return py_retval;")
169+
else:
170+
lines.append(" return Py_BuildValue(%s);" % BUILD[fn.ret])
171+
lines.append("}\n\n")
172+
return "\n".join(lines)
173+
174+
175+
MODULE_HEAD = """/* This file was generated by gopy's capi backend. Do not edit. */
176+
#define PY_SSIZE_T_CLEAN
177+
#include <Python.h>
178+
#include <stdlib.h>
179+
180+
@INCLUDES@
181+
182+
"""
183+
184+
# PyInit_ starts its own line: on Windows, cmd_build.go adds
185+
# __declspec(dllexport) after any " PyInit_" in the file, which
186+
# PyMODINIT_FUNC already has.
187+
MODULE_TAIL = """
188+
static struct PyModuleDef @MOD@_moduledef = {
189+
PyModuleDef_HEAD_INIT,
190+
"@MOD@",
191+
NULL,
192+
-1,
193+
@MOD@_functions,
194+
};
195+
196+
PyMODINIT_FUNC
197+
PyInit_@MOD@(void)
198+
{
199+
return PyModule_Create(&@MOD@_moduledef);
200+
}
201+
"""
202+
203+
204+
mod = Module('_@NAME@')
205+
mod.add_include('"@NAME@_go.h"')
206+
mod.add_function('GoPyInit', None, [])
207+
mod.add_function('DecRef', None, [param('int64_t', 'handle')])
208+
mod.add_function('IncRef', None, [param('int64_t', 'handle')])
209+
mod.add_function('NumHandles', retval('int'), [])
210+
mod.add_function('RequestGC', None, [])
211+
mod.add_function('_gopy_clear_go_tls', None, [])

‎bind/gen.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -794,6 +794,8 @@ func (g *pyGen) genPyBuildPreamble() {
794794
g.pybuild.Printf("%s", g.pybind11BuildPreamble())
795795
case g.isNanobind():
796796
g.pybuild.Printf("%s", g.nanobindBuildPreamble())
797+
case g.isCAPI():
798+
g.pybuild.Printf("%s", g.capiBuildPreamble())
797799
default:
798800
g.pybuild.Printf(PyBuildPreamble, g.cfg.Name, g.cfg.Cmd, g.cfg.Version)
799801
}

‎main_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -471,11 +471,11 @@ func TestMakefile(t *testing.T) {
471471
if err != nil {
472472
t.Fatal(err)
473473
}
474-
if backend == bind.BackendPyBindGen {
475-
// TODO: its Makefile links _simple against a separate simple_go
476-
// shared library with no rpath, so the result only imports with
477-
// that library's directory on the loader's search path.
478-
t.Skip("the pybindgen backend's Makefile output doesn't import as-is")
474+
if backend == bind.BackendPyBindGen || backend == bind.BackendCAPI {
475+
// TODO: their (shared) Makefile links _simple against a separate
476+
// simple_go shared library with no rpath, so the result only imports
477+
// with that library's directory on the loader's search path.
478+
t.Skipf("the %s backend's Makefile output doesn't import as-is", backend)
479479
}
480480
if _, err := exec.LookPath("make"); err != nil {
481481
t.Skip("make not found")

0 commit comments

Comments
 (0)