Skip to content

Commit 6c41564

Browse files
b-longb-long
committed
Fix generated Makefile for pybind11 and nanobind
1 parent d703fa8 commit 6c41564

4 files changed

Lines changed: 273 additions & 119 deletions

File tree

‎bind/cxxbuild.go‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
// Copyright 2026 The go-python Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package bind
6+
7+
import (
8+
"os"
9+
"regexp"
10+
"runtime"
11+
"strings"
12+
)
13+
14+
// How the C++ backends (pybind11, nanobind) compile and link their module,
15+
// shared by gopy build (buildCXXModule in cmd_build.go, which runs these
16+
// steps itself) and the Makefile written by gopy gen/pkg (genMakefileCXX).
17+
//
18+
// The Go side builds as a static archive (-buildmode=c-archive), not a
19+
// shared library: the generated .cpp defines the callback trampolines Go
20+
// calls into (see pybind11_callback.go) as well as calling into Go itself,
21+
// and two separately-built shared libraries can't have a dependency cycle
22+
// like that -- neither can exist as a complete, loadable file before the
23+
// other -- so both sides' symbols are resolved in the single final link
24+
// that CXXArgs describes.
25+
26+
// NanobindCXXFlags are the flags nanobind's own build uses for libnanobind
27+
// (see the comment at the top of its nb_combined.cpp), which the nanobind
28+
// backend compiles into every module; harmless for the generated .cpp too.
29+
var NanobindCXXFlags = []string{"-DNDEBUG", "-DNB_COMPACT_ASSERTIONS", "-fno-strict-aliasing"}
30+
31+
// CXX returns the C++ compiler to use: $CXX, or else c++.
32+
func CXX() string {
33+
if cxx := os.Getenv("CXX"); cxx != "" {
34+
return cxx
35+
}
36+
return "c++"
37+
}
38+
39+
// CXXArchive returns the file name of the static archive the cgo shim for
40+
// package name builds as.
41+
func CXXArchive(name string) string {
42+
return name + "_go.a"
43+
}
44+
45+
// ExtModuleName returns the file name of the extension module _<name>,
46+
// preferring the interpreter's own suffix (e.g. .cpython-312-x86_64-linux-gnu.so)
47+
// over libext.
48+
func ExtModuleName(name, libext string, pycfg PyConfig) string {
49+
if pycfg.ExtSuffix != "" {
50+
return "_" + name + pycfg.ExtSuffix
51+
}
52+
return "_" + name + libext
53+
}
54+
55+
// CXXArgs returns the C++ compiler arguments that compile <name>.cpp and
56+
// srcs (the C++ library's own sources, if any) with libflags (its include
57+
// directories and flags of its own), and link them with the archive into
58+
// modlib.
59+
func CXXArgs(name, modlib string, pycfg PyConfig, libflags, srcs []string) []string {
60+
archive := CXXArchive(name)
61+
// pycfg.CFlags/LdFlags quote each path (for the shell that CGO_CFLAGS/
62+
// CGO_LDFLAGS normally go through); these arguments go to the compiler
63+
// directly, so unquote each field here.
64+
unquote := func(fields []string) []string {
65+
o := make([]string, len(fields))
66+
for i, f := range fields {
67+
o[i] = strings.Trim(f, `"`)
68+
}
69+
return o
70+
}
71+
// modlib depends on libpython (wherever this VM's own one lives, e.g.
72+
// not on the loader's default search path for a uv- or pyenv-managed
73+
// Python); without an rpath, the loader only finds it if it happens to
74+
// already be on its search path.
75+
var libdir string
76+
if m := regexp.MustCompile(`-L(\S+)`).FindStringSubmatch(pycfg.LdFlags); m != nil {
77+
libdir = strings.Trim(m[1], `"`)
78+
}
79+
// The archive's Go runtime code calls into gopy_cb_N (defined in the
80+
// .cpp), so the linker must be told to keep every object in it -- left
81+
// to its own judgement, it would see nothing in the .cpp calling into
82+
// the archive first and drop it as unused. GNU ld (Linux, and Windows'
83+
// MinGW) and ld64 (macOS) spell that differently.
84+
var archiveArgs []string
85+
if runtime.GOOS == "darwin" {
86+
archiveArgs = []string{"-Wl,-force_load," + archive}
87+
} else {
88+
archiveArgs = []string{"-Wl,--whole-archive", archive, "-Wl,--no-whole-archive"}
89+
}
90+
args := []string{"-std=c++17", "-fPIC", "-shared", "-O2"}
91+
switch runtime.GOOS {
92+
case "darwin":
93+
args = append(args, "-Wl,-rpath,@loader_path")
94+
if libdir != "" {
95+
args = append(args, "-Wl,-rpath,"+libdir)
96+
}
97+
case "windows":
98+
// No rpath equivalent; modlib depends on nothing but libpython, the
99+
// Go side being a static archive rather than a separate DLL of its
100+
// own. MinGW's own runtime (libstdc++/libgcc/libwinpthread), which
101+
// g++ links dynamically by default, has no such fix available -- it
102+
// isn't found by name alone unless its directory happens to be on
103+
// PATH -- so link it in statically instead. The C runtime (ucrt)
104+
// stays dynamic, shared with Python's own.
105+
args = append(args, "-static-libgcc", "-static-libstdc++",
106+
"-Wl,-Bstatic,--whole-archive", "-lwinpthread", "-Wl,--no-whole-archive", "-Wl,-Bdynamic")
107+
default:
108+
args = append(args, "-Wl,-rpath,$ORIGIN")
109+
if libdir != "" {
110+
args = append(args, "-Wl,-rpath,"+libdir)
111+
}
112+
}
113+
args = append(args, libflags...)
114+
args = append(args, unquote(strings.Fields(pycfg.CFlags))...)
115+
args = append(args, name+".cpp")
116+
args = append(args, srcs...)
117+
args = append(args, archiveArgs...)
118+
args = append(args, unquote(strings.Fields(pycfg.LdFlags))...)
119+
// c-archive mode (unlike c-shared) doesn't resolve the Go runtime's own
120+
// dependencies on these itself; TODO: verified only on Linux -- unclear
121+
// yet whether Windows/macOS need anything of their own added here too.
122+
if runtime.GOOS != "windows" {
123+
args = append(args, "-lpthread", "-ldl", "-lm")
124+
}
125+
return append(args, "-o", modlib)
126+
}
127+
128+
// makeShellArg returns arg written into a Makefile recipe line, so that the
129+
// shell make runs it with receives arg itself: make variable references
130+
// ("$(...)") are left for make to expand, any other "$" is escaped from
131+
// make, and anything the shell would split or expand is single-quoted.
132+
func makeShellArg(arg string) string {
133+
if strings.HasPrefix(arg, "$(") {
134+
return arg
135+
}
136+
arg = strings.ReplaceAll(arg, "$", "$$")
137+
if arg == "" || strings.ContainsAny(arg, " \t\n'\"\\`*?[#~&;|<>()$") {
138+
return "'" + strings.ReplaceAll(arg, "'", `'\''`) + "'"
139+
}
140+
return arg
141+
}
142+
143+
// MakefileTemplateCXX is the Makefile for the C++ backends: 1 = package
144+
// name, 2 = gopy command, 3 = gencmd, 4 = vm, 5 = C++ compiler, 6 = the
145+
// backend's own make variables (CXXLIBFOUND, CXXLIBFLAGS, CXXLIBSRCS), 7 = C++ compiler
146+
// arguments, 8 = gopy version, 9 = backend name, 10 = module file name.
147+
const MakefileTemplateCXX = `# Makefile for python interface for package %[1]s, using %[9]s.
148+
# File is generated by gopy version %[8]s. Do not edit.
149+
# %[2]s
150+
151+
GOCMD=go
152+
GOBUILD=$(GOCMD) build -mod=mod
153+
GOIMPORTS=goimports
154+
PYTHON=%[4]s
155+
CXX=%[5]s
156+
%[6]s
157+
all: gen build
158+
159+
gen:
160+
%[3]s
161+
162+
build:
163+
# $(shell ...) expands to nothing, rather than failing, if $(PYTHON) can't find %[9]s
164+
@test -n "$(CXXLIBFOUND)" || { echo "%[9]s not found for $(PYTHON) (pip install %[9]s)" >&2; exit 1; }
165+
# goimports is needed to ensure that the imports list is valid
166+
$(GOIMPORTS) -w %[1]s.go
167+
# build %[1]s_go.a from %[1]s.go -- the cgo wrappers to go functions -- as a
168+
# static archive: the module below links it in, rather than loading it
169+
$(GOBUILD) -buildmode=c-archive -o %[1]s_go.a %[1]s.go
170+
# writes %[1]s.cpp, the %[9]s module wrapping it
171+
$(PYTHON) build.py
172+
# compile and link %[10]s, the module %[1]s.py imports
173+
$(CXX) %[7]s
174+
175+
`
176+
177+
// genMakefileCXX writes the Makefile for the C++ backends.
178+
func (g *pyGen) genMakefileCXX(gencmd string, pycfg PyConfig) {
179+
// The C++ library's own include directories and sources are looked up
180+
// when make runs, not now, so that gopy gen works without it installed.
181+
var libvars string
182+
switch {
183+
case g.isPyBind11():
184+
libvars = `CXXLIBFLAGS=$(shell $(PYTHON) -m pybind11 --includes)
185+
CXXLIBSRCS=
186+
CXXLIBFOUND=$(CXXLIBFLAGS)
187+
`
188+
case g.isNanobind():
189+
libvars = `NANOBIND_INC=$(shell $(PYTHON) -c "import nanobind; print(nanobind.include_dir())")
190+
NANOBIND_SRC=$(shell $(PYTHON) -c "import nanobind; print(nanobind.source_dir())")
191+
# robin_map is a dependency nanobind vendors next to its own headers
192+
CXXLIBFLAGS=-I$(NANOBIND_INC) -I$(NANOBIND_INC)/../ext/robin_map/include ` + strings.Join(NanobindCXXFlags, " ") + `
193+
CXXLIBSRCS=$(NANOBIND_SRC)/nb_combined.cpp
194+
CXXLIBFOUND=$(NANOBIND_INC)
195+
`
196+
}
197+
modlib := ExtModuleName(g.cfg.Name, g.libext, pycfg)
198+
args := CXXArgs(g.cfg.Name, modlib, pycfg, []string{"$(CXXLIBFLAGS)"}, []string{"$(CXXLIBSRCS)"})
199+
for i, a := range args {
200+
args[i] = makeShellArg(a)
201+
}
202+
g.makefile.Printf(MakefileTemplateCXX, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, CXX(), libvars,
203+
strings.Join(args, " "), g.cfg.Version, g.cfg.Backend, modlib)
204+
}

‎bind/gen.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -887,6 +887,8 @@ func (g *pyGen) genMakefile() {
887887

888888
if g.mode == ModeExe {
889889
g.makefile.Printf(MakefileExeTemplate, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, pycfg.CFlags, pycfg.LdFlags, g.cfg.Version)
890+
} else if g.isCXXShim() {
891+
g.genMakefileCXX(gencmd, pycfg)
890892
} else if g.isCFFI() {
891893
g.makefile.Printf(MakefileTemplateCFFI, g.cfg.Name, g.cfg.Cmd, gencmd, g.cfg.VM, g.libext, "", "", "", "", g.cfg.Version)
892894
} else {

‎cmd_build.go‎

Lines changed: 15 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"os"
1212
"os/exec"
1313
"path/filepath"
14-
"regexp"
1514
"runtime"
1615
"strings"
1716

@@ -135,9 +134,9 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error {
135134

136135
switch cfg.Backend {
137136
case bind.BackendPyBind11:
138-
return buildPyBind11(cfg, buildname, pycfg)
137+
return buildPyBind11(cfg, pycfg)
139138
case bind.BackendNanobind:
140-
return buildNanobind(cfg, buildname, pycfg)
139+
return buildNanobind(cfg, pycfg)
141140
}
142141

143142
if mode == bind.ModeExe {
@@ -377,20 +376,20 @@ func buildCFFI(cfg *BuildCfg, buildLib string) error {
377376
}
378377

379378
// buildPyBind11 builds the pybind11 backend's module (see buildCXXModule).
380-
func buildPyBind11(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error {
379+
func buildPyBind11(cfg *BuildCfg, pycfg bind.PyConfig) error {
381380
cmdout, err := exec.Command(cfg.VM, "-m", "pybind11", "--includes").CombinedOutput()
382381
if err != nil {
383382
fmt.Printf("cmd had error: %v output:\n%v\n(is pybind11 installed? pip install pybind11)\n", err, string(cmdout))
384383
return err
385384
}
386-
return buildCXXModule(cfg, buildname, pycfg, strings.Fields(strings.TrimSpace(string(cmdout))), nil)
385+
return buildCXXModule(cfg, pycfg, strings.Fields(strings.TrimSpace(string(cmdout))), nil)
387386
}
388387

389388
// buildNanobind builds the nanobind backend's module (see buildCXXModule).
390389
// Unlike pybind11, nanobind isn't header-only: its own runtime (libnanobind)
391390
// ships as source, meant to be compiled into each extension alongside the
392391
// extension's own code, which nb_combined.cpp does in one translation unit.
393-
func buildNanobind(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error {
392+
func buildNanobind(cfg *BuildCfg, pycfg bind.PyConfig) error {
394393
cmdout, err := exec.Command(cfg.VM, "-c",
395394
"import nanobind; print(nanobind.include_dir()); print(nanobind.source_dir())").CombinedOutput()
396395
if err != nil {
@@ -404,41 +403,27 @@ func buildNanobind(cfg *BuildCfg, buildname string, pycfg bind.PyConfig) error {
404403
incdir, srcdir := strings.TrimSpace(dirs[0]), strings.TrimSpace(dirs[1])
405404
// robin_map is a dependency nanobind vendors next to its own headers.
406405
robinmap := filepath.Join(filepath.Dir(incdir), "ext", "robin_map", "include")
407-
flags := []string{
408-
"-I" + incdir, "-I" + robinmap,
409-
// as nanobind's own build does for libnanobind (see the comment at
410-
// the top of nb_combined.cpp); harmless for the generated .cpp too.
411-
"-DNDEBUG", "-DNB_COMPACT_ASSERTIONS", "-fno-strict-aliasing",
412-
}
413-
return buildCXXModule(cfg, buildname, pycfg, flags, []string{filepath.Join(srcdir, "nb_combined.cpp")})
406+
flags := append([]string{"-I" + incdir, "-I" + robinmap}, bind.NanobindCXXFlags...)
407+
return buildCXXModule(cfg, pycfg, flags, []string{filepath.Join(srcdir, "nb_combined.cpp")})
414408
}
415409

416410
// buildCXXModule builds the cgo shim as a static archive, runs build.py to
417411
// write a C++ module wrapping it (pybind11 or nanobind), and compiles+links
418-
// that with a C++ compiler, passing it cxxflags (the C++ library's include
412+
// that with a C++ compiler (see bind.CXXArgs, and bind/cxxbuild.go for why
413+
// a static archive), passing it cxxflags (the C++ library's include
419414
// directories, and any flags of its own) and, besides the generated .cpp,
420415
// the C++ library's own sources, if any. The current directory is the
421-
// output directory.
422-
//
423-
// Unlike cffi (buildCFFI), the wrapper Go generates (pybind11_callback.go)
424-
// has Go call INTO the wrapper's own C++ code (the per-callback-shape
425-
// trampolines) as well as the other way around. Two separately-built
426-
// shared libraries can't have a dependency cycle like that -- neither can
427-
// exist as a complete, loadable file before the other -- so instead of a
428-
// shared library (buildCFFI's buildLib), the Go side here builds as a
429-
// static archive (-buildmode=c-archive), with its symbols left unresolved
430-
// until the single final link below, alongside the C++ object code that
431-
// defines them.
432-
func buildCXXModule(cfg *BuildCfg, buildname string, pycfg bind.PyConfig, cxxflags, srcs []string) error {
433-
archive := buildname + ".a"
416+
// output directory. The Makefile gopy gen writes for these backends runs
417+
// the same steps.
418+
func buildCXXModule(cfg *BuildCfg, pycfg bind.PyConfig, cxxflags, srcs []string) error {
434419
args := []string{"build", "-mod=mod", "-buildmode=c-archive"}
435420
if cfg.BuildTags != "" {
436421
args = append(args, "-tags", cfg.BuildTags)
437422
}
438423
if !cfg.Symbols {
439424
args = append(args, "-ldflags=-s -w")
440425
}
441-
args = append(args, "-o", archive, ".")
426+
args = append(args, "-o", bind.CXXArchive(cfg.Name), ".")
442427
fmt.Printf("go %v\n", strings.Join(args, " "))
443428
cmdout, err := exec.Command("go", args...).CombinedOutput()
444429
if err != nil {
@@ -453,85 +438,8 @@ func buildCXXModule(cfg *BuildCfg, buildname string, pycfg bind.PyConfig, cxxfla
453438
return err
454439
}
455440

456-
extext := libExt
457-
if runtime.GOOS == "windows" {
458-
extext = ".pyd"
459-
}
460-
if pycfg.ExtSuffix != "" {
461-
extext = pycfg.ExtSuffix
462-
}
463-
modlib := "_" + cfg.Name + extext
464-
465-
cxx := os.Getenv("CXX")
466-
if cxx == "" {
467-
cxx = "c++"
468-
}
469-
// pycfg.CFlags/LdFlags quote each path (for the shell that CGO_CFLAGS/
470-
// CGO_LDFLAGS normally go through); exec.Command runs the compiler
471-
// directly, with no shell to strip those, so unquote each field here.
472-
unquote := func(fields []string) []string {
473-
o := make([]string, len(fields))
474-
for i, f := range fields {
475-
o[i] = strings.Trim(f, `"`)
476-
}
477-
return o
478-
}
479-
// modlib depends on libpython (wherever this VM's own one lives, e.g.
480-
// not on the loader's default search path for a uv- or pyenv-managed
481-
// Python); without an rpath, the loader only finds it if it happens to
482-
// already be on its search path.
483-
var libdir string
484-
if m := regexp.MustCompile(`-L(\S+)`).FindStringSubmatch(pycfg.LdFlags); m != nil {
485-
libdir = strings.Trim(m[1], `"`)
486-
}
487-
// The archive's Go runtime code calls into gopy_cb_N (defined below, in
488-
// the .cpp), so the linker must be told to keep every object in it --
489-
// left to its own judgement, it would see nothing in the .cpp calling
490-
// into the archive first and drop it as unused. GNU ld (Linux, and
491-
// Windows' MinGW) and ld64 (macOS) spell that differently.
492-
var archiveArgs []string
493-
if runtime.GOOS == "darwin" {
494-
archiveArgs = []string{"-Wl,-force_load," + archive}
495-
} else {
496-
archiveArgs = []string{"-Wl,--whole-archive", archive, "-Wl,--no-whole-archive"}
497-
}
498-
cxxArgs := []string{"-std=c++17", "-fPIC", "-shared", "-O2"}
499-
switch runtime.GOOS {
500-
case "darwin":
501-
cxxArgs = append(cxxArgs, "-Wl,-rpath,@loader_path")
502-
if libdir != "" {
503-
cxxArgs = append(cxxArgs, "-Wl,-rpath,"+libdir)
504-
}
505-
case "windows":
506-
// No rpath equivalent; modlib depends on nothing but libpython now
507-
// that the Go side is a static archive, not a separate DLL of its
508-
// own (see the buildCXXModule doc comment). MinGW's own runtime
509-
// (libstdc++/libgcc/libwinpthread), which g++ links dynamically by
510-
// default, has no such fix available -- it isn't found by name
511-
// alone unless its directory happens to be on PATH -- so link it in
512-
// statically instead. The C runtime (ucrt) stays dynamic, shared
513-
// with Python's own.
514-
cxxArgs = append(cxxArgs, "-static-libgcc", "-static-libstdc++",
515-
"-Wl,-Bstatic,--whole-archive", "-lwinpthread", "-Wl,--no-whole-archive", "-Wl,-Bdynamic")
516-
default:
517-
cxxArgs = append(cxxArgs, "-Wl,-rpath,$ORIGIN")
518-
if libdir != "" {
519-
cxxArgs = append(cxxArgs, "-Wl,-rpath,"+libdir)
520-
}
521-
}
522-
cxxArgs = append(cxxArgs, cxxflags...)
523-
cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.CFlags))...)
524-
cxxArgs = append(cxxArgs, cfg.Name+".cpp")
525-
cxxArgs = append(cxxArgs, srcs...)
526-
cxxArgs = append(cxxArgs, archiveArgs...)
527-
cxxArgs = append(cxxArgs, unquote(strings.Fields(pycfg.LdFlags))...)
528-
// c-archive mode (unlike c-shared) doesn't resolve the Go runtime's own
529-
// dependencies on these itself; TODO: verified only on Linux -- unclear
530-
// yet whether Windows/macOS need anything of their own added here too.
531-
if runtime.GOOS != "windows" {
532-
cxxArgs = append(cxxArgs, "-lpthread", "-ldl", "-lm")
533-
}
534-
cxxArgs = append(cxxArgs, "-o", modlib)
441+
cxx := bind.CXX()
442+
cxxArgs := bind.CXXArgs(cfg.Name, bind.ExtModuleName(cfg.Name, libExt, pycfg), pycfg, cxxflags, srcs)
535443
fmt.Printf("%v %v\n", cxx, strings.Join(cxxArgs, " "))
536444
cmdout, err = exec.Command(cxx, cxxArgs...).CombinedOutput()
537445
if err != nil {

0 commit comments

Comments
 (0)