Skip to content

Commit 632357f

Browse files
aykevldeadprogram
aykevl
authored and
deadprogram
committed
builder: build wasi-libc inside TinyGo
Instead of relying on a build when TinyGo is being built, do it like all other libraries when it is needed. This brings a few benefits: * No more running `make wasi-libc` on the command line as a special case for WebAssembly. The generic `git submodule update --init` step is now enough for wasi-libc. * It becomes much easier to customize the build per system. For example: include/exclude malloc as needed, disable/enable bulk memory operations per target, etc.
1 parent 4c54aa2 commit 632357f

File tree

11 files changed

+362
-105
lines changed

11 files changed

+362
-105
lines changed

‎.circleci/config.yml

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,6 @@ commands:
7373
- go-cache-v4-{{ checksum "go.mod" }}
7474
- llvm-source-linux
7575
- run: go install -tags=llvm<<parameters.llvm>> .
76-
- restore_cache:
77-
keys:
78-
- wasi-libc-sysroot-systemclang-v7
79-
- run: make wasi-libc
80-
- save_cache:
81-
key: wasi-libc-sysroot-systemclang-v7
82-
paths:
83-
- lib/wasi-libc/sysroot
8476
- when:
8577
condition: <<parameters.fmt-check>>
8678
steps:

‎.github/workflows/build-macos.yml

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -89,15 +89,6 @@ jobs:
8989
with:
9090
key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }}
9191
path: llvm-build
92-
- name: Cache wasi-libc sysroot
93-
uses: actions/cache@v4
94-
id: cache-wasi-libc
95-
with:
96-
key: wasi-libc-sysroot-${{ matrix.os }}-v1
97-
path: lib/wasi-libc/sysroot
98-
- name: Build wasi-libc
99-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
100-
run: make wasi-libc
10192
- name: make gen-device
10293
run: make -j3 gen-device
10394
- name: Test TinyGo

‎.github/workflows/linux.yml

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -104,15 +104,6 @@ jobs:
104104
run: |
105105
apk add cmake samurai python3
106106
make binaryen STATIC=1
107-
- name: Cache wasi-libc
108-
uses: actions/cache@v4
109-
id: cache-wasi-libc
110-
with:
111-
key: wasi-libc-sysroot-linux-alpine-v2
112-
path: lib/wasi-libc/sysroot
113-
- name: Build wasi-libc
114-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
115-
run: make wasi-libc
116107
- name: Install fpm
117108
run: |
118109
gem install --version 4.0.7 public_suffix
@@ -258,15 +249,6 @@ jobs:
258249
- name: Build Binaryen
259250
if: steps.cache-binaryen.outputs.cache-hit != 'true'
260251
run: make binaryen
261-
- name: Cache wasi-libc
262-
uses: actions/cache@v4
263-
id: cache-wasi-libc
264-
with:
265-
key: wasi-libc-sysroot-linux-asserts-v6
266-
path: lib/wasi-libc/sysroot
267-
- name: Build wasi-libc
268-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
269-
run: make wasi-libc
270252
- run: make gen-device -j4
271253
- name: Test TinyGo
272254
run: make ASSERT=1 test

‎.github/workflows/windows.yml

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -91,15 +91,6 @@ jobs:
9191
with:
9292
key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }}
9393
path: llvm-build
94-
- name: Cache wasi-libc sysroot
95-
uses: actions/cache@v4
96-
id: cache-wasi-libc
97-
with:
98-
key: wasi-libc-sysroot-v5
99-
path: lib/wasi-libc/sysroot
100-
- name: Build wasi-libc
101-
if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
102-
run: make wasi-libc
10394
- name: Cache Go cache
10495
uses: actions/cache@v4
10596
with:

‎GNUmakefile

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -261,13 +261,6 @@ build/wasm-opt$(EXE):
261261
cp lib/binaryen/bin/wasm-opt$(EXE) build/wasm-opt$(EXE)
262262
endif
263263

264-
# Build wasi-libc sysroot
265-
.PHONY: wasi-libc
266-
wasi-libc: lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a
267-
lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a:
268-
@if [ ! -e lib/wasi-libc/Makefile ]; then echo "Submodules have not been downloaded. Please download them using:\n git submodule update --init"; exit 1; fi
269-
cd lib/wasi-libc && $(MAKE) -j4 EXTRA_CFLAGS="-O2 -g -DNDEBUG -mnontrapping-fptoint -msign-ext" MALLOC_IMPL=none CC="$(CLANG)" AR=$(LLVM_AR) NM=$(LLVM_NM)
270-
271264
# Generate WASI syscall bindings
272265
WASM_TOOLS_MODULE=go.bytecodealliance.org
273266
.PHONY: wasi-syscall
@@ -295,7 +288,7 @@ check-nodejs-version:
295288
tinygo: ## Build the TinyGo compiler
296289
@if [ ! -f "$(LLVM_BUILDDIR)/bin/llvm-config" ]; then echo "Fetch and build LLVM first by running:"; echo " $(MAKE) llvm-source"; echo " $(MAKE) $(LLVM_BUILDDIR)"; exit 1; fi
297290
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOENVFLAGS) $(GO) build -buildmode exe -o build/tinygo$(EXE) -tags "byollvm osusergo" .
298-
test: wasi-libc check-nodejs-version
291+
test: check-nodejs-version
299292
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags "byollvm osusergo" $(GOTESTPKGS)
300293

301294
# Standard library packages that pass tests on darwin, linux, wasi, and windows, but take over a minute in wasi
@@ -528,9 +521,9 @@ test-corpus:
528521
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml
529522
test-corpus-fast:
530523
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus -short . -corpus=testdata/corpus.yaml
531-
test-corpus-wasi: wasi-libc
524+
test-corpus-wasi:
532525
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip1
533-
test-corpus-wasip2: wasi-libc
526+
test-corpus-wasip2:
534527
CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip2
535528

536529
.PHONY: testchdir
@@ -941,7 +934,7 @@ endif
941934
wasmtest:
942935
$(GO) test ./tests/wasm
943936

944-
build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen)
937+
build/release: tinygo gen-device $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen)
945938
@mkdir -p build/release/tinygo/bin
946939
@mkdir -p build/release/tinygo/lib/bdwgc
947940
@mkdir -p build/release/tinygo/lib/clang/include
@@ -956,7 +949,7 @@ build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN
956949
@mkdir -p build/release/tinygo/lib/nrfx
957950
@mkdir -p build/release/tinygo/lib/picolibc/newlib/libc
958951
@mkdir -p build/release/tinygo/lib/picolibc/newlib/libm
959-
@mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers
952+
@mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half
960953
@mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
961954
@mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
962955
@mkdir -p build/release/tinygo/lib/wasi-cli/
@@ -1019,15 +1012,36 @@ endif
10191012
@cp -rp lib/picolibc/newlib/libm/common build/release/tinygo/lib/picolibc/newlib/libm
10201013
@cp -rp lib/picolibc/newlib/libm/math build/release/tinygo/lib/picolibc/newlib/libm
10211014
@cp -rp lib/picolibc-stdio.c build/release/tinygo/lib
1022-
@cp -rp lib/wasi-libc/libc-bottom-half/headers/public build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers
1015+
@cp -rp lib/wasi-libc/libc-bottom-half/cloudlibc build/release/tinygo/lib/wasi-libc/libc-bottom-half
1016+
@cp -rp lib/wasi-libc/libc-bottom-half/headers build/release/tinygo/lib/wasi-libc/libc-bottom-half
1017+
@cp -rp lib/wasi-libc/libc-bottom-half/sources build/release/tinygo/lib/wasi-libc/libc-bottom-half
1018+
@cp -rp lib/wasi-libc/libc-top-half/headers build/release/tinygo/lib/wasi-libc/libc-top-half
10231019
@cp -rp lib/wasi-libc/libc-top-half/musl/arch/generic build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
10241020
@cp -rp lib/wasi-libc/libc-top-half/musl/arch/wasm32 build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch
1021+
@cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl
1022+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/conf build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1023+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/dirent build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1024+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/env build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1025+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/errno build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1026+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/exit build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1027+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/fcntl build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1028+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/fenv build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10251029
@cp -rp lib/wasi-libc/libc-top-half/musl/src/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10261030
@cp -rp lib/wasi-libc/libc-top-half/musl/src/internal build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1031+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/legacy build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1032+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/locale build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10271033
@cp -rp lib/wasi-libc/libc-top-half/musl/src/math build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1034+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/misc build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1035+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/multibyte build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1036+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/network build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1037+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/stat build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1038+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/stdio build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1039+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/stdlib build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
10281040
@cp -rp lib/wasi-libc/libc-top-half/musl/src/string build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1029-
@cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl
1030-
@cp -rp lib/wasi-libc/sysroot build/release/tinygo/lib/wasi-libc/sysroot
1041+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/thread build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1042+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/time build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1043+
@cp -rp lib/wasi-libc/libc-top-half/musl/src/unistd build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src
1044+
@cp -rp lib/wasi-libc/libc-top-half/sources build/release/tinygo/lib/wasi-libc/libc-top-half
10311045
@cp -rp lib/wasi-cli/wit build/release/tinygo/lib/wasi-cli/wit
10321046
@cp -rp llvm-project/compiler-rt/lib/builtins build/release/tinygo/lib/compiler-rt-builtins
10331047
@cp -rp llvm-project/compiler-rt/LICENSE.TXT build/release/tinygo/lib/compiler-rt-builtins

‎builder/build.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"fmt"
1515
"go/types"
1616
"hash/crc32"
17-
"io/fs"
1817
"math/bits"
1918
"os"
2019
"os/exec"
@@ -168,11 +167,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe
168167
defer unlock()
169168
libcDependencies = append(libcDependencies, libcJob)
170169
case "wasi-libc":
171-
path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a")
172-
if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) {
173-
return BuildResult{}, errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?")
170+
libcJob, unlock, err := libWasiLibc.load(config, tmpdir)
171+
if err != nil {
172+
return BuildResult{}, err
174173
}
175-
libcDependencies = append(libcDependencies, dummyCompileJob(path))
174+
defer unlock()
175+
libcDependencies = append(libcDependencies, libcJob)
176176
case "wasmbuiltins":
177177
libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir)
178178
if err != nil {

‎builder/library.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ type Library struct {
2525
// cflags returns the C flags specific to this library
2626
cflags func(target, headerPath string) []string
2727

28+
// cflagsForFile returns additional C flags for a particular source file.
29+
cflagsForFile func(path string) []string
30+
2831
// needsLibc is set to true if this library needs libc headers.
2932
needsLibc bool
3033

@@ -226,6 +229,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
226229
}
227230
for _, path := range paths {
228231
// Strip leading "../" parts off the path.
232+
path := path
229233
cleanpath := path
230234
for strings.HasPrefix(cleanpath, "../") {
231235
cleanpath = cleanpath[3:]
@@ -239,6 +243,9 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
239243
run: func(*compileJob) error {
240244
var compileArgs []string
241245
compileArgs = append(compileArgs, args...)
246+
if l.cflagsForFile != nil {
247+
compileArgs = append(compileArgs, l.cflagsForFile(path)...)
248+
}
242249
compileArgs = append(compileArgs, "-o", objpath, srcpath)
243250
if config.Options.PrintCommands != nil {
244251
config.Options.PrintCommands("clang", compileArgs...)

‎builder/musl.go

Lines changed: 41 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,45 @@ import (
1212
"github.com/tinygo-org/tinygo/goenv"
1313
)
1414

15+
// Create the alltypes.h file from the appropriate alltypes.h.in files.
16+
func buildMuslAllTypes(arch, muslDir, outputBitsDir string) error {
17+
// Create the file alltypes.h.
18+
f, err := os.Create(filepath.Join(outputBitsDir, "alltypes.h"))
19+
if err != nil {
20+
return err
21+
}
22+
infiles := []string{
23+
filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"),
24+
filepath.Join(muslDir, "include", "alltypes.h.in"),
25+
}
26+
for _, infile := range infiles {
27+
data, err := os.ReadFile(infile)
28+
if err != nil {
29+
return err
30+
}
31+
lines := strings.Split(string(data), "\n")
32+
for _, line := range lines {
33+
if strings.HasPrefix(line, "TYPEDEF ") {
34+
matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line)
35+
value := matches[1]
36+
name := matches[2]
37+
line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name)
38+
}
39+
if strings.HasPrefix(line, "STRUCT ") {
40+
matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line)
41+
name := matches[1]
42+
value := matches[2]
43+
line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name)
44+
}
45+
_, err := f.WriteString(line + "\n")
46+
if err != nil {
47+
return err
48+
}
49+
}
50+
}
51+
return f.Close()
52+
}
53+
1554
var libMusl = Library{
1655
name: "musl",
1756
makeHeaders: func(target, includeDir string) error {
@@ -24,41 +63,13 @@ var libMusl = Library{
2463
arch := compileopts.MuslArchitecture(target)
2564
muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "musl")
2665

27-
// Create the file alltypes.h.
28-
f, err := os.Create(filepath.Join(bits, "alltypes.h"))
66+
err = buildMuslAllTypes(arch, muslDir, bits)
2967
if err != nil {
3068
return err
3169
}
32-
infiles := []string{
33-
filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"),
34-
filepath.Join(muslDir, "include", "alltypes.h.in"),
35-
}
36-
for _, infile := range infiles {
37-
data, err := os.ReadFile(infile)
38-
if err != nil {
39-
return err
40-
}
41-
lines := strings.Split(string(data), "\n")
42-
for _, line := range lines {
43-
if strings.HasPrefix(line, "TYPEDEF ") {
44-
matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line)
45-
value := matches[1]
46-
name := matches[2]
47-
line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name)
48-
}
49-
if strings.HasPrefix(line, "STRUCT ") {
50-
matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line)
51-
name := matches[1]
52-
value := matches[2]
53-
line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name)
54-
}
55-
f.WriteString(line + "\n")
56-
}
57-
}
58-
f.Close()
5970

6071
// Create the file syscall.h.
61-
f, err = os.Create(filepath.Join(bits, "syscall.h"))
72+
f, err := os.Create(filepath.Join(bits, "syscall.h"))
6273
if err != nil {
6374
return err
6475
}

0 commit comments

Comments
 (0)