all: include picolibc for bare metal targets

This is necessary for better CGo support on bare metal. Existing
libraries expect to be able to include parts of libc and expect to be
able to link to those symbols.

Because with this all targets have a working libc, it is now possible to
add tests to check that a libc in fact works basically.

Not all parts of picolibc are included, such as the math or stdio parts.
These should be added later, when needed.

This commit also avoids the need for the custom memcpy/memset/memcmp
symbols that are sometimes emitted by LLVM. The C library will take care
of that.

Author: aykevl

.gitmodules

@@ -17,3 +17,6 @@
 [submodule "lib/wasi-libc"]
 	path = lib/wasi-libc
 	url = https://github.com/CraneStation/wasi-libc
+[submodule "lib/picolibc"]
+	path = lib/picolibc
+	url = https://github.com/keith-packard/picolibc.git

Makefile

@@ -312,6 +312,7 @@ release: tinygo gen-device wasi-libc
 	@mkdir -p build/release/tinygo/lib/CMSIS/CMSIS
 	@mkdir -p build/release/tinygo/lib/compiler-rt/lib
 	@mkdir -p build/release/tinygo/lib/nrfx
+	@mkdir -p build/release/tinygo/lib/picolibc/newlib/libc
 	@mkdir -p build/release/tinygo/lib/wasi-libc
 	@mkdir -p build/release/tinygo/pkg/armv6m-none-eabi
 	@mkdir -p build/release/tinygo/pkg/armv7m-none-eabi
@@ -325,10 +326,19 @@ release: tinygo gen-device wasi-libc
 	@cp -rp lib/compiler-rt/LICENSE.TXT  build/release/tinygo/lib/compiler-rt
 	@cp -rp lib/compiler-rt/README.txt   build/release/tinygo/lib/compiler-rt
 	@cp -rp lib/nrfx/*                   build/release/tinygo/lib/nrfx
+	@cp -rp lib/picolibc/newlib/libc/ctype       build/release/tinygo/lib/picolibc/newlib/libc
+	@cp -rp lib/picolibc/newlib/libc/include     build/release/tinygo/lib/picolibc/newlib/libc
+	@cp -rp lib/picolibc/newlib/libc/locale      build/release/tinygo/lib/picolibc/newlib/libc
+	@cp -rp lib/picolibc/newlib/libc/string      build/release/tinygo/lib/picolibc/newlib/libc
+	@cp -rp lib/picolibc/newlib/libc/tinystdio   build/release/tinygo/lib/picolibc/newlib/libc
+	@cp -rp lib/picolibc-include         build/release/tinygo/lib
 	@cp -rp lib/wasi-libc/sysroot        build/release/tinygo/lib/wasi-libc/sysroot
 	@cp -rp src                          build/release/tinygo/src
 	@cp -rp targets                      build/release/tinygo/targets
-	./build/tinygo build-builtins -target=armv6m-none-eabi  -o build/release/tinygo/pkg/armv6m-none-eabi/compiler-rt.a
-	./build/tinygo build-builtins -target=armv7m-none-eabi  -o build/release/tinygo/pkg/armv7m-none-eabi/compiler-rt.a
-	./build/tinygo build-builtins -target=armv7em-none-eabi -o build/release/tinygo/pkg/armv7em-none-eabi/compiler-rt.a
+	./build/tinygo build-library -target=armv6m-none-eabi  -o build/release/tinygo/pkg/armv6m-none-eabi/compiler-rt.a compiler-rt
+	./build/tinygo build-library -target=armv7m-none-eabi  -o build/release/tinygo/pkg/armv7m-none-eabi/compiler-rt.a compiler-rt
+	./build/tinygo build-library -target=armv7em-none-eabi -o build/release/tinygo/pkg/armv7em-none-eabi/compiler-rt.a compiler-rt
+	./build/tinygo build-library -target=armv6m-none-eabi  -o build/release/tinygo/pkg/armv6m-none-eabi/picolibc.a picolibc
+	./build/tinygo build-library -target=armv7m-none-eabi  -o build/release/tinygo/pkg/armv7m-none-eabi/picolibc.a picolibc
+	./build/tinygo build-library -target=armv7em-none-eabi -o build/release/tinygo/pkg/armv7em-none-eabi/picolibc.a picolibc
 	tar -czf build/release.tar.gz -C build/release tinygo

builder/build.go

@@ -145,6 +145,15 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(stri
 			ldflags = append(ldflags, librt)
 		}
 
+		// Add libc.
+		if config.Target.Libc == "picolibc" {
+			libc, err := Picolibc.Load(config.Triple())
+			if err != nil {
+				return err
+			}
+			ldflags = append(ldflags, libc)
+		}
+
 		// Compile extra files.
 		root := goenv.Get("TINYGOROOT")
 		for i, path := range config.ExtraFiles() {

builder/picolibc.go

@@ -0,0 +1,127 @@
+package builder
+
+import (
+	"path/filepath"
+
+	"github.com/tinygo-org/tinygo/goenv"
+)
+
+// Picolibc is a C library for bare metal embedded devices. It was originally
+// based on newlib.
+var Picolibc = Library{
+	name: "picolibc",
+	cflags: func() []string {
+		picolibcDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib/picolibc/newlib/libc")
+		return []string{"-Werror", "-Wall", "-std=gnu11", "-D_COMPILING_NEWLIB", "-fshort-enums", "--sysroot=" + picolibcDir, "-I" + picolibcDir + "/tinystdio", "-I" + goenv.Get("TINYGOROOT") + "/lib/picolibc-include"}
+	},
+	sourceDir: "lib/picolibc/newlib/libc",
+	sources: func(target string) []string {
+		return picolibcSources
+	},
+}
+
+var picolibcSources = []string{
+	"string/bcmp.c",
+	"string/bcopy.c",
+	"string/bzero.c",
+	"string/explicit_bzero.c",
+	"string/ffsl.c",
+	"string/ffsll.c",
+	"string/fls.c",
+	"string/flsl.c",
+	"string/flsll.c",
+	"string/gnu_basename.c",
+	"string/index.c",
+	"string/memccpy.c",
+	"string/memchr.c",
+	"string/memcmp.c",
+	"string/memcpy.c",
+	"string/memmem.c",
+	"string/memmove.c",
+	"string/mempcpy.c",
+	"string/memrchr.c",
+	"string/memset.c",
+	"string/rawmemchr.c",
+	"string/rindex.c",
+	"string/stpcpy.c",
+	"string/stpncpy.c",
+	"string/strcasecmp.c",
+	"string/strcasecmp_l.c",
+	"string/strcasestr.c",
+	"string/strcat.c",
+	"string/strchr.c",
+	"string/strchrnul.c",
+	"string/strcmp.c",
+	"string/strcoll.c",
+	"string/strcoll_l.c",
+	"string/strcpy.c",
+	"string/strcspn.c",
+	"string/strdup.c",
+	"string/strerror.c",
+	"string/strerror_r.c",
+	"string/strlcat.c",
+	"string/strlcpy.c",
+	"string/strlen.c",
+	"string/strlwr.c",
+	"string/strncasecmp.c",
+	"string/strncasecmp_l.c",
+	"string/strncat.c",
+	"string/strncmp.c",
+	"string/strncpy.c",
+	"string/strndup.c",
+	"string/strnlen.c",
+	"string/strnstr.c",
+	"string/strpbrk.c",
+	"string/strrchr.c",
+	"string/strsep.c",
+	"string/strsignal.c",
+	"string/strspn.c",
+	"string/strstr.c",
+	"string/strtok.c",
+	"string/strtok_r.c",
+	"string/strupr.c",
+	"string/strverscmp.c",
+	"string/strxfrm.c",
+	"string/strxfrm_l.c",
+	"string/swab.c",
+	"string/timingsafe_bcmp.c",
+	"string/timingsafe_memcmp.c",
+	"string/u_strerr.c",
+	"string/wcpcpy.c",
+	"string/wcpncpy.c",
+	"string/wcscasecmp.c",
+	"string/wcscasecmp_l.c",
+	"string/wcscat.c",
+	"string/wcschr.c",
+	"string/wcscmp.c",
+	"string/wcscoll.c",
+	"string/wcscoll_l.c",
+	"string/wcscpy.c",
+	"string/wcscspn.c",
+	"string/wcsdup.c",
+	"string/wcslcat.c",
+	"string/wcslcpy.c",
+	"string/wcslen.c",
+	"string/wcsncasecmp.c",
+	"string/wcsncasecmp_l.c",
+	"string/wcsncat.c",
+	"string/wcsncmp.c",
+	"string/wcsncpy.c",
+	"string/wcsnlen.c",
+	"string/wcspbrk.c",
+	"string/wcsrchr.c",
+	"string/wcsspn.c",
+	"string/wcsstr.c",
+	"string/wcstok.c",
+	"string/wcswidth.c",
+	"string/wcsxfrm.c",
+	"string/wcsxfrm_l.c",
+	"string/wcwidth.c",
+	"string/wmemchr.c",
+	"string/wmemcmp.c",
+	"string/wmemcpy.c",
+	"string/wmemmove.c",
+	"string/wmempcpy.c",
+	"string/wmemset.c",
+	"string/xpg_strerror_r.c",
+}

cgo/cgo.go

@@ -170,6 +170,11 @@ func Process(files []*ast.File, dir string, fset *token.FileSet, cflags []string
 		enums:           map[string]enumInfo{},
 	}
 
+	// Disable _FORTIFY_SOURCE as it causes problems on macOS.
+	// Note that it is only disabled for memcpy (etc) calls made from Go, which
+	// have better alternatives anyway.
+	cflags = append(cflags, "-D_FORTIFY_SOURCE=0")
+
 	// Add a new location for the following file.
 	generatedTokenPos := p.fset.AddFile(dir+"/!cgo.go", -1, 0)
 	generatedTokenPos.SetLines([]int{0})

compileopts/config.go

@@ -5,6 +5,7 @@ package compileopts
 import (
 	"errors"
 	"fmt"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -163,6 +164,11 @@ func (c *Config) CFlags() []string {
 	for _, flag := range c.Target.CFlags {
 		cflags = append(cflags, strings.Replace(flag, "{root}", goenv.Get("TINYGOROOT"), -1))
 	}
+	if c.Target.Libc == "picolibc" {
+		root := goenv.Get("TINYGOROOT")
+		cflags = append(cflags, "--sysroot="+filepath.Join(root, "lib", "picolibc", "newlib", "libc"))
+		cflags = append(cflags, "-I"+filepath.Join(root, "lib/picolibc-include"))
+	}
 	return cflags
 }
 

compileopts/target.go

@@ -32,6 +32,7 @@ type TargetSpec struct {
 	Compiler         string   `json:"compiler"`
 	Linker           string   `json:"linker"`
 	RTLib            string   `json:"rtlib"` // compiler runtime library (libgcc, compiler-rt)
+	Libc             string   `json:"libc"`
 	CFlags           []string `json:"cflags"`
 	LDFlags          []string `json:"ldflags"`
 	LinkerScript     string   `json:"linkerscript"`
@@ -84,6 +85,9 @@ func (spec *TargetSpec) copyProperties(spec2 *TargetSpec) {
 	if spec2.RTLib != "" {
 		spec.RTLib = spec2.RTLib
 	}
+	if spec2.Libc != "" {
+		spec.Libc = spec2.Libc
+	}
 	spec.CFlags = append(spec.CFlags, spec2.CFlags...)
 	spec.LDFlags = append(spec.LDFlags, spec2.LDFlags...)
 	if spec2.LinkerScript != "" {

lib/picolibc

@@ -786,7 +786,7 @@ func main() {
 		}
 		err := Build(pkgName, *outpath, options)
 		handleCompilerError(err)
-	case "build-builtins":
+	case "build-library":
 		// Note: this command is only meant to be used while making a release!
 		if *outpath == "" {
 			fmt.Fprintln(os.Stderr, "No output filename supplied (-o).")
@@ -796,7 +796,22 @@ func main() {
 		if *target == "" {
 			fmt.Fprintln(os.Stderr, "No target (-target).")
 		}
-		path, err := builder.CompilerRT.Load(*target)
+		if flag.NArg() != 1 {
+			fmt.Fprintf(os.Stderr, "Build-library only accepts exactly one library name as argument, %d given\n", flag.NArg())
+			usage()
+			os.Exit(1)
+		}
+		var lib *builder.Library
+		switch name := flag.Arg(0); name {
+		case "compiler-rt":
+			lib = &builder.CompilerRT
+		case "picolibc":
+			lib = &builder.Picolibc
+		default:
+			fmt.Fprintf(os.Stderr, "Unknown library: %s\n", name)
+			os.Exit(1)
+		}
+		path, err := lib.Load(*target)
 		handleCompilerError(err)
 		copyFile(path, *outpath)
 	case "flash", "gdb":