build: switch c scheds to make

Currently we have two build systems: Cargo and Meson. Meson builds the C
schedulers and is the only way, but can also drive Cargo providing a
bunch of extra parameters. Meson also provides the framework for our
current CI stress testing.

Add a Makefile based system for building the C schedulers. We lose a lot
of knobs in this migration, but that's deliberate - the very large
matrix of Meson options was really confusing, and led to weird edge
cases that we'd often miss in CI and with local testing.

Also add the builds for the C schedulers to the CI for the first time.
No real testing other than "it builds" yet.

This change does not remove the Meson builds, but that is the direction.
Letting them run in parallel for a while as the CI currently still
relies on Meson and we should iron out any quirks with the build first.
For example, we may need to vendor libbpf in Make like it was in Meson,
but let's work that out in tree. All documentation is updated to remove
references to Meson according to this plan.

Implementation notes:
- Only allows out of source builds. In source builds annoy me, and I'd
  rather we didn't maintain both types. Follows the `O=build` style of
  Linux but defaults to `build` if absent.
- Uses `pkg-config` to find `libbpf`, making it easy to override the
  path if needed. We may still need to vendor libbpf after we find out
  how it does on distros.
- Only supports calling `make` from the root. I tried to make it easier
  to call `cd scheds/c; make` but had a lot of trouble with `lib` not
  being nested under it.
- Supports environment variables for the other tools, and these largely
  line up with our Cargo based build system. Aspirationally we should
  keep them matching wherever it makes sense.

Test plan:
- CI

```
jake@rooster:/data/users/jake/repos/scx/ > make install INSTALL_DIR=/tmp/nix-shell.lnmzMH/tmp.zBN35egECj
make[2]: Nothing to be done for 'all'.
make[2]: Nothing to be done for 'all'.
make -C /data/users/jake/repos/scx/scheds/c install
Installing schedulers to /tmp/nix-shell.lnmzMH/tmp.zBN35egECj
Installing /data/users/jake/repos/scx/build/scheds/c/scx_simple
Installing /data/users/jake/repos/scx/build/scheds/c/scx_qmap
Installing /data/users/jake/repos/scx/build/scheds/c/scx_central
Installing /data/users/jake/repos/scx/build/scheds/c/scx_userland
Installing /data/users/jake/repos/scx/build/scheds/c/scx_nest
Installing /data/users/jake/repos/scx/build/scheds/c/scx_flatcg
Installing /data/users/jake/repos/scx/build/scheds/c/scx_pair
Installing /data/users/jake/repos/scx/build/scheds/c/scx_prev
Installing /data/users/jake/repos/scx/build/scheds/c/scx_sdt
jake@rooster:/data/users/jake/repos/scx/ > ls /tmp/nix-shell.lnmzMH/tmp.zBN35egECj
scx_central  scx_flatcg  scx_nest  scx_pair  scx_prev  scx_qmap  scx_sdt  scx_simple  scx_userland
jake@rooster:/data/users/jake/repos/scx/ > sudo /tmp/nix-shell.lnmzMH/tmp.zBN35egECj/scx_central
[SEQ 0]
total   :        19    local:         0   queued:         1  lost:         0
timer   :        22 dispatch:        34 mismatch:        13 retry:         0
overflow:         0
[SEQ 1]
total   :      2285    local:        18   queued:         0  lost:         0
timer   :       995 dispatch:      6908 mismatch:        25 retry:         0
overflow:         0
[SEQ 2]
total   :      2981    local:        61   queued:         0  lost:         0
timer   :      1964 dispatch:      9048 mismatch:        29 retry:         0
overflow:         0
^CEXIT: unregistered from user space
```

Author: JakeHillion

.github/include/ci.py

@@ -227,7 +227,11 @@ async def run_build():
     """Build all targets."""
     print("Running build...", flush=True)
 
+    print("Building C schedulers...", flush=True)
+    await run_command(["make", "all"], no_capture=True)
+    print("Building Rust schedulers...", flush=True)
     await run_command(["cargo", "build", "--all-targets", "--locked"], no_capture=True)
+
     print("✓ Build completed successfully", flush=True)
 
 

.github/include/flake.nix

@@ -94,6 +94,7 @@
               buildInputs = with pkgs; [
                 bash
                 binutils
+                bpftools
                 clang
                 coreutils
                 elfutils
@@ -102,6 +103,7 @@
                 glibc
                 gnumake
                 jq
+                libbpf-git
                 libseccomp
                 pkg-config
                 protobuf
@@ -170,6 +172,7 @@
                 bash
                 binutils
                 black
+                bpftools
                 clang
                 coreutils
                 gcc
@@ -179,6 +182,7 @@
                 gnused
                 isort
                 jq
+                libbpf-git
                 libseccomp.lib
                 llvmPackages.libclang
                 llvmPackages.libllvm

.gitignore

@@ -2,6 +2,7 @@ libbpf/
 *.gitignore
 PKGBUILD
 target
+build/
 *.swp
 .cache/
 .vscode/

INSTALL.md

@@ -38,22 +38,23 @@ $ sudo reboot
 #### Setting up Dev Environment
 
 ```
-$ sudo apt install -y build-essential meson cmake cargo rustc clang llvm pkg-config libelf-dev protobuf-compiler libseccomp-dev
+$ sudo apt install -y build-essential cmake cargo rustc clang llvm pkg-config libelf-dev protobuf-compiler libseccomp-dev
 ```
 
 #### Build the scx schedulers from source
 
 ```
 $ git clone https://github.com/sched-ext/scx.git
 $ cd scx
-$ meson setup build
-$ meson compile -C build
+$ make all                    # Build C schedulers
+$ cargo build --release       # Build Rust schedulers
 ```
 
 #### Install the scx schedulers from source
 
 ```
-$ meson install -C build
+$ make install INSTALL_DIR=~/bin                                         # Install C schedulers
+$ ls -d scheds/rust/scx_* | xargs -I{} cargo install --path {}          # Install Rust schedulers
 ```
 
 ## Arch Linux
@@ -67,7 +68,7 @@ sudo pacman -S scx-scheds
 In addition to the packages from the previous step, install the following.
 
 ```
-$ sudo pacman -Sy meson cargo bpf pahole
+$ sudo pacman -Sy cargo bpf pahole
 ```
 
 ## Gentoo Linux

Makefile

@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
+
+# Always require out-of-source builds - default to O=build if not specified
+ifeq ("$(origin O)", "command line")
+  KBUILD_OUTPUT := $(abspath $(O))
+else
+  KBUILD_OUTPUT := $(CURDIR)/build
+endif
+
+# Always redirect to out-of-tree build directory unless we're already there
+ifneq ($(KBUILD_OUTPUT), $(CURDIR))
+# Only redirect if ROOT_SRC_DIR is not set (i.e., we're not already redirected)
+ifeq ($(ROOT_SRC_DIR),)
+
+PHONY += _all $(MAKECMDGOALS) sub-make
+
+$(filter-out _all sub-make,$(MAKECMDGOALS)) _all: sub-make
+	@:
+
+sub-make:
+	@mkdir -p $(KBUILD_OUTPUT)
+	@$(MAKE) --no-print-directory -C $(KBUILD_OUTPUT) -f $(CURDIR)/Makefile \
+		ROOT_SRC_DIR=$(CURDIR) \
+		LIB_OBJ_DIR=$(KBUILD_OUTPUT)/lib SCHED_OBJ_DIR=$(KBUILD_OUTPUT)/scheds/c \
+		$(MAKECMDGOALS)
+
+.PHONY: $(PHONY)
+
+# Skip the rest of the makefile when redirecting
+skip-makefile := 1
+endif
+endif
+
+ifeq ($(skip-makefile),)
+
+export BPF_CLANG ?= clang
+export BPFTOOL ?= bpftool
+
+ARCH := $(shell uname -m)
+ifeq ($(ARCH),x86_64)
+	TARGET_ARCH := x86
+else ifeq ($(ARCH),aarch64)
+	TARGET_ARCH := arm64
+else ifeq ($(ARCH),s390x)
+	TARGET_ARCH := s390
+else
+	TARGET_ARCH := $(ARCH)
+endif
+
+ENDIAN ?= $(shell printf '\1\0' | od -An -t x2 | awk '{print ($$1=="0001"?"little":"big")}')
+
+LIBBPF_CFLAGS := $(shell pkg-config --cflags libbpf)
+LIBBPF_LIBS := $(shell pkg-config --libs libbpf)
+
+export BPF_CFLAGS := -g -O2 -Wall -Wno-compare-distinct-pointer-types \
+	-D__TARGET_ARCH_$(TARGET_ARCH) -mcpu=v3 -m$(ENDIAN)-endian
+
+# ROOT_SRC_DIR is set by the top-level make call for out-of-source builds
+export ROOT_SRC_DIR ?= $(CURDIR)
+export BPF_INCLUDES := -I$(ROOT_SRC_DIR)/scheds/include \
+	-I$(ROOT_SRC_DIR)/scheds/include/arch/$(TARGET_ARCH) \
+	-I$(ROOT_SRC_DIR)/scheds/include/bpf-compat \
+	-I$(ROOT_SRC_DIR)/scheds/include/lib \
+	$(LIBBPF_CFLAGS)
+
+export CFLAGS := -std=gnu11 -I$(ROOT_SRC_DIR)/scheds/include -I$(SCHED_OBJ_DIR) $(LIBBPF_CFLAGS)
+
+export LIBBPF_DEPS := $(LIBBPF_LIBS) -lelf -lz -lzstd
+export THREAD_DEPS := -lpthread
+
+export OBJ_DIR := $(CURDIR)
+ifeq ($(LIB_OBJ_DIR),)
+  export LIB_OBJ_DIR := $(ROOT_SRC_DIR)/lib
+endif
+ifeq ($(SCHED_OBJ_DIR),)
+  export SCHED_OBJ_DIR := $(ROOT_SRC_DIR)/scheds/c
+endif
+
+# Scheduler lists for convenience targets
+C_SCHEDS := scx_simple scx_qmap scx_central scx_userland scx_nest scx_flatcg scx_pair scx_prev
+C_SCHEDS_LIB := scx_sdt
+
+all: lib scheds-c
+
+# Individual scheduler targets
+$(C_SCHEDS) $(C_SCHEDS_LIB): lib
+	@mkdir -p $(SCHED_OBJ_DIR)
+	@$(MAKE) -C $(ROOT_SRC_DIR)/scheds/c SRC_DIR=$(ROOT_SRC_DIR)/scheds/c $@
+
+$(LIB_OBJ_DIR) $(SCHED_OBJ_DIR):
+	@mkdir -p $@
+
+lib:
+	@mkdir -p $(LIB_OBJ_DIR)
+	@$(MAKE) -C $(ROOT_SRC_DIR)/lib SRC_DIR=$(ROOT_SRC_DIR)/lib
+
+scheds-c: lib
+	@mkdir -p $(SCHED_OBJ_DIR)
+	@$(MAKE) -C $(ROOT_SRC_DIR)/scheds/c SRC_DIR=$(ROOT_SRC_DIR)/scheds/c
+
+clean:
+	$(MAKE) -C $(ROOT_SRC_DIR)/lib clean
+	$(MAKE) -C $(ROOT_SRC_DIR)/scheds/c clean
+
+install: all
+	$(MAKE) -C $(ROOT_SRC_DIR)/scheds/c install
+
+.PHONY: all lib scheds-c clean install $(C_SCHEDS) $(C_SCHEDS_LIB)
+
+endif  # End of ifeq ($(skip-makefile),)