Add CI for running benchmarks

.github/workflows/benchmarks.yml

@@ -0,0 +1,36 @@
+name: benchmarks
+
+on: 
+  pull_request:
+    types:
+      - opened
+      - reopened
+  issue_comment:
+    types:
+      - created
+
+jobs:
+  run_benchmarks:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    # run  either when pull request is opened or when comment body (only on pr) is /run-bench
+    if: (github.event_name == 'pull_request') || ((github.event.issue.pull_request != null) && github.event.comment.body == '/run-bench')
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install hyperfine
+        run: cargo install hyperfine
+      - name: Install cmake
+        run: sudo apt-get update && sudo apt-get install cmake -y
+      - name: Build Binaries
+        run: make binaries
+      - name: Run Benchmarks
+        run: make bench-all
+      - name: Post result comment
+        uses: mshick/add-pr-comment@v2
+        with:
+          message-path: bench-output.md

.gitignore

@@ -2,3 +2,8 @@ target
 comrak-*
 .vscode
 .idea
+vendor/comrak
+vendor/progit
+benches/cmark-gfm
+benches/comrak-*
+benches/pulldown-cmark

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "vendor/cmark-gfm"]
 	path = vendor/cmark-gfm
 	url = https://github.com/kivikakk/cmark-gfm.git
+[submodule "vendor/pulldown-cmark"]
+	path = vendor/pulldown-cmark
+	url = https://github.com/raphlinus/pulldown-cmark.git

Makefile

@@ -1,7 +1,43 @@
+ROOT:=$(shell git rev-parse --show-toplevel)
+COMMIT:=$(shell git rev-parse --short HEAD)
+MIN_RUNS:=25
+
 src/scanners.rs: src/scanners.re
 	re2rust -W -Werror -i --no-generation-date -o $@ $<
 	cargo fmt
 
 bench:
 	cargo build --release
 	(cd vendor/cmark-gfm/; make bench PROG=../../target/release/comrak)
+
+binaries: build-comrak-branch build-comrak-master build-cmark-gfm build-pulldown-cmark
+
+build-comrak-branch:
+	cargo build --release
+	cp ${ROOT}/target/release/comrak ${ROOT}/benches/comrak-${COMMIT}
+
+build-comrak-master:
+	git clone https://github.com/kivikakk/comrak.git --depth 1 --single-branch ${ROOT}/vendor/comrak || true
+	cd ${ROOT}/vendor/comrak && \
+	cargo build --release && \
+	cp ./target/release/comrak ${ROOT}/benches/comrak-main
+
+build-cmark-gfm:
+	cd ${ROOT}/vendor/cmark-gfm && \
+	make && \
+	cp build/src/cmark-gfm ${ROOT}/benches/cmark-gfm
+
+build-pulldown-cmark:
+	cd ${ROOT}/vendor/pulldown-cmark && \
+	cargo build --release && \
+	cp target/release/pulldown-cmark ${ROOT}/benches/pulldown-cmark
+
+bench-comrak: build-comrak-branch
+	git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null
+	cd benches && \
+	hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' --warmup 3 --min-runs ${MIN_RUNS}  -L binary comrak-${COMMIT} './bench.sh ./{binary}'
+
+bench-all: binaries
+	git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null
+	cd benches && \
+	hyperfine --prepare 'sudo sync; echo 3 | sudo tee /proc/sys/vm/drop_caches' --warmup 3 --min-runs ${MIN_RUNS}  -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md

README.md

@@ -197,6 +197,33 @@ assert_eq!(
      </ol>\n");
 ```
 
+## Benchmarking
+
+For running benchmarks, you will need to [install hyperfine](https://github.com/sharkdp/hyperfine#installation) and optionally cmake.
+
+If you want to just run benchmark for comrak, with current state of repo, you can simply run
+```bash
+make bench-comrak
+```
+
+This will build comrak in release mode, and run benchmark on it. You will see the time measurements as reported by hyperfine in the console.
+
+Makefile also provides a way to run benchmarks for comark current state (with your changes), comrak main branch, cmark-gfm and pulldown-cmark. For this you will need to install cmake. After that make sure that you have set-up the git submodules. In case you have not installed submodules when cloning, you can do it by running
+```bash
+git submodule update --init
+```
+
+After this is done, you can run 
+```bash
+make bench-all
+```
+
+which will run benchmarks across all, and report the time take by each as well as relative time.
+
+Apart from this, CI is also setup for running benchmarks when a pull request is first opened. It will comment the results on the pull request in a tabular format comparing the 4 versions. After that you can manually trigger this CI by commenting `/run-bench` on the PR, this will update the existing comment with new results. Note Benchmarks won't be automatically run on each push.
+
+
+
 ## Security
 
 As with [`cmark`](https://github.com/commonmark/cmark) and [`cmark-gfm`](https://github.com/github/cmark-gfm#security),

benches/bench.sh

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+PROG=$1
+ROOTDIR=$(git rev-parse --show-toplevel)
+
+for lang in ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw; do \
+    cat $ROOTDIR/vendor/progit/$lang/*/*.markdown | $PROG > /dev/null
+done