Merge pull request #326 from YJDoc2/add-bench-ci

Add CI for running benchmarks

Author: kivikakk

.github/workflows/benchmarks.yml

@@ -0,0 +1,36 @@
+name: benchmarks
+
+on: 
+  pull_request:
+    types:
+      - opened
+      - reopened
+  issue_comment:
+    types:
+      - created
+
+jobs:
+  run_benchmarks:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    # run  either when pull request is opened or when comment body (only on pr) is /run-bench
+    if: (github.event_name == 'pull_request') || ((github.event.issue.pull_request != null) && github.event.comment.body == '/run-bench')
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Install hyperfine
+        run: cargo install hyperfine
+      - name: Install cmake
+        run: sudo apt-get update && sudo apt-get install cmake -y
+      - name: Build Binaries
+        run: make binaries
+      - name: Run Benchmarks
+        run: make bench-all
+      - name: Post result comment
+        uses: mshick/add-pr-comment@v2
+        with:
+          message-path: bench-output.md

.gitignore

@@ -2,3 +2,9 @@ target
 comrak-*
 .vscode
 .idea
+vendor/comrak
+vendor/progit
+benches/cmark-gfm
+benches/comrak-*
+benches/pulldown-cmark
+benches/markdown-it

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "vendor/cmark-gfm"]
 	path = vendor/cmark-gfm
 	url = https://github.com/kivikakk/cmark-gfm.git
+[submodule "vendor/pulldown-cmark"]
+	path = vendor/pulldown-cmark
+	url = https://github.com/raphlinus/pulldown-cmark.git
+[submodule "vendor/markdown-it"]
+	path = vendor/markdown-it
+	url = https://github.com/rlidwka/markdown-it.rs.git

Cargo.lock

@@ -1,7 +1,50 @@
+ROOT:=$(shell git rev-parse --show-toplevel)
+COMMIT:=$(shell git rev-parse --short HEAD)
+MIN_RUNS:=25
+
 src/scanners.rs: src/scanners.re
 	re2rust -W -Werror -i --no-generation-date -o $@ $<
 	cargo fmt
 
 bench:
 	cargo build --release
 	(cd vendor/cmark-gfm/; make bench PROG=../../target/release/comrak)
+
+binaries: build-comrak-branch build-comrak-master build-cmark-gfm build-pulldown-cmark build-markdown-it
+
+build-comrak-branch:
+	cargo build --release
+	cp ${ROOT}/target/release/comrak ${ROOT}/benches/comrak-${COMMIT}
+
+build-comrak-master:
+	git clone https://github.com/kivikakk/comrak.git --depth 1 --single-branch ${ROOT}/vendor/comrak || true
+	cd ${ROOT}/vendor/comrak && \
+	cargo build --release && \
+	cp ./target/release/comrak ${ROOT}/benches/comrak-main
+
+build-cmark-gfm:
+	cd ${ROOT}/vendor/cmark-gfm && \
+	make && \
+	cp build/src/cmark-gfm ${ROOT}/benches/cmark-gfm
+
+build-markdown-it:
+	cd ${ROOT}/vendor/markdown-it && \
+	cargo build --release && \
+	cp target/release/markdown-it ${ROOT}/benches/markdown-it
+
+build-pulldown-cmark:
+	cd ${ROOT}/vendor/pulldown-cmark && \
+	cargo build --release && \
+	cp target/release/pulldown-cmark ${ROOT}/benches/pulldown-cmark
+
+bench-comrak: build-comrak-branch
+	git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null
+	cd benches && \
+	hyperfine --warmup 3 --min-runs ${MIN_RUNS}  -L binary comrak-${COMMIT} './bench.sh ./{binary}'
+
+bench-all: binaries
+	git clone https://github.com/progit/progit.git ${ROOT}/vendor/progit || true > /dev/null
+	cd benches && \
+	hyperfine --warmup 10 --min-runs ${MIN_RUNS}  -L binary comrak-${COMMIT},comrak-main,pulldown-cmark,cmark-gfm,markdown-it './bench.sh ./{binary}' --export-markdown ${ROOT}/bench-output.md &&\
+	echo "\n\nRun on" `date -u` >> ${ROOT}/bench-output.md
+

Makefile

@@ -197,6 +197,33 @@ assert_eq!(
      </ol>\n");
 ```
 
+## Benchmarking
+
+For running benchmarks, you will need to [install hyperfine](https://github.com/sharkdp/hyperfine#installation) and optionally cmake.
+
+If you want to just run the benchmark for `comrak`, with the current state of the repo, you can simply run
+```bash
+make bench-comrak
+```
+
+This will build comrak in release mode, and run benchmark on it. You will see the time measurements as reported by hyperfine in the console.
+
+Makefile also provides a way to run benchmarks for `comrak` current state (with your changes), `comrak` main branch, [`cmark-gfm`](https://github.com/github/cmark-gfm), [`pulldown-cmark`](https://github.com/raphlinus/pulldown-cmark) and [`markdown-it.rs`](https://github.com/rlidwka/markdown-it.rs). For this you will need to install `cmake`. After that make sure that you have set-up the git submodules. In case you have not installed submodules when cloning, you can do it by running
+```bash
+git submodule update --init
+```
+
+After this is done, you can run 
+```bash
+make bench-all
+```
+
+which will run benchmarks across all, and report the time take by each as well as relative time.
+
+Apart from this, CI is also setup for running benchmarks when a pull request is first opened. It will add a comment with the results on the pull request in a tabular format comparing the 5 versions. After that you can manually trigger this CI by commenting `/run-bench` on the PR, this will update the existing comment with new results. Note benchmarks won't be automatically run on each push.
+
+
+
 ## Security
 
 As with [`cmark`](https://github.com/commonmark/cmark) and [`cmark-gfm`](https://github.com/github/cmark-gfm#security),

README.md

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+PROG=$1
+ROOTDIR=$(git rev-parse --show-toplevel)
+
+for lang in ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw; do \
+    cat $ROOTDIR/vendor/progit/$lang/*/*.markdown | $PROG > /dev/null
+done