ci: Deploy bench results on bencher.dev

.github/workflows/compile.yml

@@ -7,10 +7,7 @@ on:
 
 permissions:
   contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
+  checks: write
 
 env:
   COMPILE_DIR: Benchmarks/Compile
@@ -61,7 +58,7 @@ jobs:
         with:
           lake-package-directory: ${{ env.COMPILE_DIR }}
           build: false
-          use-github-cache: false 
+          use-github-cache: false
       # FLT and FC take a few minutes to rebuild, so we cache the build artifacts
       - if: matrix.cache_pkg
         uses: actions/cache@v4
@@ -73,5 +70,58 @@ jobs:
       - run: lake build Compile${{ matrix.bench }}
         working-directory: ${{ env.COMPILE_DIR }}
       # Run the `ix` compiler over the input library env
-      - run: ix compile --path ${{ env.COMPILE_DIR }}/Compile${{ matrix.bench }}.lean
-
+      - name: Run ix compile
+        run: ix compile --path ${{ env.COMPILE_DIR }}/Compile${{ matrix.bench }}.lean 2>&1 | tee output.txt
+      # Parse the ##benchmark## line into Bencher Metric Format JSON
+      - name: Generate benchmark JSON
+        run: |
+          line=$(grep '^##benchmark##' output.txt)
+          elapsed_s=$(echo "$line" | awk '{printf "%.3f", $2 / 1000}')
+          bytes=$(echo "$line" | awk '{print $3}')
+          constants=$(echo "$line" | awk '{print $4}')
+          throughput=$(echo "$line" | awk '{if ($2 > 0) printf "%.2f", $4 * 1000 / $2; else print 0}')
+          cat > benchmark.json <<EOF
+          {
+            "${{ matrix.bench }}": {
+              "build-time": {"value": ${elapsed_s}},
+              "file-size": {"value": ${bytes}},
+              "throughput": {"value": ${throughput}},
+              "constants": {"value": ${constants}}
+            }
+          }
+          EOF
+      # Deploy benchmark results to https://bencher.dev/console/projects/ix/plots
+      - uses: bencherdev/bencher@main
+      - name: Track benchmarks
+        run: |
+          bencher run \
+            --project ix \
+            --token '${{ secrets.BENCHER_API_TOKEN }}' \
+            --branch '${{ github.ref_name }}' \
+            --hash '${{ github.sha }}' \
+            --testbed warp-ubuntu-x64-32x \
+            --adapter json \
+            --err \
+            --github-actions '${{ secrets.GITHUB_TOKEN }}' \
+            --threshold-measure build-time \
+            --threshold-test log_normal \
+            --threshold-max-sample-size 64 \
+            --threshold-upper-boundary 0.95 \
+            --threshold-lower-boundary _ \
+            --threshold-measure throughput \
+            --threshold-test log_normal \
+            --threshold-max-sample-size 64 \
+            --threshold-upper-boundary _ \
+            --threshold-lower-boundary 0.95 \
+            --threshold-measure file-size \
+            --threshold-test percentage \
+            --threshold-max-sample-size 64 \
+            --threshold-upper-boundary 0.10 \
+            --threshold-lower-boundary _ \
+            --threshold-measure constants \
+            --threshold-test percentage \
+            --threshold-max-sample-size 64 \
+            --threshold-upper-boundary 0.10 \
+            --threshold-lower-boundary _ \
+            --thresholds-reset \
+            --file benchmark.json

.github/workflows/nix.yml

@@ -17,7 +17,7 @@ jobs:
   # Runs Lean tests via Nix
   nix-test:
     name: Nix Tests
-    runs-on: ubuntu-latest
+    runs-on: warp-ubuntu-latest-x64-8x
     steps:
       - uses: actions/checkout@v5
       - uses: cachix/install-nix-action@v31

Ix/Cli/CompileCmd.lean

@@ -64,6 +64,8 @@ def runCompileCmd (p : Cli.Parsed) : IO UInt32 := do
   let elapsed := (← IO.monoMsNow) - start
 
   println! "Compiled {fmtBytes bytes.size} env in {elapsed.formatMs}"
+  -- Machine-readable line for CI benchmark tracking
+  IO.println s!"##benchmark## {elapsed} {bytes.size} {totalConsts}"
   return 0
 
 

README.md

@@ -172,6 +172,10 @@ Ix consists of the following core components:
 - Integration with the [iroh p2p network](https://www.iroh.computer/) so that
   different ix users can easily share `ixon` data between themselves.
 
+## Benchmarks
+
+Compiler performance benchmarks are tracked at https://bencher.dev/console/projects/ix/plots
+
 ## Build & Install
 
 - Build and test the Ix library with `lake build` and `lake test`