Integrate hyperlight-ci into CI workflows and Just recipes

- Add cargo alias (`cargo ci`) for convenient hyperlight-ci invocation
- Update dep_benchmarks workflow to use `cargo ci bench` and generate
  a markdown report via `cargo ci bench-report`, posting results as
  a PR comment per hypervisor/cpu matrix entry
- Add benchmarks job to ValidatePullRequest workflow with hypervisor
  and cpu matrix, gated behind docs-only and build-guests checks
- Grant pull-requests: write permission for PR comment posting
- Simplify Justfile bench recipes to delegate to `cargo ci bench`
- Update benchmarking docs to reflect the new workflow

Signed-off-by: Jorge Prendes <jorge.prendes@gmail.com>

Author: jprendes

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias] # command aliases
+ci = ["run", "--quiet", "--package=hyperlight-ci", "--"]

.github/hyperlight-bot.yml

@@ -0,0 +1,8 @@
+# Configuration for the hyperlight-gh-bot GitHub App.
+# See: https://github.com/jprendes/hyperlight-gh-bot
+
+# Name of the artifact containing the comment body.
+artifact_name: "pr-comment"
+
+# Regex matched against the job name to filter which jobs trigger the bot.
+job_filter: "post-benchmark-comment"

.github/workflows/ValidatePullRequest.yml

@@ -140,6 +140,61 @@ jobs:
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
     secrets: inherit
 
+  # Run benchmarks and post results as PR comment
+  benchmarks:
+    needs:
+      - docs-pr
+      - build-guests
+    # Required because update-guest-locks is skipped on non-dependabot PRs,
+    # and a skipped dependency transitively skips all downstream jobs.
+    # See: https://github.com/actions/runner/issues/2205
+    if: ${{ !cancelled() && !failure() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        hypervisor: ['hyperv-ws2025', mshv3, kvm]
+        cpu: [amd, intel]
+    uses: ./.github/workflows/dep_benchmarks.yml
+    secrets: inherit
+    with:
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      hypervisor: ${{ matrix.hypervisor }}
+      cpu: ${{ matrix.cpu }}
+
+  # Combine benchmark reports into a single artifact for the hyperlight-gh-bot
+  # to post as a PR comment. Only runs for PRs (not merge groups) with code changes.
+  benchmark-comment:
+    name: post-benchmark-comment
+    needs:
+      - docs-pr
+      - benchmarks
+    if: ${{ !cancelled() && !failure() && needs.docs-pr.outputs.docs-only == 'false' && github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download benchmark reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          pattern: benchmark-report_*
+          path: reports/
+
+      - name: Combine benchmark reports
+        run: |
+          echo '## Benchmark Results' > pr-comment.md
+          echo '' >> pr-comment.md
+          for f in reports/benchmark-report_*/benchmark.md; do
+            [ -f "$f" ] || continue
+            cat "$f" >> pr-comment.md
+            echo '' >> pr-comment.md
+          done
+
+      - name: Upload PR comment artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: pr-comment
+          path: pr-comment.md
+          if-no-files-found: warn
+          retention-days: 1
+
   spelling:
     name: spell check with typos
     runs-on: ubuntu-latest
@@ -167,6 +222,7 @@ jobs:
       - build-test
       - run-examples
       - fuzzing
+      - benchmarks
       - spelling
       - license-headers
     if: always()

.github/workflows/dep_benchmarks.yml

@@ -56,7 +56,6 @@ on:
         required: false
         type: number
         default: 5
-
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
@@ -135,6 +134,16 @@ jobs:
       - name: Run benchmarks
         run: just bench-ci main
 
+      - name: Create benchmarks report
+        run: cargo ci bench-report --collapsible '${{ inputs.hypervisor }} / ${{ inputs.cpu }} (${{ runner.os }})' > target/criterion/benchmark.md
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: benchmark-report_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }}
+          path: target/criterion/benchmark.md
+          if-no-files-found: error
+          retention-days: 1
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:
           name: benchmarks_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }}

Justfile

@@ -398,12 +398,10 @@ bench-download os hypervisor cpu tag="":
 
 # Warning: compares to and then OVERWRITES the given baseline
 bench-ci baseline features="":
-    @# Benchmarks are always run with release builds for meaningful results
-    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
+    cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose --save-baseline {{ baseline }}
 
 bench features="":
-    @# Benchmarks are always run with release builds for meaningful results
-    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
+    cargo ci bench {{ if features == "" {''} else { "--features " + features } }} --verbose
 
 ###############
 ### FUZZING ###