From 308afc46773b7f68ef2d39c1c2211f93e43f5521 Mon Sep 17 00:00:00 2001 From: Bill Murdock Date: Wed, 20 May 2026 16:52:16 -0400 Subject: [PATCH] fix: post leaderboard validation comments via workflow_run for fork PRs Fork PRs get a scoped-down GITHUB_TOKEN that lacks write permissions, so the inline github-script comment step always 403s. Split the comment posting into a separate workflow_run-triggered workflow that runs in the upstream repo context with elevated permissions, following the same pattern as coverage-comment.yml. The validate job now saves results as an artifact; the new leaderboard-comment.yml downloads it and posts/updates the PR comment. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/leaderboard-comment.yml | 112 ++++++++++++++++++++++ .github/workflows/leaderboard.yml | 42 ++++---- 2 files changed, 132 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/leaderboard-comment.yml diff --git a/.github/workflows/leaderboard-comment.yml b/.github/workflows/leaderboard-comment.yml new file mode 100644 index 00000000..fc18c0f4 --- /dev/null +++ b/.github/workflows/leaderboard-comment.yml @@ -0,0 +1,112 @@ +name: Leaderboard Validation Comment + +# Post validation results as a PR comment after the leaderboard workflow completes. +# Uses workflow_run to run in the upstream repo context with write permissions, +# enabling comments on fork PRs (same pattern as coverage-comment.yml). + +on: + workflow_run: + workflows: ["Leaderboard Management"] + types: [completed] + +jobs: + post-comment: + name: Post Validation Comment + runs-on: ubuntu-latest + if: github.event.workflow_run.event == 'pull_request' + permissions: + pull-requests: write + actions: read + + steps: + - name: Get PR number from workflow run + id: pr_info + env: + GH_TOKEN: ${{ github.token }} + run: | + HEAD_REPO="${{ github.event.workflow_run.head_repository.full_name }}" + HEAD_BRANCH="${{ github.event.workflow_run.head_branch }}" + TARGET_REPO="${{ github.repository }}" + + if [ "$HEAD_REPO" = "$TARGET_REPO" ]; then + BRANCH_QUERY="${HEAD_BRANCH}" + else + BRANCH_QUERY="${HEAD_REPO%%/*}:${HEAD_BRANCH}" + fi + + PR_NUMBER=$(gh pr view \ + --repo "$TARGET_REPO" \ + "$BRANCH_QUERY" \ + --json number --jq .number 2>/dev/null || echo "") + + if [ -z "$PR_NUMBER" ]; then + echo "::warning::Could not find PR for ${BRANCH_QUERY} in ${TARGET_REPO}" + exit 0 + fi + + echo "pr_number=${PR_NUMBER}" >> "$GITHUB_OUTPUT" + + - name: Download validation data + if: steps.pr_info.outputs.pr_number + uses: actions/download-artifact@v4 + with: + name: leaderboard-validation + path: validation-data + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Post validation comment + if: steps.pr_info.outputs.pr_number + uses: actions/github-script@v9 + env: + TRUSTED_PR_NUMBER: ${{ steps.pr_info.outputs.pr_number }} + with: + script: | + const fs = require('fs'); + const data = JSON.parse(fs.readFileSync('validation-data/validation.json', 'utf8')); + + const artifactPR = data.pr_number; + const trustedPR = parseInt(process.env.TRUSTED_PR_NUMBER); + if (parseInt(artifactPR) !== trustedPR) { + core.setFailed(`PR number mismatch: artifact=${artifactPR}, expected=${trustedPR}`); + return; + } + + const claimed = data.claimed_score || 'N/A'; + const actual = data.actual_score || 'N/A'; + const diff = actual !== 'N/A' && actual !== '' + ? Math.abs(parseFloat(actual) - parseFloat(claimed)).toFixed(1) + : 'N/A'; + + const status = parseFloat(diff) <= 2.0 ? '✅ **PASSED**' : '❌ **FAILED**'; + const body = `## Leaderboard Validation\n\n${status}\n\n` + + `**Claimed**: ${claimed}/100\n` + + `**Verified**: ${actual}/100\n` + + `**Diff**: ${diff} points (±2 tolerance)`; + + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: trustedPR, + }); + + const existing = comments.data.find(c => + c.user.login === 'github-actions[bot]' && + c.body.includes('Leaderboard Validation') + ); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: trustedPR, + body: body, + }); + } diff --git a/.github/workflows/leaderboard.yml b/.github/workflows/leaderboard.yml index 2102ba01..be541277 100644 --- a/.github/workflows/leaderboard.yml +++ b/.github/workflows/leaderboard.yml @@ -26,8 +26,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: read # Checkout PR branch - pull-requests: write # Post validation results as PR comment - issues: write # Required by github-script (PRs are issues in GH API) steps: # Fix for outdated forks: Always use upstream for validation tools/schema # The PR branch may be from a fork that hasn't synced with upstream, @@ -203,34 +201,34 @@ jobs: exit 1 fi - - name: Post validation results + - name: Save validation results if: always() - uses: actions/github-script@v9 env: CLAIMED_SCORE: ${{ steps.extract.outputs.claimed_score }} ACTUAL_SCORE: ${{ env.ACTUAL_SCORE }} REPO_NAME: ${{ steps.extract.outputs.repo_name }} CLAIMED_RESEARCH_VERSION: ${{ steps.extract.outputs.research_version }} ACTUAL_RESEARCH_VERSION: ${{ env.ACTUAL_RESEARCH_VERSION }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + mkdir -p /tmp/validation-data + jq -n \ + --arg claimed "$CLAIMED_SCORE" \ + --arg actual "$ACTUAL_SCORE" \ + --arg repo_name "$REPO_NAME" \ + --arg claimed_rv "$CLAIMED_RESEARCH_VERSION" \ + --arg actual_rv "$ACTUAL_RESEARCH_VERSION" \ + --arg pr_number "$PR_NUMBER" \ + '{claimed_score: $claimed, actual_score: $actual, repo_name: $repo_name, claimed_research_version: $claimed_rv, actual_research_version: $actual_rv, pr_number: $pr_number}' \ + > /tmp/validation-data/validation.json + + - name: Upload validation results + if: always() + uses: actions/upload-artifact@v4 with: - script: | - // SAFE: All values from environment variables - const claimed = process.env.CLAIMED_SCORE || 'N/A'; - const actual = process.env.ACTUAL_SCORE || 'N/A'; - const diff = actual !== 'N/A' ? Math.abs(parseFloat(actual) - parseFloat(claimed)).toFixed(1) : 'N/A'; - - const status = parseFloat(diff) <= 2.0 ? '✅ **PASSED**' : '❌ **FAILED**'; - const body = `## Leaderboard Validation\n\n${status}\n\n` + - `**Claimed**: ${claimed}/100\n` + - `**Verified**: ${actual}/100\n` + - `**Diff**: ${diff} points (±2 tolerance)`; - - github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: body - }); + name: leaderboard-validation + path: /tmp/validation-data/validation.json + retention-days: 1 # Job 2: Update leaderboard data (after merge to main, or manual trigger) update: