From 0a321c78d0604b3aa8a2256a7b3cb9c9fae809cc Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Tue, 20 Oct 2026 06:52:05 +0000 Subject: [PATCH 01/22] ci: dispatch external CI to the external repositories (ext-ci orchestrator) Adds ext-ci-trigger.yml (no-op pull_request gate), ext-ci.yml (workflow_run orchestrator: dispatch, poll, one Check Run per external repo, failed-only re-run, notify/block policy) and .github/ext-ci-manifest.json onboarding led-blinker, zephyr-reference and generic-hub-reference in the notify tier. Existing ext-*.yml workflows are untouched and keep running alongside. --- .github/ext-ci-manifest.json | 43 ++ .github/scripts/ext-ci-orchestrator.mjs | 545 ++++++++++++++++++++++++ .github/workflows/ext-ci-trigger.yml | 21 + .github/workflows/ext-ci.yml | 74 ++++ 4 files changed, 683 insertions(+) create mode 100644 .github/ext-ci-manifest.json create mode 100644 .github/scripts/ext-ci-orchestrator.mjs create mode 100644 .github/workflows/ext-ci-trigger.yml create mode 100644 .github/workflows/ext-ci.yml diff --git a/.github/ext-ci-manifest.json b/.github/ext-ci-manifest.json new file mode 100644 index 00000000000..6dfe5514c05 --- /dev/null +++ b/.github/ext-ci-manifest.json @@ -0,0 +1,43 @@ +{ + "upstream_repository": "nasa/fprime", + "check_prefix": "ext-ci", + "paths_ignore": [ + "docs/**", + "**.md", + ".github/actions/spelling/**", + ".github/ISSUE_TEMPLATE/**", + ".github/agents/**" + ], + "repos": [ + { + "id": "led-blinker", + "repository": "fprime-community/fprime-workshop-led-blinker", + "workflow": "ci.yml", + "default_ref": "devel", + "timeout_minutes": 60, + "failure_policy": "notify", + "owners": ["thomas-bc"], + "on_pr": true + }, + { + "id": "zephyr-reference", + "repository": "fprime-community/fprime-zephyr-reference", + "workflow": "ci.yml", + "default_ref": "devel", + "timeout_minutes": 90, + "failure_policy": "notify", + "owners": ["thomas-bc"], + "on_pr": true + }, + { + "id": "generic-hub-reference", + "repository": "fprime-community/fprime-generic-hub-reference", + "workflow": "ci.yml", + "default_ref": "devel", + "timeout_minutes": 60, + "failure_policy": "notify", + "owners": ["thomas-bc"], + "on_pr": true + } + ] +} diff --git a/.github/scripts/ext-ci-orchestrator.mjs b/.github/scripts/ext-ci-orchestrator.mjs new file mode 100644 index 00000000000..8c1f7a3e43e --- /dev/null +++ b/.github/scripts/ext-ci-orchestrator.mjs @@ -0,0 +1,545 @@ +#!/usr/bin/env node +// External-CI orchestrator (dependency-free Node >= 20). +// +// node ext-ci-orchestrator.mjs run dispatch, poll, report +// node ext-ci-orchestrator.mjs finalize close anything still open (runs under if: always()) +// +// Two credentials, never mixed: +// GITHUB_TOKEN this repo only (checks/pull-requests/issues write) - reports results +// DISPATCH_TOKEN external repos only (actions: write) - starts remote runs +import fs from 'node:fs'; + +const env = process.env; +const mode = process.argv[2] ?? 'run'; +const API = env.GITHUB_API_URL || 'https://api.github.com'; +const SERVER = env.GITHUB_SERVER_URL || 'https://github.com'; +const STATE_FILE = 'ext-ci-state.json'; +const RESULTS_FILE = 'ext-ci-results.json'; +const MANIFEST_PATH = '.github/ext-ci-manifest.json'; +const POLL_MS = Number(env.POLL_INTERVAL_SECONDS || 60) * 1000; +const CORRELATE_TIMEOUT_MS = 5 * 60 * 1000; +const BOT_LOGIN = 'github-actions[bot]'; +const NOTIFY_LABEL = 'ext-ci'; + +const repo = env.GITHUB_REPOSITORY; +const runId = env.GITHUB_RUN_ID; +const attempt = Number(env.GITHUB_RUN_ATTEMPT || 1); +const eventName = env.GITHUB_EVENT_NAME; +const event = JSON.parse(fs.readFileSync(env.GITHUB_EVENT_PATH, 'utf8')); +const orchestratorUrl = `${SERVER}/${repo}/actions/runs/${runId}${attempt > 1 ? `/attempts/${attempt}` : ''}`; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +const short = (sha) => sha.slice(0, 7); +// Remote-controlled text (job/step names, branch names) is only ever rendered inside code spans. +const code = (s) => '`' + String(s).replace(/[`\r\n]/g, ' ') + '`'; + +// ---------------------------------------------------------------- REST helpers +class ApiError extends Error { constructor(msg, status) { super(msg); this.status = status; } } + +function client(token, label) { + if (!token) throw new Error(`${label} token missing`); + const etags = new Map(); + async function request(method, url, body, { conditional = false } = {}) { + const full = url.startsWith('http') ? url : API + url; + const headers = { + authorization: `Bearer ${token}`, + accept: 'application/vnd.github+json', + 'x-github-api-version': '2022-11-28', + 'user-agent': 'fprime-ext-ci-orchestrator', + ...(body ? { 'content-type': 'application/json' } : {}), + }; + if (conditional && etags.has(full)) headers['if-none-match'] = etags.get(full).etag; + for (let i = 0; ; i++) { + let res; + try { + res = await fetch(full, { method, headers, body: body ? JSON.stringify(body) : undefined }); + } catch (e) { + if (i >= 4) throw new ApiError(`${label}: ${method} ${url} -> network error ${e.message}`, 0); + await sleep(2000 * 2 ** i); continue; + } + if (res.status === 304 && conditional) return { status: 200, ok: true, data: etags.get(full).data, cached: true }; + const text = await res.text(); + let data = null; + try { data = text ? JSON.parse(text) : null; } catch { data = text; } + const remaining = Number(res.headers.get('x-ratelimit-remaining')); + const limited = res.status === 429 || (res.status === 403 && (remaining === 0 || /rate limit/i.test(text))); + if ((limited || res.status >= 500) && i < 4) { + let wait = 2000 * 2 ** i; + const ra = Number(res.headers.get('retry-after')); + const reset = Number(res.headers.get('x-ratelimit-reset')); + if (ra) wait = ra * 1000; + else if (limited && reset) wait = Math.max(wait, reset * 1000 - Date.now() + 1000); + console.log(`::warning::${label}: ${method} ${url} -> ${res.status}; retrying in ${Math.round(wait / 1000)} s`); + await sleep(Math.min(wait, 15 * 60_000)); continue; + } + if (res.ok && conditional && res.headers.get('etag')) etags.set(full, { etag: res.headers.get('etag'), data }); + return { status: res.status, ok: res.ok, data }; + } + } + async function api(method, url, body, opts) { + const r = await request(method, url, body, opts); + if (!r.ok) throw new ApiError(`${label}: ${method} ${url} -> ${r.status} ${JSON.stringify(r.data).slice(0, 300)}`, r.status); + return r.data; + } + return { + get: (u, opts) => api('GET', u, undefined, opts), + post: (u, b) => api('POST', u, b), + patch: (u, b) => api('PATCH', u, b), + // true/false for 200/404; anything else is an error, never silently "absent". + exists: async (u) => { + const r = await request('GET', u); + if (r.status === 200) return true; + if (r.status === 404) return false; + throw new ApiError(`${label}: GET ${u} -> ${r.status}`, r.status); + }, + }; +} +const own = client(env.GITHUB_TOKEN, 'GITHUB_TOKEN'); +const ext = client(env.DISPATCH_TOKEN, 'DISPATCH_TOKEN'); + +// ---------------------------------------------------------------- target resolution +// What F´ revision are we testing, and which PR (if any) gets the Check Runs? +async function resolveTarget() { + const t = { pr: null, headRepo: repo, headSha: null, headBranch: null, baseRef: null, beforeSha: null }; + if (eventName === 'workflow_run') { + const wr = event.workflow_run; + // Trust only the event payload: nothing produced by the (untrusted) trigger run is read. + t.headSha = wr.head_sha; + t.headRepo = wr.head_repository.full_name; + t.headBranch = wr.head_branch; + if (wr.event === 'merge_group') { + // Merge-queue branch gh-readonly-queue//pr--: checks go on the queue SHA, no PR. + const m = /^gh-readonly-queue\/(.+?)\/pr-\d+-/.exec(t.headBranch || ''); + t.baseRef = m ? m[1] : env.GITHUB_REF_NAME; + return t; + } + // workflow_run.pull_requests is empty for fork PRs; find the PR by commit, then by head ref. + let pr = (await own.get(`/repos/${repo}/commits/${t.headSha}/pulls?per_page=20`)) + .find((p) => p.state === 'open' && p.head.sha === t.headSha && p.base.repo.full_name === repo); + if (!pr) { + const owner = t.headRepo.split('/')[0]; + pr = (await own.get(`/repos/${repo}/pulls?state=open&head=${encodeURIComponent(`${owner}:${t.headBranch}`)}`)) + .find((p) => p.head.sha === t.headSha); + } + if (!pr) throw new Error(`no open PR found for ${t.headRepo}@${t.headSha} (${t.headBranch}); the PR may have been updated or closed`); + t.pr = pr.number; + t.baseRef = pr.base.ref; + } else if (eventName === 'workflow_dispatch') { + const prNumber = (env.INPUT_PR_NUMBER || '').trim(); + if (prNumber) { + if (!/^\d+$/.test(prNumber)) throw new Error(`pr_number must be numeric, got ${JSON.stringify(prNumber)}`); + const pr = await own.get(`/repos/${repo}/pulls/${prNumber}`); + t.pr = pr.number; t.headSha = pr.head.sha; t.headRepo = pr.head.repo.full_name; + t.headBranch = pr.head.ref; t.baseRef = pr.base.ref; + } else { + const ref = (env.INPUT_FPRIME_REF || '').trim() || env.GITHUB_REF_NAME; + const commit = await own.get(`/repos/${repo}/commits/${encodeURIComponent(ref)}`); + t.headSha = commit.sha; t.headBranch = ref; t.baseRef = ref; + } + } else { // push, schedule + t.headSha = env.GITHUB_SHA; t.headBranch = env.GITHUB_REF_NAME; t.baseRef = env.GITHUB_REF_NAME; + if (eventName === 'push' && event.before && !/^0+$/.test(event.before)) t.beforeSha = event.before; + } + return t; +} + +// Manifest is read from the base branch of the PR (the orchestrator itself always runs from +// the default branch), so release/** PRs use their own branch's manifest. Only a missing file +// falls back to the checked-out copy; any other error is fatal. +async function loadManifest(ref) { + try { + const file = await own.get(`/repos/${repo}/contents/${MANIFEST_PATH}?ref=${encodeURIComponent(ref)}`); + return JSON.parse(Buffer.from(file.content, 'base64').toString('utf8')); + } catch (e) { + if (!(e instanceof ApiError && e.status === 404)) throw e; + console.log(`::warning::manifest missing at ${ref}; using the checked-out copy`); + return JSON.parse(fs.readFileSync(MANIFEST_PATH, 'utf8')); + } +} + +const checkName = (manifest, id) => `${manifest.check_prefix || 'ext-ci'} / ${id}`; +const ourExternalId = (id) => `${runId}-${id}`; // same across attempts of one run + +// ---------------------------------------------------------------- change filter +// paths-ignore lives here, not on the trigger workflow, so that filtered PRs still get a +// (skipped) Check Run per repo and required checks can be satisfied. +function globToRegExp(glob) { + let re = '^'; + for (let i = 0; i < glob.length; i++) { + const c = glob[i]; + if (c === '*') { + if (glob[i + 1] === '*') { re += '.*'; i++; if (glob[i + 1] === '/') i++; } + else re += '[^/]*'; + } else if (c === '?') re += '[^/]'; + else re += c.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + } + return new RegExp(re + '$'); +} + +async function changedFiles(target) { + let url = null; + if (target.pr) url = `/repos/${repo}/pulls/${target.pr}/files?per_page=100`; + else if (target.beforeSha) url = `/repos/${repo}/compare/${target.beforeSha}...${target.headSha}?per_page=100`; + if (!url) return null; // unknown -> run everything + const files = []; + for (let page = 1; page <= 30; page++) { + const r = await own.get(`${url}&page=${page}`); + const batch = target.pr ? r : (r.files || []); + files.push(...batch.map((f) => f.filename)); + if (batch.length < 100) break; + } + return files; +} + +function relevantChange(files, manifest) { + const ignore = (manifest.paths_ignore || []).map(globToRegExp); + if (!files || !ignore.length) return true; + return files.some((f) => !ignore.some((re) => re.test(f))); +} + +// ---------------------------------------------------------------- selection +async function latestOwnCheckRun(name, id, sha) { + const r = await own.get(`/repos/${repo}/commits/${sha}/check-runs?check_name=${encodeURIComponent(name)}&filter=all&per_page=50`); + // Only reuse a check this very orchestrator run created (external_id stamped with our run id). + return r.check_runs.filter((c) => c.external_id === ourExternalId(id) && c.app && c.app.slug === 'github-actions') + .sort((a, b) => b.id - a.id)[0] ?? null; +} + +async function selectRepos(manifest, target) { + let repos = manifest.repos; + const wanted = (env.INPUT_REPOS || 'all').trim(); + if (eventName === 'workflow_dispatch' && wanted && wanted !== 'all') { + const ids = wanted.split(',').map((s) => s.trim()).filter(Boolean); + const unknown = ids.filter((id) => !repos.some((r) => r.id === id)); + if (unknown.length) throw new Error(`unknown manifest ids: ${unknown.join(', ')}`); + repos = repos.filter((r) => ids.includes(r.id)); + } else if (target.pr) { + repos = repos.filter((r) => r.on_pr !== false); + } + // Failed-only re-run: on attempt > 1 keep every repo whose Check Run from attempt 1..n-1 of + // this same run already succeeded, and dispatch only the rest. + const carried = []; + const dispatch = []; + const forceAll = (env.INPUT_FORCE_ALL || '').toString() === 'true'; + for (const entry of repos) { + const prior = attempt > 1 && !forceAll ? await latestOwnCheckRun(checkName(manifest, entry.id), entry.id, target.headSha) : null; + if (prior && prior.status === 'completed' && prior.conclusion === 'success') carried.push({ entry, prior }); + else dispatch.push(entry); + } + return { carried, dispatch }; +} + +// pr- branch on the ext repo, else the F´ base branch name, else the manifest default. +async function resolveExtRef(entry, target) { + const candidates = []; + if (target.pr) candidates.push(`pr-${target.pr}`); + if (target.baseRef) candidates.push(target.baseRef); + for (const c of candidates) { + if (await ext.exists(`/repos/${entry.repository}/branches/${encodeURIComponent(c)}`)) return c; + } + return entry.default_ref || 'main'; +} + +// ---------------------------------------------------------------- state +function saveState(state) { fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2)); } +function loadState() { return fs.existsSync(STATE_FILE) ? JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')) : null; } + +// ---------------------------------------------------------------- dispatch +async function dispatchOne(manifest, entry, target, state) { + const dispatchId = `${runId}-${attempt}-${entry.id}`; + const name = checkName(manifest, entry.id); + const s = state.repos[entry.id] = { + id: entry.id, repository: entry.repository, workflow: entry.workflow, ext_ref: null, + failure_policy: entry.failure_policy || 'block', owners: entry.owners || [], + timeout_minutes: entry.timeout_minutes || 60, dispatch_id: dispatchId, dispatched_at: new Date().toISOString(), + check_run_id: null, remote_run_id: null, remote_run_url: null, remote_started_at: null, remote_conclusion: null, conclusion: null, + }; + try { + s.ext_ref = await resolveExtRef(entry, target); + const check = await own.post(`/repos/${repo}/check-runs`, { + name, head_sha: target.headSha, status: 'queued', external_id: ourExternalId(entry.id), details_url: orchestratorUrl, + output: { + title: `Dispatching ${entry.repository} @ ${s.ext_ref}`, + summary: `Dispatching ${code(entry.workflow)} on ${code(entry.repository)} @ ${code(s.ext_ref)} with F´ ${code(`${target.headRepo}@${short(target.headSha)}`)}.\n\nOrchestrator: ${orchestratorUrl}`, + }, + }); + s.check_run_id = check.id; + saveState(state); + const r = await ext.post(`/repos/${entry.repository}/actions/workflows/${entry.workflow}/dispatches`, { + ref: s.ext_ref, + inputs: { + fprime_repository: target.headRepo, fprime_ref: target.headSha, + fprime_pr_number: target.pr ? String(target.pr) : '', dispatch_id: dispatchId, + }, + return_run_details: true, + }); + if (r && r.workflow_run_id) { + s.remote_run_id = r.workflow_run_id; s.remote_run_url = r.html_url; + await markRunning(s); + } + console.log(`[${entry.id}] dispatched ${entry.repository}/${entry.workflow} @ ${s.ext_ref}${s.remote_run_url ? ` -> ${s.remote_run_url}` : ` (dispatch_id=${dispatchId})`}`); + } catch (e) { + s.error = `dispatch failed: ${e.message}`; + s.remote_conclusion = 'dispatch_failed'; + console.log(`::error::[${entry.id}] ${s.error}`); + } + saveState(state); +} + +async function markRunning(s) { + await own.patch(`/repos/${repo}/check-runs/${s.check_run_id}`, { + status: 'in_progress', details_url: s.remote_run_url, + output: { title: `Running on ${s.repository} @ ${s.ext_ref}`, summary: `Remote run: ${s.remote_run_url}\n\nOrchestrator: ${orchestratorUrl}` }, + }); +} + +// Fallback only for API servers that still answer 204 to the dispatch call: find the run whose +// title carries our dispatch_id (the ext workflow echoes it into run-name). +async function correlate(state) { + const pending = Object.values(state.repos).filter((s) => !s.remote_run_id && !s.remote_conclusion); + const deadline = Date.now() + CORRELATE_TIMEOUT_MS; + while (pending.length && Date.now() < deadline) { + for (const s of [...pending]) { + try { + const since = new Date(new Date(s.dispatched_at).getTime() - 60_000).toISOString(); + const runs = await ext.get(`/repos/${s.repository}/actions/runs?event=workflow_dispatch&created=${encodeURIComponent('>=' + since)}&per_page=50`); + const run = runs.workflow_runs.find((r) => (r.display_title || r.name || '').split(/\s+/).includes(s.dispatch_id)); + if (!run) continue; + s.remote_run_id = run.id; s.remote_run_url = run.html_url; + pending.splice(pending.indexOf(s), 1); + console.log(`[${s.id}] remote run ${run.html_url}`); + await markRunning(s); + saveState(state); + } catch (e) { console.log(`::warning::[${s.id}] correlate: ${e.message}`); } + } + if (pending.length) await sleep(15_000); + } + for (const s of pending) { + s.remote_conclusion = 'not_found'; + s.error = `no run with dispatch_id ${s.dispatch_id} appeared on ${s.repository} within ${CORRELATE_TIMEOUT_MS / 60000} min`; + console.log(`::error::[${s.id}] ${s.error}`); + } + saveState(state); +} + +// ---------------------------------------------------------------- poll +async function pollAll(state) { + const active = () => Object.values(state.repos).filter((s) => s.remote_run_id && !s.remote_conclusion); + while (active().length) { + await sleep(POLL_MS); + for (const s of active()) { + try { + const run = await ext.get(`/repos/${s.repository}/actions/runs/${s.remote_run_id}`, { conditional: true }); + if (run.run_started_at && !s.remote_started_at) s.remote_started_at = run.run_started_at; + if (run.status === 'completed') { + s.remote_conclusion = run.conclusion; + console.log(`[${s.id}] completed: ${run.conclusion}`); + } else if (s.remote_started_at && Date.now() - new Date(s.remote_started_at).getTime() > s.timeout_minutes * 60_000) { + // Timeout counts from when the remote run started, not from dispatch (queue time is not the ext repo's fault). + console.log(`::warning::[${s.id}] timed out after ${s.timeout_minutes} min of running; cancelling remote run`); + await ext.post(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/cancel`).catch(() => {}); + s.remote_conclusion = 'timed_out'; + } else if (!s.remote_started_at && Date.now() - new Date(s.dispatched_at).getTime() > 2 * s.timeout_minutes * 60_000) { + console.log(`::warning::[${s.id}] still queued after ${2 * s.timeout_minutes} min; giving up`); + await ext.post(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/cancel`).catch(() => {}); + s.remote_conclusion = 'queue_timeout'; + s.error = 'remote run never started (no runner available?)'; + } + } catch (e) { + // A single bad poll must not take the whole fan-out down; give up on this repo only + // when the credential is gone or the run vanished. + console.log(`::warning::[${s.id}] poll: ${e.message}`); + if (e instanceof ApiError && (e.status === 401 || e.status === 404)) { + s.remote_conclusion = 'error'; s.error = `poll failed: ${e.message}`; + } + } + } + saveState(state); + } +} + +// ---------------------------------------------------------------- report +async function failureDetails(s) { + if (!s.remote_run_id) return ''; + try { + const jobs = await ext.get(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/jobs?per_page=100`); + const bad = jobs.jobs.filter((j) => j.conclusion && j.conclusion !== 'success' && j.conclusion !== 'skipped'); + return bad.map((j) => { + const steps = (j.steps || []).filter((st) => st.conclusion && st.conclusion !== 'success' && st.conclusion !== 'skipped') + .map((st) => ` - step ${code(st.name)}: ${st.conclusion}`).join('\n'); + return `- [${code(j.name)}](${j.html_url}): **${j.conclusion}**${steps ? '\n' + steps : ''}`; + }).join('\n'); + } catch (e) { return `_(job details unavailable: ${code(e.message)})_`; } +} + +function checkConclusion(s) { + if (s.remote_conclusion === 'success') return 'success'; + if (s.failure_policy === 'notify') return 'neutral'; + if (s.remote_conclusion === 'cancelled') return 'cancelled'; + if (s.remote_conclusion === 'timed_out' || s.remote_conclusion === 'queue_timeout') return 'timed_out'; + return 'failure'; +} + +async function finalizeOne(target, s) { + s.conclusion = checkConclusion(s); + const details = s.remote_conclusion === 'success' ? '' : await failureDetails(s); + const header = `${code(s.repository)} @ ${code(s.ext_ref || '-')} with F´ ${code(`${target.headRepo}@${short(target.headSha)}`)}: **${s.remote_conclusion}**`; + const lines = [header, '']; + if (s.remote_run_url) lines.push(`Remote run: ${s.remote_run_url}`); + if (s.error) lines.push(`Error: ${code(s.error)}`); + if (s.failure_policy === 'notify' && s.remote_conclusion !== 'success') { + lines.push('', `This repo is ${code('failure_policy: notify')}: the failure does not block the PR; owners (${s.owners.map((o) => '@' + o).join(' ')}) have been notified.`); + } + lines.push('', `Orchestrator run (use "Re-run failed jobs" there to re-dispatch only the failed repos): ${orchestratorUrl}`); + if (s.check_run_id) { + await own.patch(`/repos/${repo}/check-runs/${s.check_run_id}`, { + status: 'completed', conclusion: s.conclusion, details_url: s.remote_run_url || orchestratorUrl, + output: { title: `${s.remote_conclusion} (${s.repository} @ ${s.ext_ref || '-'})`, summary: lines.join('\n'), text: details || undefined }, + }); + } + if (s.failure_policy === 'notify') await notifyOwners(target, s, details).catch((e) => console.log(`::warning::[${s.id}] notify: ${e.message}`)); +} + +// notify policy: one PR comment per ext repo (updated in place), or one tracking issue when +// there is no PR (closed again once the repo passes). Only comments/issues authored by the +// Actions bot are considered ours, so nobody can plant a marker to suppress notifications. +async function ensureLabel() { + if (await own.exists(`/repos/${repo}/labels/${NOTIFY_LABEL}`)) return; + await own.post(`/repos/${repo}/labels`, { name: NOTIFY_LABEL, color: 'ededed', description: 'Opened by the external-CI orchestrator' }).catch(() => {}); +} + +async function notifyOwners(target, s, details) { + const marker = ``; + const mentions = s.owners.map((o) => '@' + o).join(' '); + const failed = s.remote_conclusion !== 'success'; + const ours = (x) => x.user && x.user.login === BOT_LOGIN && x.body && x.body.includes(marker); + if (target.pr) { + let existing = null; + for (let page = 1; page <= 10 && !existing; page++) { + const comments = await own.get(`/repos/${repo}/issues/${target.pr}/comments?per_page=100&page=${page}`); + existing = comments.find(ours) || null; + if (comments.length < 100) break; + } + if (!failed && !existing) return; + const body = failed + ? `${marker}\n### External CI: ${code(s.id)} failed (non-blocking)\n\n${code(s.repository)} @ ${code(s.ext_ref)} does not pass against ${short(target.headSha)}: ${s.remote_run_url || code(s.error)}\n\n${details}\n\n${mentions} — this repo is ${code('failure_policy: notify')}, so the PR is not blocked, but it needs a look.` + : `${marker}\n### External CI: ${code(s.id)} passing again\n\n${code(s.repository)} @ ${code(s.ext_ref)} passes against ${short(target.headSha)}: ${s.remote_run_url}`; + if (existing) await own.patch(`/repos/${repo}/issues/comments/${existing.id}`, { body }); + else await own.post(`/repos/${repo}/issues/${target.pr}/comments`, { body }); + return; + } + const title = `[ext-ci] ${s.id} failing on ${target.headBranch}`; + await ensureLabel(); + const issues = await own.get(`/repos/${repo}/issues?state=open&labels=${NOTIFY_LABEL}&creator=${encodeURIComponent(BOT_LOGIN)}&per_page=100`); + const existing = issues.find((i) => !i.pull_request && ours(i) && i.title === title); + if (failed) { + const body = `${marker}\n${code(s.repository)} @ ${code(s.ext_ref)} fails against ${code(target.headBranch)} (${short(target.headSha)}): ${s.remote_run_url || code(s.error)}\n\n${details}\n\n${mentions}\n\nThis issue is closed automatically once the repo passes again.`; + if (existing) await own.post(`/repos/${repo}/issues/${existing.number}/comments`, { body: `Still failing at ${short(target.headSha)}: ${s.remote_run_url || code(s.error)}` }); + else await own.post(`/repos/${repo}/issues`, { title, body, labels: [NOTIFY_LABEL] }); + } else if (existing) { + await own.post(`/repos/${repo}/issues/${existing.number}/comments`, { body: `Passing again at ${short(target.headSha)}: ${s.remote_run_url}` }); + await own.patch(`/repos/${repo}/issues/${existing.number}`, { state: 'closed', state_reason: 'completed' }); + } +} + +async function carryForward(manifest, target, { entry, prior }, state) { + state.repos[entry.id] = { + id: entry.id, repository: entry.repository, failure_policy: entry.failure_policy || 'block', carried_forward: true, + check_run_id: prior.id, remote_run_url: prior.details_url, remote_conclusion: 'success', conclusion: 'success', + }; + await own.patch(`/repos/${repo}/check-runs/${prior.id}`, { + output: { + title: prior.output.title, + summary: `${prior.output.summary}\n\n_Not re-run in attempt ${attempt} of ${orchestratorUrl}: previous success carried forward._`, + }, + }); + console.log(`[${entry.id}] carried forward success from ${prior.details_url}`); +} + +async function skipAll(manifest, target, repos, state) { + for (const entry of repos) { + const check = await own.post(`/repos/${repo}/check-runs`, { + name: checkName(manifest, entry.id), head_sha: target.headSha, status: 'completed', conclusion: 'skipped', + external_id: ourExternalId(entry.id), details_url: orchestratorUrl, + output: { title: 'Not dispatched: no relevant files changed', summary: `Only paths matching ${code((manifest.paths_ignore || []).join(', '))} changed.\n\nOrchestrator: ${orchestratorUrl}` }, + }); + state.repos[entry.id] = { id: entry.id, repository: entry.repository, failure_policy: entry.failure_policy || 'block', skipped: true, check_run_id: check.id, remote_conclusion: 'skipped', conclusion: 'skipped' }; + } +} + +function writeSummary(target, state) { + const rows = Object.values(state.repos).map((s) => + `| ${code(s.id)} | ${s.repository} | ${code(s.ext_ref || '-')} | ${s.failure_policy} | ${s.carried_forward ? 'carried forward' : s.remote_conclusion} | ${s.conclusion} | ${s.remote_run_url ? `[run](${s.remote_run_url})` : (s.error ? code(s.error) : '')} |`); + const md = [ + `### External CI for ${code(`${target.headRepo}@${short(target.headSha)}`)}${target.pr ? ` (PR #${target.pr})` : ` (${target.headBranch})`} — attempt ${attempt}`, + '', '| id | repository | ext ref | policy | remote | check | link |', '|---|---|---|---|---|---|---|', ...rows, '', + ].join('\n'); + if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, md); + console.log(md); + fs.writeFileSync(RESULTS_FILE, JSON.stringify({ target, attempt, orchestrator_url: orchestratorUrl, repos: state.repos }, null, 2)); +} + +// ---------------------------------------------------------------- modes +async function run() { + const target = await resolveTarget(); + console.log(`target: ${JSON.stringify(target)}`); + const manifest = await loadManifest(target.baseRef); + // Fork preflight: a fork of F´ that copies this workflow must not dispatch into the ext repos. + if (manifest.upstream_repository && manifest.upstream_repository !== repo && eventName !== 'workflow_dispatch') { + console.log(`::notice::${repo} is not ${manifest.upstream_repository}; external CI is only dispatched from upstream. Use workflow_dispatch to run it here deliberately.`); + return; + } + const { carried, dispatch } = await selectRepos(manifest, target); + const state = { target, attempt, repos: {} }; + saveState(state); + for (const c of carried) await carryForward(manifest, target, c, state); + if (eventName !== 'workflow_dispatch' && !relevantChange(await changedFiles(target), manifest)) { + console.log(`no relevant files changed; marking [${dispatch.map((e) => e.id)}] as skipped`); + await skipAll(manifest, target, dispatch, state); + writeSummary(target, state); + return; + } + console.log(`dispatch: [${dispatch.map((e) => e.id)}] carried forward: [${carried.map((c) => c.entry.id)}]`); + for (const entry of dispatch) await dispatchOne(manifest, entry, target, state); + await correlate(state); + await pollAll(state); + for (const s of Object.values(state.repos)) { + if (s.carried_forward) continue; + try { await finalizeOne(target, s); } + catch (e) { console.log(`::error::[${s.id}] finalize: ${e.message}`); s.conclusion = s.conclusion || 'failure'; } + saveState(state); + } + writeSummary(target, state); + const blocking = Object.values(state.repos).filter((s) => s.failure_policy !== 'notify' && s.conclusion !== 'success' && s.conclusion !== 'skipped'); + if (blocking.length) { + console.log(`::error::blocking external repos failed: ${blocking.map((s) => s.id).join(', ')}`); + process.exit(1); + } +} + +// Runs under `if: always()`: whatever the run step left open (crash, cancellation, timeout) is +// closed here so no Check Run stays in_progress and no remote run keeps burning runners. +async function finalize() { + const state = loadState(); + if (!state) return console.log('no state file; nothing to finalize'); + const cancelled = env.JOB_STATUS === 'cancelled'; + for (const s of Object.values(state.repos)) { + if (s.carried_forward || s.skipped || s.conclusion) continue; + if (s.remote_run_id && !s.remote_conclusion) { + console.log(`[${s.id}] cancelling ${s.remote_run_url}`); + await ext.post(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/cancel`).catch((e) => console.log(` ${e.message}`)); + } + if (s.check_run_id) { + const current = await own.get(`/repos/${repo}/check-runs/${s.check_run_id}`).catch(() => null); + if (current && current.status === 'completed') continue; + const conclusion = cancelled ? 'cancelled' : (s.failure_policy === 'notify' ? 'neutral' : 'failure'); + const why = cancelled ? 'The orchestrator run was cancelled' : `The orchestrator run ended before this repo was reported (${code(s.error || 'orchestrator error')})`; + await own.patch(`/repos/${repo}/check-runs/${s.check_run_id}`, { + status: 'completed', conclusion, + output: { title: cancelled ? 'Cancelled by orchestrator' : 'Orchestrator error', summary: `${why}; remote run ${s.remote_run_url || '(not started)'} was cancelled.\n\n${orchestratorUrl}` }, + }).catch((e) => console.log(` ${e.message}`)); + } + } +} + +await (mode === 'finalize' ? finalize() : run()); diff --git a/.github/workflows/ext-ci-trigger.yml b/.github/workflows/ext-ci-trigger.yml new file mode 100644 index 00000000000..fd8bd281977 --- /dev/null +++ b/.github/workflows/ext-ci-trigger.yml @@ -0,0 +1,21 @@ +name: ext-ci-trigger + +# Deliberately trivial. This workflow runs under the repository's normal `pull_request` +# rules (including "require approval for workflows from forks"), and its completion is what +# starts the privileged orchestrator in ext-ci.yml via `workflow_run`. It must never need +# secrets and must never produce anything the orchestrator trusts. No paths filter here: the +# orchestrator applies the manifest's paths_ignore itself and emits skipped Check Runs, so +# docs-only PRs still satisfy required checks. +on: + pull_request: + branches: [ devel, release/** ] + merge_group: + +permissions: {} + +jobs: + approved: + name: "Gate passed" + runs-on: ubuntu-22.04 + steps: + - run: echo "${GITHUB_EVENT_NAME} workflow ran for ${GITHUB_SHA}; ext-ci may proceed" diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml new file mode 100644 index 00000000000..44311034855 --- /dev/null +++ b/.github/workflows/ext-ci.yml @@ -0,0 +1,74 @@ +name: ext-ci + +on: + # PR path: fires after ext-ci-trigger completed, i.e. after GitHub's own approval gate. + # The workflow file and the checked-out scripts come from the default branch, never from the PR. + workflow_run: + workflows: [ ext-ci-trigger ] + types: [ completed ] + push: + branches: [ devel, release/** ] + workflow_dispatch: + inputs: + repos: + description: "Comma-separated manifest ids to run, or 'all'" + type: string + default: all + pr_number: + description: "PR to test and report on (takes precedence over fprime_ref)" + type: string + default: "" + fprime_ref: + description: "Branch/SHA to test when no PR is given (default: the branch this run was started from)" + type: string + default: "" + force_all: + description: "On a re-run, re-dispatch every selected repo instead of only the failed ones" + type: boolean + default: false + +# One orchestrator per F´ head (repo + branch), whatever started it; a newer push cancels the +# older fan-out and the finalize step below propagates that to the remote runs. +concurrency: + group: ext-ci-${{ github.event.workflow_run.head_repository.full_name || github.repository }}-${{ github.event.workflow_run.head_branch || inputs.pr_number || inputs.fprime_ref || github.ref_name }} + cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }} + +permissions: + contents: read # manifest + checks: write # per-repo Check Runs + pull-requests: write # notify-policy PR comments + issues: write # notify-policy tracking issues + +jobs: + orchestrate: + name: "External CI" + # Only chain from a trigger that actually ran (not skipped/cancelled) for a PR or merge group. + if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) + runs-on: ubuntu-22.04 + timeout-minutes: 180 + env: + GITHUB_TOKEN: ${{ github.token }} + DISPATCH_TOKEN: ${{ secrets.EXT_CI_DISPATCH_TOKEN }} + steps: + - uses: actions/checkout@v4 + - name: "Dispatch, poll and report" + env: + INPUT_REPOS: ${{ inputs.repos }} + INPUT_PR_NUMBER: ${{ inputs.pr_number }} + INPUT_FPRIME_REF: ${{ inputs.fprime_ref }} + INPUT_FORCE_ALL: ${{ inputs.force_all }} + POLL_INTERVAL_SECONDS: 60 + run: node .github/scripts/ext-ci-orchestrator.mjs run + - name: "Finalize (close open checks, cancel orphaned remote runs)" + if: always() + env: + JOB_STATUS: ${{ job.status }} + run: node .github/scripts/ext-ci-orchestrator.mjs finalize + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ext-ci-results-attempt-${{ github.run_attempt }} + path: | + ext-ci-results.json + ext-ci-state.json + if-no-files-found: ignore From b1f8570f7a65d7ef6ca1e94b370c2045a41f3da4 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Tue, 20 Oct 2026 07:02:41 +0000 Subject: [PATCH 02/22] ci: lazy DISPATCH_TOKEN check so the fork preflight runs without the secret --- .github/scripts/ext-ci-orchestrator.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/ext-ci-orchestrator.mjs b/.github/scripts/ext-ci-orchestrator.mjs index 8c1f7a3e43e..28f32c9c80d 100644 --- a/.github/scripts/ext-ci-orchestrator.mjs +++ b/.github/scripts/ext-ci-orchestrator.mjs @@ -37,9 +37,9 @@ const code = (s) => '`' + String(s).replace(/[`\r\n]/g, ' ') + '`'; class ApiError extends Error { constructor(msg, status) { super(msg); this.status = status; } } function client(token, label) { - if (!token) throw new Error(`${label} token missing`); const etags = new Map(); async function request(method, url, body, { conditional = false } = {}) { + if (!token) throw new ApiError(`${label} token missing (secret not configured?)`, 0); const full = url.startsWith('http') ? url : API + url; const headers = { authorization: `Bearer ${token}`, From 01b8673ddc1987683be2c08f4f2ce2cd43e682ce Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Tue, 20 Oct 2026 07:03:20 +0000 Subject: [PATCH 03/22] ci: ext-ci nightly schedule; cancel-in-progress only for PR fan-outs --- .github/workflows/ext-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index 44311034855..f911a8ef9b8 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -7,7 +7,9 @@ on: workflows: [ ext-ci-trigger ] types: [ completed ] push: - branches: [ devel, release/** ] + branches: [ release/** ] + schedule: + - cron: "0 21 * * *" # Nightly at 2:00 PM US/Pacific (PDT), as the ext-*.yml workflows today workflow_dispatch: inputs: repos: @@ -27,11 +29,11 @@ on: type: boolean default: false -# One orchestrator per F´ head (repo + branch), whatever started it; a newer push cancels the -# older fan-out and the finalize step below propagates that to the remote runs. +# One orchestrator per F´ head (repo + branch); a newer PR push cancels the older PR fan-out and +# the finalize step below propagates that to the remote runs. Branch/nightly runs are never cancelled. concurrency: group: ext-ci-${{ github.event.workflow_run.head_repository.full_name || github.repository }}-${{ github.event.workflow_run.head_branch || inputs.pr_number || inputs.fprime_ref || github.ref_name }} - cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }} + cancel-in-progress: ${{ github.event_name == 'workflow_run' }} permissions: contents: read # manifest From 76bb7331cbc4c459036fb6d1adfae09751e0badd Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Tue, 20 Oct 2026 16:29:34 +0000 Subject: [PATCH 04/22] ci: create the per-repo Check Run before touching the external repo so dispatch errors are visible --- .github/scripts/ext-ci-orchestrator.mjs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.mjs b/.github/scripts/ext-ci-orchestrator.mjs index 28f32c9c80d..9ef510e7c9f 100644 --- a/.github/scripts/ext-ci-orchestrator.mjs +++ b/.github/scripts/ext-ci-orchestrator.mjs @@ -255,16 +255,18 @@ async function dispatchOne(manifest, entry, target, state) { check_run_id: null, remote_run_id: null, remote_run_url: null, remote_started_at: null, remote_conclusion: null, conclusion: null, }; try { - s.ext_ref = await resolveExtRef(entry, target); + // Check Run first, so a credential or API problem on the ext side still ends in a visible + // failed check rather than a required check that never appears. const check = await own.post(`/repos/${repo}/check-runs`, { name, head_sha: target.headSha, status: 'queued', external_id: ourExternalId(entry.id), details_url: orchestratorUrl, output: { - title: `Dispatching ${entry.repository} @ ${s.ext_ref}`, - summary: `Dispatching ${code(entry.workflow)} on ${code(entry.repository)} @ ${code(s.ext_ref)} with F´ ${code(`${target.headRepo}@${short(target.headSha)}`)}.\n\nOrchestrator: ${orchestratorUrl}`, + title: `Dispatching ${entry.repository}`, + summary: `Dispatching ${code(entry.workflow)} on ${code(entry.repository)} with F´ ${code(`${target.headRepo}@${short(target.headSha)}`)}.\n\nOrchestrator: ${orchestratorUrl}`, }, }); s.check_run_id = check.id; saveState(state); + s.ext_ref = await resolveExtRef(entry, target); const r = await ext.post(`/repos/${entry.repository}/actions/workflows/${entry.workflow}/dispatches`, { ref: s.ext_ref, inputs: { From ab930cf10adccdfcd685a13f0d6cb97e2cfe06ed Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Wed, 4 Nov 2026 22:59:19 +0000 Subject: [PATCH 05/22] ci: ext-ci manifest lists led-blinker only for the first dispatch pilot --- .github/ext-ci-manifest.json | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/.github/ext-ci-manifest.json b/.github/ext-ci-manifest.json index 6dfe5514c05..b20effe6f25 100644 --- a/.github/ext-ci-manifest.json +++ b/.github/ext-ci-manifest.json @@ -16,27 +16,9 @@ "default_ref": "devel", "timeout_minutes": 60, "failure_policy": "notify", - "owners": ["thomas-bc"], - "on_pr": true - }, - { - "id": "zephyr-reference", - "repository": "fprime-community/fprime-zephyr-reference", - "workflow": "ci.yml", - "default_ref": "devel", - "timeout_minutes": 90, - "failure_policy": "notify", - "owners": ["thomas-bc"], - "on_pr": true - }, - { - "id": "generic-hub-reference", - "repository": "fprime-community/fprime-generic-hub-reference", - "workflow": "ci.yml", - "default_ref": "devel", - "timeout_minutes": 60, - "failure_policy": "notify", - "owners": ["thomas-bc"], + "owners": [ + "thomas-bc" + ], "on_pr": true } ] From a18ed2c1aa466fa5a86913b373797449174aa264 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Wed, 4 Nov 2026 23:11:02 +0000 Subject: [PATCH 06/22] ci: ext-ci orchestrator in Python (stdlib only) instead of Node Same behaviour and workflows; folds in review findings: no automatic retry of the non-idempotent dispatch call (a lost response falls back to dispatch_id correlation), a failed check-status update is not treated as a failed dispatch, and a check whose completion update failed is closed by the finalize step with the remote result. --- .github/scripts/ext-ci-orchestrator.mjs | 547 ----------------- .github/scripts/ext-ci-orchestrator.py | 745 ++++++++++++++++++++++++ .github/workflows/ext-ci.yml | 4 +- 3 files changed, 747 insertions(+), 549 deletions(-) delete mode 100644 .github/scripts/ext-ci-orchestrator.mjs create mode 100644 .github/scripts/ext-ci-orchestrator.py diff --git a/.github/scripts/ext-ci-orchestrator.mjs b/.github/scripts/ext-ci-orchestrator.mjs deleted file mode 100644 index 9ef510e7c9f..00000000000 --- a/.github/scripts/ext-ci-orchestrator.mjs +++ /dev/null @@ -1,547 +0,0 @@ -#!/usr/bin/env node -// External-CI orchestrator (dependency-free Node >= 20). -// -// node ext-ci-orchestrator.mjs run dispatch, poll, report -// node ext-ci-orchestrator.mjs finalize close anything still open (runs under if: always()) -// -// Two credentials, never mixed: -// GITHUB_TOKEN this repo only (checks/pull-requests/issues write) - reports results -// DISPATCH_TOKEN external repos only (actions: write) - starts remote runs -import fs from 'node:fs'; - -const env = process.env; -const mode = process.argv[2] ?? 'run'; -const API = env.GITHUB_API_URL || 'https://api.github.com'; -const SERVER = env.GITHUB_SERVER_URL || 'https://github.com'; -const STATE_FILE = 'ext-ci-state.json'; -const RESULTS_FILE = 'ext-ci-results.json'; -const MANIFEST_PATH = '.github/ext-ci-manifest.json'; -const POLL_MS = Number(env.POLL_INTERVAL_SECONDS || 60) * 1000; -const CORRELATE_TIMEOUT_MS = 5 * 60 * 1000; -const BOT_LOGIN = 'github-actions[bot]'; -const NOTIFY_LABEL = 'ext-ci'; - -const repo = env.GITHUB_REPOSITORY; -const runId = env.GITHUB_RUN_ID; -const attempt = Number(env.GITHUB_RUN_ATTEMPT || 1); -const eventName = env.GITHUB_EVENT_NAME; -const event = JSON.parse(fs.readFileSync(env.GITHUB_EVENT_PATH, 'utf8')); -const orchestratorUrl = `${SERVER}/${repo}/actions/runs/${runId}${attempt > 1 ? `/attempts/${attempt}` : ''}`; - -const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); -const short = (sha) => sha.slice(0, 7); -// Remote-controlled text (job/step names, branch names) is only ever rendered inside code spans. -const code = (s) => '`' + String(s).replace(/[`\r\n]/g, ' ') + '`'; - -// ---------------------------------------------------------------- REST helpers -class ApiError extends Error { constructor(msg, status) { super(msg); this.status = status; } } - -function client(token, label) { - const etags = new Map(); - async function request(method, url, body, { conditional = false } = {}) { - if (!token) throw new ApiError(`${label} token missing (secret not configured?)`, 0); - const full = url.startsWith('http') ? url : API + url; - const headers = { - authorization: `Bearer ${token}`, - accept: 'application/vnd.github+json', - 'x-github-api-version': '2022-11-28', - 'user-agent': 'fprime-ext-ci-orchestrator', - ...(body ? { 'content-type': 'application/json' } : {}), - }; - if (conditional && etags.has(full)) headers['if-none-match'] = etags.get(full).etag; - for (let i = 0; ; i++) { - let res; - try { - res = await fetch(full, { method, headers, body: body ? JSON.stringify(body) : undefined }); - } catch (e) { - if (i >= 4) throw new ApiError(`${label}: ${method} ${url} -> network error ${e.message}`, 0); - await sleep(2000 * 2 ** i); continue; - } - if (res.status === 304 && conditional) return { status: 200, ok: true, data: etags.get(full).data, cached: true }; - const text = await res.text(); - let data = null; - try { data = text ? JSON.parse(text) : null; } catch { data = text; } - const remaining = Number(res.headers.get('x-ratelimit-remaining')); - const limited = res.status === 429 || (res.status === 403 && (remaining === 0 || /rate limit/i.test(text))); - if ((limited || res.status >= 500) && i < 4) { - let wait = 2000 * 2 ** i; - const ra = Number(res.headers.get('retry-after')); - const reset = Number(res.headers.get('x-ratelimit-reset')); - if (ra) wait = ra * 1000; - else if (limited && reset) wait = Math.max(wait, reset * 1000 - Date.now() + 1000); - console.log(`::warning::${label}: ${method} ${url} -> ${res.status}; retrying in ${Math.round(wait / 1000)} s`); - await sleep(Math.min(wait, 15 * 60_000)); continue; - } - if (res.ok && conditional && res.headers.get('etag')) etags.set(full, { etag: res.headers.get('etag'), data }); - return { status: res.status, ok: res.ok, data }; - } - } - async function api(method, url, body, opts) { - const r = await request(method, url, body, opts); - if (!r.ok) throw new ApiError(`${label}: ${method} ${url} -> ${r.status} ${JSON.stringify(r.data).slice(0, 300)}`, r.status); - return r.data; - } - return { - get: (u, opts) => api('GET', u, undefined, opts), - post: (u, b) => api('POST', u, b), - patch: (u, b) => api('PATCH', u, b), - // true/false for 200/404; anything else is an error, never silently "absent". - exists: async (u) => { - const r = await request('GET', u); - if (r.status === 200) return true; - if (r.status === 404) return false; - throw new ApiError(`${label}: GET ${u} -> ${r.status}`, r.status); - }, - }; -} -const own = client(env.GITHUB_TOKEN, 'GITHUB_TOKEN'); -const ext = client(env.DISPATCH_TOKEN, 'DISPATCH_TOKEN'); - -// ---------------------------------------------------------------- target resolution -// What F´ revision are we testing, and which PR (if any) gets the Check Runs? -async function resolveTarget() { - const t = { pr: null, headRepo: repo, headSha: null, headBranch: null, baseRef: null, beforeSha: null }; - if (eventName === 'workflow_run') { - const wr = event.workflow_run; - // Trust only the event payload: nothing produced by the (untrusted) trigger run is read. - t.headSha = wr.head_sha; - t.headRepo = wr.head_repository.full_name; - t.headBranch = wr.head_branch; - if (wr.event === 'merge_group') { - // Merge-queue branch gh-readonly-queue//pr--: checks go on the queue SHA, no PR. - const m = /^gh-readonly-queue\/(.+?)\/pr-\d+-/.exec(t.headBranch || ''); - t.baseRef = m ? m[1] : env.GITHUB_REF_NAME; - return t; - } - // workflow_run.pull_requests is empty for fork PRs; find the PR by commit, then by head ref. - let pr = (await own.get(`/repos/${repo}/commits/${t.headSha}/pulls?per_page=20`)) - .find((p) => p.state === 'open' && p.head.sha === t.headSha && p.base.repo.full_name === repo); - if (!pr) { - const owner = t.headRepo.split('/')[0]; - pr = (await own.get(`/repos/${repo}/pulls?state=open&head=${encodeURIComponent(`${owner}:${t.headBranch}`)}`)) - .find((p) => p.head.sha === t.headSha); - } - if (!pr) throw new Error(`no open PR found for ${t.headRepo}@${t.headSha} (${t.headBranch}); the PR may have been updated or closed`); - t.pr = pr.number; - t.baseRef = pr.base.ref; - } else if (eventName === 'workflow_dispatch') { - const prNumber = (env.INPUT_PR_NUMBER || '').trim(); - if (prNumber) { - if (!/^\d+$/.test(prNumber)) throw new Error(`pr_number must be numeric, got ${JSON.stringify(prNumber)}`); - const pr = await own.get(`/repos/${repo}/pulls/${prNumber}`); - t.pr = pr.number; t.headSha = pr.head.sha; t.headRepo = pr.head.repo.full_name; - t.headBranch = pr.head.ref; t.baseRef = pr.base.ref; - } else { - const ref = (env.INPUT_FPRIME_REF || '').trim() || env.GITHUB_REF_NAME; - const commit = await own.get(`/repos/${repo}/commits/${encodeURIComponent(ref)}`); - t.headSha = commit.sha; t.headBranch = ref; t.baseRef = ref; - } - } else { // push, schedule - t.headSha = env.GITHUB_SHA; t.headBranch = env.GITHUB_REF_NAME; t.baseRef = env.GITHUB_REF_NAME; - if (eventName === 'push' && event.before && !/^0+$/.test(event.before)) t.beforeSha = event.before; - } - return t; -} - -// Manifest is read from the base branch of the PR (the orchestrator itself always runs from -// the default branch), so release/** PRs use their own branch's manifest. Only a missing file -// falls back to the checked-out copy; any other error is fatal. -async function loadManifest(ref) { - try { - const file = await own.get(`/repos/${repo}/contents/${MANIFEST_PATH}?ref=${encodeURIComponent(ref)}`); - return JSON.parse(Buffer.from(file.content, 'base64').toString('utf8')); - } catch (e) { - if (!(e instanceof ApiError && e.status === 404)) throw e; - console.log(`::warning::manifest missing at ${ref}; using the checked-out copy`); - return JSON.parse(fs.readFileSync(MANIFEST_PATH, 'utf8')); - } -} - -const checkName = (manifest, id) => `${manifest.check_prefix || 'ext-ci'} / ${id}`; -const ourExternalId = (id) => `${runId}-${id}`; // same across attempts of one run - -// ---------------------------------------------------------------- change filter -// paths-ignore lives here, not on the trigger workflow, so that filtered PRs still get a -// (skipped) Check Run per repo and required checks can be satisfied. -function globToRegExp(glob) { - let re = '^'; - for (let i = 0; i < glob.length; i++) { - const c = glob[i]; - if (c === '*') { - if (glob[i + 1] === '*') { re += '.*'; i++; if (glob[i + 1] === '/') i++; } - else re += '[^/]*'; - } else if (c === '?') re += '[^/]'; - else re += c.replace(/[.+^${}()|[\]\\]/g, '\\$&'); - } - return new RegExp(re + '$'); -} - -async function changedFiles(target) { - let url = null; - if (target.pr) url = `/repos/${repo}/pulls/${target.pr}/files?per_page=100`; - else if (target.beforeSha) url = `/repos/${repo}/compare/${target.beforeSha}...${target.headSha}?per_page=100`; - if (!url) return null; // unknown -> run everything - const files = []; - for (let page = 1; page <= 30; page++) { - const r = await own.get(`${url}&page=${page}`); - const batch = target.pr ? r : (r.files || []); - files.push(...batch.map((f) => f.filename)); - if (batch.length < 100) break; - } - return files; -} - -function relevantChange(files, manifest) { - const ignore = (manifest.paths_ignore || []).map(globToRegExp); - if (!files || !ignore.length) return true; - return files.some((f) => !ignore.some((re) => re.test(f))); -} - -// ---------------------------------------------------------------- selection -async function latestOwnCheckRun(name, id, sha) { - const r = await own.get(`/repos/${repo}/commits/${sha}/check-runs?check_name=${encodeURIComponent(name)}&filter=all&per_page=50`); - // Only reuse a check this very orchestrator run created (external_id stamped with our run id). - return r.check_runs.filter((c) => c.external_id === ourExternalId(id) && c.app && c.app.slug === 'github-actions') - .sort((a, b) => b.id - a.id)[0] ?? null; -} - -async function selectRepos(manifest, target) { - let repos = manifest.repos; - const wanted = (env.INPUT_REPOS || 'all').trim(); - if (eventName === 'workflow_dispatch' && wanted && wanted !== 'all') { - const ids = wanted.split(',').map((s) => s.trim()).filter(Boolean); - const unknown = ids.filter((id) => !repos.some((r) => r.id === id)); - if (unknown.length) throw new Error(`unknown manifest ids: ${unknown.join(', ')}`); - repos = repos.filter((r) => ids.includes(r.id)); - } else if (target.pr) { - repos = repos.filter((r) => r.on_pr !== false); - } - // Failed-only re-run: on attempt > 1 keep every repo whose Check Run from attempt 1..n-1 of - // this same run already succeeded, and dispatch only the rest. - const carried = []; - const dispatch = []; - const forceAll = (env.INPUT_FORCE_ALL || '').toString() === 'true'; - for (const entry of repos) { - const prior = attempt > 1 && !forceAll ? await latestOwnCheckRun(checkName(manifest, entry.id), entry.id, target.headSha) : null; - if (prior && prior.status === 'completed' && prior.conclusion === 'success') carried.push({ entry, prior }); - else dispatch.push(entry); - } - return { carried, dispatch }; -} - -// pr- branch on the ext repo, else the F´ base branch name, else the manifest default. -async function resolveExtRef(entry, target) { - const candidates = []; - if (target.pr) candidates.push(`pr-${target.pr}`); - if (target.baseRef) candidates.push(target.baseRef); - for (const c of candidates) { - if (await ext.exists(`/repos/${entry.repository}/branches/${encodeURIComponent(c)}`)) return c; - } - return entry.default_ref || 'main'; -} - -// ---------------------------------------------------------------- state -function saveState(state) { fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2)); } -function loadState() { return fs.existsSync(STATE_FILE) ? JSON.parse(fs.readFileSync(STATE_FILE, 'utf8')) : null; } - -// ---------------------------------------------------------------- dispatch -async function dispatchOne(manifest, entry, target, state) { - const dispatchId = `${runId}-${attempt}-${entry.id}`; - const name = checkName(manifest, entry.id); - const s = state.repos[entry.id] = { - id: entry.id, repository: entry.repository, workflow: entry.workflow, ext_ref: null, - failure_policy: entry.failure_policy || 'block', owners: entry.owners || [], - timeout_minutes: entry.timeout_minutes || 60, dispatch_id: dispatchId, dispatched_at: new Date().toISOString(), - check_run_id: null, remote_run_id: null, remote_run_url: null, remote_started_at: null, remote_conclusion: null, conclusion: null, - }; - try { - // Check Run first, so a credential or API problem on the ext side still ends in a visible - // failed check rather than a required check that never appears. - const check = await own.post(`/repos/${repo}/check-runs`, { - name, head_sha: target.headSha, status: 'queued', external_id: ourExternalId(entry.id), details_url: orchestratorUrl, - output: { - title: `Dispatching ${entry.repository}`, - summary: `Dispatching ${code(entry.workflow)} on ${code(entry.repository)} with F´ ${code(`${target.headRepo}@${short(target.headSha)}`)}.\n\nOrchestrator: ${orchestratorUrl}`, - }, - }); - s.check_run_id = check.id; - saveState(state); - s.ext_ref = await resolveExtRef(entry, target); - const r = await ext.post(`/repos/${entry.repository}/actions/workflows/${entry.workflow}/dispatches`, { - ref: s.ext_ref, - inputs: { - fprime_repository: target.headRepo, fprime_ref: target.headSha, - fprime_pr_number: target.pr ? String(target.pr) : '', dispatch_id: dispatchId, - }, - return_run_details: true, - }); - if (r && r.workflow_run_id) { - s.remote_run_id = r.workflow_run_id; s.remote_run_url = r.html_url; - await markRunning(s); - } - console.log(`[${entry.id}] dispatched ${entry.repository}/${entry.workflow} @ ${s.ext_ref}${s.remote_run_url ? ` -> ${s.remote_run_url}` : ` (dispatch_id=${dispatchId})`}`); - } catch (e) { - s.error = `dispatch failed: ${e.message}`; - s.remote_conclusion = 'dispatch_failed'; - console.log(`::error::[${entry.id}] ${s.error}`); - } - saveState(state); -} - -async function markRunning(s) { - await own.patch(`/repos/${repo}/check-runs/${s.check_run_id}`, { - status: 'in_progress', details_url: s.remote_run_url, - output: { title: `Running on ${s.repository} @ ${s.ext_ref}`, summary: `Remote run: ${s.remote_run_url}\n\nOrchestrator: ${orchestratorUrl}` }, - }); -} - -// Fallback only for API servers that still answer 204 to the dispatch call: find the run whose -// title carries our dispatch_id (the ext workflow echoes it into run-name). -async function correlate(state) { - const pending = Object.values(state.repos).filter((s) => !s.remote_run_id && !s.remote_conclusion); - const deadline = Date.now() + CORRELATE_TIMEOUT_MS; - while (pending.length && Date.now() < deadline) { - for (const s of [...pending]) { - try { - const since = new Date(new Date(s.dispatched_at).getTime() - 60_000).toISOString(); - const runs = await ext.get(`/repos/${s.repository}/actions/runs?event=workflow_dispatch&created=${encodeURIComponent('>=' + since)}&per_page=50`); - const run = runs.workflow_runs.find((r) => (r.display_title || r.name || '').split(/\s+/).includes(s.dispatch_id)); - if (!run) continue; - s.remote_run_id = run.id; s.remote_run_url = run.html_url; - pending.splice(pending.indexOf(s), 1); - console.log(`[${s.id}] remote run ${run.html_url}`); - await markRunning(s); - saveState(state); - } catch (e) { console.log(`::warning::[${s.id}] correlate: ${e.message}`); } - } - if (pending.length) await sleep(15_000); - } - for (const s of pending) { - s.remote_conclusion = 'not_found'; - s.error = `no run with dispatch_id ${s.dispatch_id} appeared on ${s.repository} within ${CORRELATE_TIMEOUT_MS / 60000} min`; - console.log(`::error::[${s.id}] ${s.error}`); - } - saveState(state); -} - -// ---------------------------------------------------------------- poll -async function pollAll(state) { - const active = () => Object.values(state.repos).filter((s) => s.remote_run_id && !s.remote_conclusion); - while (active().length) { - await sleep(POLL_MS); - for (const s of active()) { - try { - const run = await ext.get(`/repos/${s.repository}/actions/runs/${s.remote_run_id}`, { conditional: true }); - if (run.run_started_at && !s.remote_started_at) s.remote_started_at = run.run_started_at; - if (run.status === 'completed') { - s.remote_conclusion = run.conclusion; - console.log(`[${s.id}] completed: ${run.conclusion}`); - } else if (s.remote_started_at && Date.now() - new Date(s.remote_started_at).getTime() > s.timeout_minutes * 60_000) { - // Timeout counts from when the remote run started, not from dispatch (queue time is not the ext repo's fault). - console.log(`::warning::[${s.id}] timed out after ${s.timeout_minutes} min of running; cancelling remote run`); - await ext.post(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/cancel`).catch(() => {}); - s.remote_conclusion = 'timed_out'; - } else if (!s.remote_started_at && Date.now() - new Date(s.dispatched_at).getTime() > 2 * s.timeout_minutes * 60_000) { - console.log(`::warning::[${s.id}] still queued after ${2 * s.timeout_minutes} min; giving up`); - await ext.post(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/cancel`).catch(() => {}); - s.remote_conclusion = 'queue_timeout'; - s.error = 'remote run never started (no runner available?)'; - } - } catch (e) { - // A single bad poll must not take the whole fan-out down; give up on this repo only - // when the credential is gone or the run vanished. - console.log(`::warning::[${s.id}] poll: ${e.message}`); - if (e instanceof ApiError && (e.status === 401 || e.status === 404)) { - s.remote_conclusion = 'error'; s.error = `poll failed: ${e.message}`; - } - } - } - saveState(state); - } -} - -// ---------------------------------------------------------------- report -async function failureDetails(s) { - if (!s.remote_run_id) return ''; - try { - const jobs = await ext.get(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/jobs?per_page=100`); - const bad = jobs.jobs.filter((j) => j.conclusion && j.conclusion !== 'success' && j.conclusion !== 'skipped'); - return bad.map((j) => { - const steps = (j.steps || []).filter((st) => st.conclusion && st.conclusion !== 'success' && st.conclusion !== 'skipped') - .map((st) => ` - step ${code(st.name)}: ${st.conclusion}`).join('\n'); - return `- [${code(j.name)}](${j.html_url}): **${j.conclusion}**${steps ? '\n' + steps : ''}`; - }).join('\n'); - } catch (e) { return `_(job details unavailable: ${code(e.message)})_`; } -} - -function checkConclusion(s) { - if (s.remote_conclusion === 'success') return 'success'; - if (s.failure_policy === 'notify') return 'neutral'; - if (s.remote_conclusion === 'cancelled') return 'cancelled'; - if (s.remote_conclusion === 'timed_out' || s.remote_conclusion === 'queue_timeout') return 'timed_out'; - return 'failure'; -} - -async function finalizeOne(target, s) { - s.conclusion = checkConclusion(s); - const details = s.remote_conclusion === 'success' ? '' : await failureDetails(s); - const header = `${code(s.repository)} @ ${code(s.ext_ref || '-')} with F´ ${code(`${target.headRepo}@${short(target.headSha)}`)}: **${s.remote_conclusion}**`; - const lines = [header, '']; - if (s.remote_run_url) lines.push(`Remote run: ${s.remote_run_url}`); - if (s.error) lines.push(`Error: ${code(s.error)}`); - if (s.failure_policy === 'notify' && s.remote_conclusion !== 'success') { - lines.push('', `This repo is ${code('failure_policy: notify')}: the failure does not block the PR; owners (${s.owners.map((o) => '@' + o).join(' ')}) have been notified.`); - } - lines.push('', `Orchestrator run (use "Re-run failed jobs" there to re-dispatch only the failed repos): ${orchestratorUrl}`); - if (s.check_run_id) { - await own.patch(`/repos/${repo}/check-runs/${s.check_run_id}`, { - status: 'completed', conclusion: s.conclusion, details_url: s.remote_run_url || orchestratorUrl, - output: { title: `${s.remote_conclusion} (${s.repository} @ ${s.ext_ref || '-'})`, summary: lines.join('\n'), text: details || undefined }, - }); - } - if (s.failure_policy === 'notify') await notifyOwners(target, s, details).catch((e) => console.log(`::warning::[${s.id}] notify: ${e.message}`)); -} - -// notify policy: one PR comment per ext repo (updated in place), or one tracking issue when -// there is no PR (closed again once the repo passes). Only comments/issues authored by the -// Actions bot are considered ours, so nobody can plant a marker to suppress notifications. -async function ensureLabel() { - if (await own.exists(`/repos/${repo}/labels/${NOTIFY_LABEL}`)) return; - await own.post(`/repos/${repo}/labels`, { name: NOTIFY_LABEL, color: 'ededed', description: 'Opened by the external-CI orchestrator' }).catch(() => {}); -} - -async function notifyOwners(target, s, details) { - const marker = ``; - const mentions = s.owners.map((o) => '@' + o).join(' '); - const failed = s.remote_conclusion !== 'success'; - const ours = (x) => x.user && x.user.login === BOT_LOGIN && x.body && x.body.includes(marker); - if (target.pr) { - let existing = null; - for (let page = 1; page <= 10 && !existing; page++) { - const comments = await own.get(`/repos/${repo}/issues/${target.pr}/comments?per_page=100&page=${page}`); - existing = comments.find(ours) || null; - if (comments.length < 100) break; - } - if (!failed && !existing) return; - const body = failed - ? `${marker}\n### External CI: ${code(s.id)} failed (non-blocking)\n\n${code(s.repository)} @ ${code(s.ext_ref)} does not pass against ${short(target.headSha)}: ${s.remote_run_url || code(s.error)}\n\n${details}\n\n${mentions} — this repo is ${code('failure_policy: notify')}, so the PR is not blocked, but it needs a look.` - : `${marker}\n### External CI: ${code(s.id)} passing again\n\n${code(s.repository)} @ ${code(s.ext_ref)} passes against ${short(target.headSha)}: ${s.remote_run_url}`; - if (existing) await own.patch(`/repos/${repo}/issues/comments/${existing.id}`, { body }); - else await own.post(`/repos/${repo}/issues/${target.pr}/comments`, { body }); - return; - } - const title = `[ext-ci] ${s.id} failing on ${target.headBranch}`; - await ensureLabel(); - const issues = await own.get(`/repos/${repo}/issues?state=open&labels=${NOTIFY_LABEL}&creator=${encodeURIComponent(BOT_LOGIN)}&per_page=100`); - const existing = issues.find((i) => !i.pull_request && ours(i) && i.title === title); - if (failed) { - const body = `${marker}\n${code(s.repository)} @ ${code(s.ext_ref)} fails against ${code(target.headBranch)} (${short(target.headSha)}): ${s.remote_run_url || code(s.error)}\n\n${details}\n\n${mentions}\n\nThis issue is closed automatically once the repo passes again.`; - if (existing) await own.post(`/repos/${repo}/issues/${existing.number}/comments`, { body: `Still failing at ${short(target.headSha)}: ${s.remote_run_url || code(s.error)}` }); - else await own.post(`/repos/${repo}/issues`, { title, body, labels: [NOTIFY_LABEL] }); - } else if (existing) { - await own.post(`/repos/${repo}/issues/${existing.number}/comments`, { body: `Passing again at ${short(target.headSha)}: ${s.remote_run_url}` }); - await own.patch(`/repos/${repo}/issues/${existing.number}`, { state: 'closed', state_reason: 'completed' }); - } -} - -async function carryForward(manifest, target, { entry, prior }, state) { - state.repos[entry.id] = { - id: entry.id, repository: entry.repository, failure_policy: entry.failure_policy || 'block', carried_forward: true, - check_run_id: prior.id, remote_run_url: prior.details_url, remote_conclusion: 'success', conclusion: 'success', - }; - await own.patch(`/repos/${repo}/check-runs/${prior.id}`, { - output: { - title: prior.output.title, - summary: `${prior.output.summary}\n\n_Not re-run in attempt ${attempt} of ${orchestratorUrl}: previous success carried forward._`, - }, - }); - console.log(`[${entry.id}] carried forward success from ${prior.details_url}`); -} - -async function skipAll(manifest, target, repos, state) { - for (const entry of repos) { - const check = await own.post(`/repos/${repo}/check-runs`, { - name: checkName(manifest, entry.id), head_sha: target.headSha, status: 'completed', conclusion: 'skipped', - external_id: ourExternalId(entry.id), details_url: orchestratorUrl, - output: { title: 'Not dispatched: no relevant files changed', summary: `Only paths matching ${code((manifest.paths_ignore || []).join(', '))} changed.\n\nOrchestrator: ${orchestratorUrl}` }, - }); - state.repos[entry.id] = { id: entry.id, repository: entry.repository, failure_policy: entry.failure_policy || 'block', skipped: true, check_run_id: check.id, remote_conclusion: 'skipped', conclusion: 'skipped' }; - } -} - -function writeSummary(target, state) { - const rows = Object.values(state.repos).map((s) => - `| ${code(s.id)} | ${s.repository} | ${code(s.ext_ref || '-')} | ${s.failure_policy} | ${s.carried_forward ? 'carried forward' : s.remote_conclusion} | ${s.conclusion} | ${s.remote_run_url ? `[run](${s.remote_run_url})` : (s.error ? code(s.error) : '')} |`); - const md = [ - `### External CI for ${code(`${target.headRepo}@${short(target.headSha)}`)}${target.pr ? ` (PR #${target.pr})` : ` (${target.headBranch})`} — attempt ${attempt}`, - '', '| id | repository | ext ref | policy | remote | check | link |', '|---|---|---|---|---|---|---|', ...rows, '', - ].join('\n'); - if (env.GITHUB_STEP_SUMMARY) fs.appendFileSync(env.GITHUB_STEP_SUMMARY, md); - console.log(md); - fs.writeFileSync(RESULTS_FILE, JSON.stringify({ target, attempt, orchestrator_url: orchestratorUrl, repos: state.repos }, null, 2)); -} - -// ---------------------------------------------------------------- modes -async function run() { - const target = await resolveTarget(); - console.log(`target: ${JSON.stringify(target)}`); - const manifest = await loadManifest(target.baseRef); - // Fork preflight: a fork of F´ that copies this workflow must not dispatch into the ext repos. - if (manifest.upstream_repository && manifest.upstream_repository !== repo && eventName !== 'workflow_dispatch') { - console.log(`::notice::${repo} is not ${manifest.upstream_repository}; external CI is only dispatched from upstream. Use workflow_dispatch to run it here deliberately.`); - return; - } - const { carried, dispatch } = await selectRepos(manifest, target); - const state = { target, attempt, repos: {} }; - saveState(state); - for (const c of carried) await carryForward(manifest, target, c, state); - if (eventName !== 'workflow_dispatch' && !relevantChange(await changedFiles(target), manifest)) { - console.log(`no relevant files changed; marking [${dispatch.map((e) => e.id)}] as skipped`); - await skipAll(manifest, target, dispatch, state); - writeSummary(target, state); - return; - } - console.log(`dispatch: [${dispatch.map((e) => e.id)}] carried forward: [${carried.map((c) => c.entry.id)}]`); - for (const entry of dispatch) await dispatchOne(manifest, entry, target, state); - await correlate(state); - await pollAll(state); - for (const s of Object.values(state.repos)) { - if (s.carried_forward) continue; - try { await finalizeOne(target, s); } - catch (e) { console.log(`::error::[${s.id}] finalize: ${e.message}`); s.conclusion = s.conclusion || 'failure'; } - saveState(state); - } - writeSummary(target, state); - const blocking = Object.values(state.repos).filter((s) => s.failure_policy !== 'notify' && s.conclusion !== 'success' && s.conclusion !== 'skipped'); - if (blocking.length) { - console.log(`::error::blocking external repos failed: ${blocking.map((s) => s.id).join(', ')}`); - process.exit(1); - } -} - -// Runs under `if: always()`: whatever the run step left open (crash, cancellation, timeout) is -// closed here so no Check Run stays in_progress and no remote run keeps burning runners. -async function finalize() { - const state = loadState(); - if (!state) return console.log('no state file; nothing to finalize'); - const cancelled = env.JOB_STATUS === 'cancelled'; - for (const s of Object.values(state.repos)) { - if (s.carried_forward || s.skipped || s.conclusion) continue; - if (s.remote_run_id && !s.remote_conclusion) { - console.log(`[${s.id}] cancelling ${s.remote_run_url}`); - await ext.post(`/repos/${s.repository}/actions/runs/${s.remote_run_id}/cancel`).catch((e) => console.log(` ${e.message}`)); - } - if (s.check_run_id) { - const current = await own.get(`/repos/${repo}/check-runs/${s.check_run_id}`).catch(() => null); - if (current && current.status === 'completed') continue; - const conclusion = cancelled ? 'cancelled' : (s.failure_policy === 'notify' ? 'neutral' : 'failure'); - const why = cancelled ? 'The orchestrator run was cancelled' : `The orchestrator run ended before this repo was reported (${code(s.error || 'orchestrator error')})`; - await own.patch(`/repos/${repo}/check-runs/${s.check_run_id}`, { - status: 'completed', conclusion, - output: { title: cancelled ? 'Cancelled by orchestrator' : 'Orchestrator error', summary: `${why}; remote run ${s.remote_run_url || '(not started)'} was cancelled.\n\n${orchestratorUrl}` }, - }).catch((e) => console.log(` ${e.message}`)); - } - } -} - -await (mode === 'finalize' ? finalize() : run()); diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py new file mode 100644 index 00000000000..001b1cb9f02 --- /dev/null +++ b/.github/scripts/ext-ci-orchestrator.py @@ -0,0 +1,745 @@ +#!/usr/bin/env python3 +"""External-CI orchestrator (standard library only, Python >= 3.10). + + python3 ext-ci-orchestrator.py run dispatch, poll, report + python3 ext-ci-orchestrator.py finalize close anything still open (runs under if: always()) + +Two credentials, never mixed: + GITHUB_TOKEN this repo only (checks/pull-requests/issues write) - reports results + DISPATCH_TOKEN external repos only (actions: write; contents: read too if an external repo + is private, for the branch lookup) - starts remote runs +""" +import base64 +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone + +env = os.environ +MODE = sys.argv[1] if len(sys.argv) > 1 else "run" +API = env.get("GITHUB_API_URL", "https://api.github.com") +SERVER = env.get("GITHUB_SERVER_URL", "https://github.com") +STATE_FILE = "ext-ci-state.json" +RESULTS_FILE = "ext-ci-results.json" +MANIFEST_PATH = ".github/ext-ci-manifest.json" +POLL_S = int(env.get("POLL_INTERVAL_SECONDS") or 60) +CORRELATE_TIMEOUT_S = 5 * 60 +BOT_LOGIN = "github-actions[bot]" +NOTIFY_LABEL = "ext-ci" + +REPO = env.get("GITHUB_REPOSITORY") +RUN_ID = env.get("GITHUB_RUN_ID") +ATTEMPT = int(env.get("GITHUB_RUN_ATTEMPT") or 1) +EVENT_NAME = env.get("GITHUB_EVENT_NAME") +with open(env["GITHUB_EVENT_PATH"]) as _f: + EVENT = json.load(_f) +ORCHESTRATOR_URL = f"{SERVER}/{REPO}/actions/runs/{RUN_ID}" + (f"/attempts/{ATTEMPT}" if ATTEMPT > 1 else "") + + +def log(msg): + print(msg, flush=True) + + +def short(sha): + return sha[:7] + + +def code(s): + # Remote-controlled text (job/step names, branch names) is only ever rendered inside code spans. + return "`" + re.sub(r"[`\r\n]", " ", str(s)) + "`" + + +def q(s): + return urllib.parse.quote(str(s), safe="") + + +def now(): + return datetime.now(timezone.utc) + + +def iso(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_ts(s): + return datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + + +def fprime_rev(target): + return f"{target['head_repo']}@{short(target['head_sha'])}" + + +def mentions(s): + return " ".join("@" + o for o in s["owners"]) + + +# ---------------------------------------------------------------- REST helpers +class ApiError(Exception): + def __init__(self, msg, status=0): + super().__init__(msg) + self.status = status + + +class Client: + def __init__(self, token, label): + self.token, self.label, self.etags = token, label, {} + + def request(self, method, url, body=None, conditional=False, idempotent=True): + """Returns (status, parsed body). Retries rate limits always, network errors and 5xx only + when the call is idempotent (a workflow dispatch is not: the run may have started).""" + if not self.token: + raise ApiError(f"{self.label} token missing (secret not configured?)") + full = url if url.startswith("http") else API + url + headers = { + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "fprime-ext-ci-orchestrator", + } + data = None + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if conditional and full in self.etags: + headers["If-None-Match"] = self.etags[full][0] + i = 0 + while True: + req = urllib.request.Request(full, data=data, method=method, headers=headers) + try: + with urllib.request.urlopen(req, timeout=60) as res: + status, hdrs, text = res.status, res.headers, res.read().decode() + except urllib.error.HTTPError as e: + status, hdrs, text = e.code, e.headers, e.read().decode(errors="replace") + except (urllib.error.URLError, OSError) as e: + if not idempotent or i >= 4: + raise ApiError(f"{self.label}: {method} {url} -> network error {e}") + time.sleep(2 * 2**i) + i += 1 + continue + if status == 304 and conditional: + return 200, self.etags[full][1] + try: + parsed = json.loads(text) if text else None + except ValueError: + parsed = text + limited = status == 429 or ( + status == 403 and (hdrs.get("x-ratelimit-remaining") == "0" or re.search("rate limit", text, re.I)) + ) + if (limited or (status >= 500 and idempotent)) and i < 4: + wait = 2 * 2**i + retry_after, reset = hdrs.get("retry-after"), hdrs.get("x-ratelimit-reset") + if retry_after: + wait = int(retry_after) + elif limited and reset: + wait = max(wait, int(reset) - int(time.time()) + 1) + log(f"::warning::{self.label}: {method} {url} -> {status}; retrying in {wait} s") + time.sleep(min(wait, 15 * 60)) + i += 1 + continue + if 200 <= status < 300 and conditional and hdrs.get("etag"): + self.etags[full] = (hdrs.get("etag"), parsed) + return status, parsed + + def api(self, method, url, body=None, **kw): + status, data = self.request(method, url, body, **kw) + if not 200 <= status < 300: + raise ApiError(f"{self.label}: {method} {url} -> {status} {json.dumps(data)[:300]}", status) + return data + + def get(self, url, **kw): + return self.api("GET", url, **kw) + + def post(self, url, body=None, **kw): + return self.api("POST", url, body, **kw) + + def patch(self, url, body): + return self.api("PATCH", url, body) + + def exists(self, url): + # True/False for 200/404; anything else is an error, never silently "absent". + status, _ = self.request("GET", url) + if status == 200: + return True + if status == 404: + return False + raise ApiError(f"{self.label}: GET {url} -> {status}", status) + + +own = Client(env.get("GITHUB_TOKEN"), "GITHUB_TOKEN") +ext = Client(env.get("DISPATCH_TOKEN"), "DISPATCH_TOKEN") + + +# ---------------------------------------------------------------- target resolution +def resolve_target(): + """What F´ revision are we testing, and which PR (if any) gets the Check Runs?""" + t = {"pr": None, "head_repo": REPO, "head_sha": None, "head_branch": None, "base_ref": None, "before_sha": None} + if EVENT_NAME == "workflow_run": + wr = EVENT["workflow_run"] + # Trust only the event payload: nothing produced by the (untrusted) trigger run is read. + t["head_sha"] = wr["head_sha"] + t["head_repo"] = wr["head_repository"]["full_name"] + t["head_branch"] = wr["head_branch"] + if wr["event"] == "merge_group": + # Merge-queue branch gh-readonly-queue//pr--: checks go on the queue SHA, no PR. + m = re.match(r"^gh-readonly-queue/(.+?)/pr-\d+-", t["head_branch"] or "") + t["base_ref"] = m.group(1) if m else env.get("GITHUB_REF_NAME") + return t + # workflow_run.pull_requests is empty for fork PRs; find the PR by commit, then by head ref. + prs = own.get(f"/repos/{REPO}/commits/{t['head_sha']}/pulls?per_page=20") + pr = next((p for p in prs if p["state"] == "open" and p["head"]["sha"] == t["head_sha"] + and p["base"]["repo"]["full_name"] == REPO), None) + if not pr: + owner = t["head_repo"].split("/")[0] + prs = own.get(f"/repos/{REPO}/pulls?state=open&head={q(owner + ':' + t['head_branch'])}") + pr = next((p for p in prs if p["head"]["sha"] == t["head_sha"]), None) + if not pr: + raise RuntimeError(f"no open PR found for {t['head_repo']}@{t['head_sha']} ({t['head_branch']}); " + "the PR may have been updated or closed") + t["pr"], t["base_ref"] = pr["number"], pr["base"]["ref"] + elif EVENT_NAME == "workflow_dispatch": + pr_number = (env.get("INPUT_PR_NUMBER") or "").strip() + if pr_number: + if not pr_number.isdigit(): + raise RuntimeError(f"pr_number must be numeric, got {pr_number!r}") + pr = own.get(f"/repos/{REPO}/pulls/{pr_number}") + t.update(pr=pr["number"], head_sha=pr["head"]["sha"], head_repo=pr["head"]["repo"]["full_name"], + head_branch=pr["head"]["ref"], base_ref=pr["base"]["ref"]) + else: + ref = (env.get("INPUT_FPRIME_REF") or "").strip() or env["GITHUB_REF_NAME"] + commit = own.get(f"/repos/{REPO}/commits/{q(ref)}") + t.update(head_sha=commit["sha"], head_branch=ref, base_ref=ref) + else: # push, schedule + t.update(head_sha=env["GITHUB_SHA"], head_branch=env["GITHUB_REF_NAME"], base_ref=env["GITHUB_REF_NAME"]) + before = EVENT.get("before") + if EVENT_NAME == "push" and before and not re.fullmatch("0+", before): + t["before_sha"] = before + return t + + +def load_manifest(ref): + """Manifest is read from the base branch of the PR (the orchestrator itself always runs from + the default branch), so release/** PRs use their own branch's manifest. Only a missing file + falls back to the checked-out copy; any other error is fatal.""" + try: + file = own.get(f"/repos/{REPO}/contents/{MANIFEST_PATH}?ref={q(ref)}") + return json.loads(base64.b64decode(file["content"])) + except ApiError as e: + if e.status != 404: + raise + log(f"::warning::manifest missing at {ref}; using the checked-out copy") + with open(MANIFEST_PATH) as f: + return json.load(f) + + +def check_name(manifest, repo_id): + return f"{manifest.get('check_prefix', 'ext-ci')} / {repo_id}" + + +def our_external_id(repo_id): + return f"{RUN_ID}-{repo_id}" # same across attempts of one run + + +# ---------------------------------------------------------------- change filter +# paths-ignore lives here, not on the trigger workflow, so that filtered PRs still get a +# (skipped) Check Run per repo and required checks can be satisfied. +def glob_to_regex(glob): + out, i = "^", 0 + while i < len(glob): + c = glob[i] + if c == "*": + if glob[i + 1:i + 2] == "*": + out += ".*" + i += 1 + if glob[i + 1:i + 2] == "/": + i += 1 + else: + out += "[^/]*" + elif c == "?": + out += "[^/]" + else: + out += re.escape(c) + i += 1 + return re.compile(out + "$") + + +def changed_files(target): + if target["pr"]: + url = f"/repos/{REPO}/pulls/{target['pr']}/files?per_page=100" + elif target["before_sha"]: + url = f"/repos/{REPO}/compare/{target['before_sha']}...{target['head_sha']}?per_page=100" + else: + return None # unknown -> run everything + files = [] + for page in range(1, 31): + r = own.get(f"{url}&page={page}") + batch = r if target["pr"] else r.get("files", []) + files += [f["filename"] for f in batch] + if len(batch) < 100: + break + return files + + +def relevant_change(files, manifest): + ignore = [glob_to_regex(g) for g in manifest.get("paths_ignore", [])] + if files is None or not ignore: + return True + return any(not any(rx.match(f) for rx in ignore) for f in files) + + +# ---------------------------------------------------------------- selection +def latest_own_check_run(name, repo_id, sha): + r = own.get(f"/repos/{REPO}/commits/{sha}/check-runs?check_name={q(name)}&filter=all&per_page=50") + # Only reuse a check this very orchestrator run created (external_id stamped with our run id). + ours = [c for c in r["check_runs"] + if c.get("external_id") == our_external_id(repo_id) and (c.get("app") or {}).get("slug") == "github-actions"] + return max(ours, key=lambda c: c["id"], default=None) + + +def select_repos(manifest, target): + repos = manifest["repos"] + wanted = (env.get("INPUT_REPOS") or "all").strip() + if EVENT_NAME == "workflow_dispatch" and wanted and wanted != "all": + ids = [s.strip() for s in wanted.split(",") if s.strip()] + unknown = [i for i in ids if not any(r["id"] == i for r in repos)] + if unknown: + raise RuntimeError("unknown manifest ids: " + ", ".join(unknown)) + repos = [r for r in repos if r["id"] in ids] + elif target["pr"]: + repos = [r for r in repos if r.get("on_pr", True)] + # Failed-only re-run: on attempt > 1 keep every repo whose Check Run from attempt 1..n-1 of + # this same run already succeeded, and dispatch only the rest. + carried, dispatch = [], [] + force_all = (env.get("INPUT_FORCE_ALL") or "") == "true" + for entry in repos: + prior = None + if ATTEMPT > 1 and not force_all: + prior = latest_own_check_run(check_name(manifest, entry["id"]), entry["id"], target["head_sha"]) + if prior and prior["status"] == "completed" and prior["conclusion"] == "success": + carried.append((entry, prior)) + else: + dispatch.append(entry) + return carried, dispatch + + +def resolve_ext_ref(entry, target): + """pr- branch on the ext repo, else the F´ base branch name, else the manifest default.""" + for c in (f"pr-{target['pr']}" if target["pr"] else None, target["base_ref"]): + if c and ext.exists(f"/repos/{entry['repository']}/branches/{q(c)}"): + return c + return entry.get("default_ref", "main") + + +# ---------------------------------------------------------------- state +def save_state(state): + with open(STATE_FILE, "w") as f: + json.dump(state, f, indent=2) + + +def load_state(): + if not os.path.exists(STATE_FILE): + return None + with open(STATE_FILE) as f: + return json.load(f) + + +# ---------------------------------------------------------------- dispatch +def dispatch_one(manifest, entry, target, state): + dispatch_id = f"{RUN_ID}-{ATTEMPT}-{entry['id']}" + s = state["repos"][entry["id"]] = { + "id": entry["id"], "repository": entry["repository"], "workflow": entry["workflow"], "ext_ref": None, + "failure_policy": entry.get("failure_policy", "block"), "owners": entry.get("owners", []), + "timeout_minutes": entry.get("timeout_minutes", 60), "dispatch_id": dispatch_id, "dispatched_at": iso(now()), + "check_run_id": None, "remote_run_id": None, "remote_run_url": None, "remote_started_at": None, + "remote_conclusion": None, "conclusion": None, "error": None, + } + try: + # Check Run first, so a credential or API problem on the ext side still ends in a visible + # failed check rather than a required check that never appears. + check = own.post(f"/repos/{REPO}/check-runs", { + "name": check_name(manifest, entry["id"]), "head_sha": target["head_sha"], "status": "queued", + "external_id": our_external_id(entry["id"]), "details_url": ORCHESTRATOR_URL, + "output": { + "title": f"Dispatching {entry['repository']}", + "summary": f"Dispatching {code(entry['workflow'])} on {code(entry['repository'])} with F´ " + f"{code(fprime_rev(target))}.\n\nOrchestrator: {ORCHESTRATOR_URL}", + }, + }) + s["check_run_id"] = check["id"] + save_state(state) + s["ext_ref"] = resolve_ext_ref(entry, target) + try: + r = ext.post(f"/repos/{entry['repository']}/actions/workflows/{entry['workflow']}/dispatches", { + "ref": s["ext_ref"], + "inputs": { + "fprime_repository": target["head_repo"], "fprime_ref": target["head_sha"], + "fprime_pr_number": str(target["pr"] or ""), "dispatch_id": dispatch_id, + }, + "return_run_details": True, + }, idempotent=False) + except ApiError as e: + if 0 < e.status < 500: + raise + # Lost response: the run may exist; correlate() finds it by dispatch_id or times out. + log(f"::warning::[{entry['id']}] dispatch outcome unknown ({e})") + r = None + if r and r.get("workflow_run_id"): + s["remote_run_id"], s["remote_run_url"] = r["workflow_run_id"], r.get("html_url") + mark_running(s) + where = f" -> {s['remote_run_url']}" if s["remote_run_url"] else f" (dispatch_id={dispatch_id})" + log(f"[{entry['id']}] dispatched {entry['repository']}/{entry['workflow']} @ {s['ext_ref']}{where}") + except Exception as e: # noqa: BLE001 - one repo's failure must not stop the fan-out + s["error"], s["remote_conclusion"] = f"dispatch failed: {e}", "dispatch_failed" + log(f"::error::[{entry['id']}] {s['error']}") + save_state(state) + + +def mark_running(s): + # A failed status update is not a failed dispatch: the remote run exists and is polled anyway. + try: + own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { + "status": "in_progress", "details_url": s["remote_run_url"], + "output": {"title": f"Running on {s['repository']} @ {s['ext_ref']}", + "summary": f"Remote run: {s['remote_run_url']}\n\nOrchestrator: {ORCHESTRATOR_URL}"}, + }) + except ApiError as e: + log(f"::warning::[{s['id']}] check update: {e}") + + +def cancel_run(s): + try: + ext.post(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}/cancel") + except ApiError as e: + log(f" {e}") + + +def correlate(state): + """For dispatches without a run id (lost response, or an API server that still answers 204): + find the run whose title carries our dispatch_id (the ext workflow echoes it into run-name).""" + pending = [s for s in state["repos"].values() if not s["remote_run_id"] and not s["remote_conclusion"]] + deadline = time.time() + CORRELATE_TIMEOUT_S + while pending and time.time() < deadline: + for s in list(pending): + try: + since = iso(parse_ts(s["dispatched_at"]) - timedelta(seconds=60)) + runs = ext.get(f"/repos/{s['repository']}/actions/runs?event=workflow_dispatch" + f"&created={q('>=' + since)}&per_page=50") + run = next((r for r in runs["workflow_runs"] + if s["dispatch_id"] in (r.get("display_title") or r.get("name") or "").split()), None) + if not run: + continue + s["remote_run_id"], s["remote_run_url"] = run["id"], run["html_url"] + pending.remove(s) + log(f"[{s['id']}] remote run {run['html_url']}") + mark_running(s) + save_state(state) + except ApiError as e: + log(f"::warning::[{s['id']}] correlate: {e}") + if pending: + time.sleep(15) + for s in pending: + s["remote_conclusion"] = "not_found" + s["error"] = (f"no run with dispatch_id {s['dispatch_id']} appeared on {s['repository']} " + f"within {CORRELATE_TIMEOUT_S // 60} min") + log(f"::error::[{s['id']}] {s['error']}") + save_state(state) + + +# ---------------------------------------------------------------- poll +def poll_all(state): + def active(): + return [s for s in state["repos"].values() if s["remote_run_id"] and not s["remote_conclusion"]] + + while active(): + time.sleep(POLL_S) + for s in active(): + try: + run = ext.get(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}", conditional=True) + if run.get("run_started_at") and not s["remote_started_at"]: + s["remote_started_at"] = run["run_started_at"] + timeout = timedelta(minutes=s["timeout_minutes"]) + if run["status"] == "completed": + s["remote_conclusion"] = run["conclusion"] + log(f"[{s['id']}] completed: {run['conclusion']}") + elif s["remote_started_at"] and now() - parse_ts(s["remote_started_at"]) > timeout: + # Timeout counts from when the remote run started, not from dispatch (queue time is not the ext repo's fault). + log(f"::warning::[{s['id']}] timed out after {s['timeout_minutes']} min of running; cancelling remote run") + cancel_run(s) + s["remote_conclusion"] = "timed_out" + elif not s["remote_started_at"] and now() - parse_ts(s["dispatched_at"]) > 2 * timeout: + log(f"::warning::[{s['id']}] still queued after {2 * s['timeout_minutes']} min; giving up") + cancel_run(s) + s["remote_conclusion"], s["error"] = "queue_timeout", "remote run never started (no runner available?)" + except ApiError as e: + # A single bad poll must not take the whole fan-out down; give up on this repo only + # when the credential is gone or the run vanished. + log(f"::warning::[{s['id']}] poll: {e}") + if e.status in (401, 404): + s["remote_conclusion"], s["error"] = "error", f"poll failed: {e}" + save_state(state) + + +# ---------------------------------------------------------------- report +def failure_details(s): + if not s["remote_run_id"]: + return "" + try: + jobs = ext.get(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}/jobs?per_page=100")["jobs"] + except ApiError as e: + return f"_(job details unavailable: {code(e)})_" + lines = [] + for j in jobs: + if j.get("conclusion") in (None, "success", "skipped"): + continue + lines.append(f"- [{code(j['name'])}]({j['html_url']}): **{j['conclusion']}**") + lines += [f" - step {code(st['name'])}: {st['conclusion']}" + for st in j.get("steps", []) if st.get("conclusion") not in (None, "success", "skipped")] + return "\n".join(lines) + + +def check_conclusion(s): + rc = s["remote_conclusion"] + if rc == "success": + return "success" + if s["failure_policy"] == "notify": + return "neutral" + if rc == "cancelled": + return "cancelled" + if rc in ("timed_out", "queue_timeout"): + return "timed_out" + return "failure" + + +def finalize_one(target, s): + conclusion = check_conclusion(s) + details = "" if s["remote_conclusion"] == "success" else failure_details(s) + lines = [f"{code(s['repository'])} @ {code(s['ext_ref'] or '-')} with F´ {code(fprime_rev(target))}: " + f"**{s['remote_conclusion']}**", ""] + if s["remote_run_url"]: + lines.append(f"Remote run: {s['remote_run_url']}") + if s.get("error"): + lines.append(f"Error: {code(s['error'])}") + if s["failure_policy"] == "notify" and s["remote_conclusion"] != "success": + lines += ["", f"This repo is {code('failure_policy: notify')}: the failure does not block the PR; " + f"owners ({mentions(s)}) have been notified."] + lines += ["", f'Orchestrator run (use "Re-run failed jobs" there to re-dispatch only the failed repos): {ORCHESTRATOR_URL}'] + if s["check_run_id"]: + output = {"title": f"{s['remote_conclusion']} ({s['repository']} @ {s['ext_ref'] or '-'})", "summary": "\n".join(lines)} + if details: + output["text"] = details + own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { + "status": "completed", "conclusion": conclusion, "details_url": s["remote_run_url"] or ORCHESTRATOR_URL, + "output": output, + }) + s["conclusion"] = conclusion # recorded only once the check is closed, so finalize() retries otherwise + if s["failure_policy"] == "notify": + try: + notify_owners(target, s, details) + except ApiError as e: + log(f"::warning::[{s['id']}] notify: {e}") + + +# notify policy: one PR comment per ext repo (updated in place), or one tracking issue when +# there is no PR (closed again once the repo passes). Only comments/issues authored by the +# Actions bot are considered ours, so nobody can plant a marker to suppress notifications. +def ensure_label(): + if own.exists(f"/repos/{REPO}/labels/{NOTIFY_LABEL}"): + return + try: + own.post(f"/repos/{REPO}/labels", + {"name": NOTIFY_LABEL, "color": "ededed", "description": "Opened by the external-CI orchestrator"}) + except ApiError: + pass + + +def notify_owners(target, s, details): + marker = f"" + failed = s["remote_conclusion"] != "success" + run_or_error = s["remote_run_url"] or code(s.get("error")) + + def ours(x): + return (x.get("user") or {}).get("login") == BOT_LOGIN and marker in (x.get("body") or "") + + if target["pr"]: + existing = None + for page in range(1, 11): + comments = own.get(f"/repos/{REPO}/issues/{target['pr']}/comments?per_page=100&page={page}") + existing = next((c for c in comments if ours(c)), None) + if existing or len(comments) < 100: + break + if not failed and not existing: + return + if failed: + body = (f"{marker}\n### External CI: {code(s['id'])} failed (non-blocking)\n\n" + f"{code(s['repository'])} @ {code(s['ext_ref'])} does not pass against {short(target['head_sha'])}: " + f"{run_or_error}\n\n{details}\n\n{mentions(s)} — this repo is {code('failure_policy: notify')}, " + "so the PR is not blocked, but it needs a look.") + else: + body = (f"{marker}\n### External CI: {code(s['id'])} passing again\n\n" + f"{code(s['repository'])} @ {code(s['ext_ref'])} passes against {short(target['head_sha'])}: " + f"{s['remote_run_url']}") + if existing: + own.patch(f"/repos/{REPO}/issues/comments/{existing['id']}", {"body": body}) + else: + own.post(f"/repos/{REPO}/issues/{target['pr']}/comments", {"body": body}) + return + title = f"[ext-ci] {s['id']} failing on {target['head_branch']}" + ensure_label() + issues = own.get(f"/repos/{REPO}/issues?state=open&labels={NOTIFY_LABEL}&creator={q(BOT_LOGIN)}&per_page=100") + existing = next((i for i in issues if not i.get("pull_request") and ours(i) and i["title"] == title), None) + if failed: + if existing: + own.post(f"/repos/{REPO}/issues/{existing['number']}/comments", + {"body": f"Still failing at {short(target['head_sha'])}: {run_or_error}"}) + else: + body = (f"{marker}\n{code(s['repository'])} @ {code(s['ext_ref'])} fails against {code(target['head_branch'])} " + f"({short(target['head_sha'])}): {run_or_error}\n\n{details}\n\n{mentions(s)}\n\n" + "This issue is closed automatically once the repo passes again.") + own.post(f"/repos/{REPO}/issues", {"title": title, "body": body, "labels": [NOTIFY_LABEL]}) + elif existing: + own.post(f"/repos/{REPO}/issues/{existing['number']}/comments", + {"body": f"Passing again at {short(target['head_sha'])}: {s['remote_run_url']}"}) + own.patch(f"/repos/{REPO}/issues/{existing['number']}", {"state": "closed", "state_reason": "completed"}) + + +def carry_forward(entry, prior, state): + state["repos"][entry["id"]] = { + "id": entry["id"], "repository": entry["repository"], "failure_policy": entry.get("failure_policy", "block"), + "carried_forward": True, "check_run_id": prior["id"], "remote_run_url": prior["details_url"], + "remote_conclusion": "success", "conclusion": "success", + } + own.patch(f"/repos/{REPO}/check-runs/{prior['id']}", {"output": { + "title": prior["output"]["title"], + "summary": f"{prior['output']['summary']}\n\n_Not re-run in attempt {ATTEMPT} of {ORCHESTRATOR_URL}: " + "previous success carried forward._", + }}) + log(f"[{entry['id']}] carried forward success from {prior['details_url']}") + + +def skip_all(manifest, target, repos, state): + ignored = ", ".join(manifest.get("paths_ignore", [])) + for entry in repos: + check = own.post(f"/repos/{REPO}/check-runs", { + "name": check_name(manifest, entry["id"]), "head_sha": target["head_sha"], + "status": "completed", "conclusion": "skipped", + "external_id": our_external_id(entry["id"]), "details_url": ORCHESTRATOR_URL, + "output": {"title": "Not dispatched: no relevant files changed", + "summary": f"Only paths matching {code(ignored)} changed.\n\nOrchestrator: {ORCHESTRATOR_URL}"}, + }) + state["repos"][entry["id"]] = { + "id": entry["id"], "repository": entry["repository"], "failure_policy": entry.get("failure_policy", "block"), + "skipped": True, "check_run_id": check["id"], "remote_conclusion": "skipped", "conclusion": "skipped", + } + + +def write_summary(target, state): + rows = [] + for s in state["repos"].values(): + remote = "carried forward" if s.get("carried_forward") else s.get("remote_conclusion") + link = f"[run]({s['remote_run_url']})" if s.get("remote_run_url") else (code(s["error"]) if s.get("error") else "") + rows.append(f"| {code(s['id'])} | {s['repository']} | {code(s.get('ext_ref') or '-')} | {s['failure_policy']} " + f"| {remote} | {s.get('conclusion')} | {link} |") + where = f" (PR #{target['pr']})" if target["pr"] else f" ({target['head_branch']})" + md = "\n".join([f"### External CI for {code(fprime_rev(target))}{where} — attempt {ATTEMPT}", "", + "| id | repository | ext ref | policy | remote | check | link |", + "|---|---|---|---|---|---|---|", *rows, ""]) + if env.get("GITHUB_STEP_SUMMARY"): + with open(env["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(md) + log(md) + with open(RESULTS_FILE, "w") as f: + json.dump({"target": target, "attempt": ATTEMPT, "orchestrator_url": ORCHESTRATOR_URL, "repos": state["repos"]}, + f, indent=2) + + +# ---------------------------------------------------------------- modes +def run(): + target = resolve_target() + log(f"target: {json.dumps(target)}") + manifest = load_manifest(target["base_ref"]) + # Fork preflight: a fork of F´ that copies this workflow must not dispatch into the ext repos. + upstream = manifest.get("upstream_repository") + if upstream and upstream != REPO and EVENT_NAME != "workflow_dispatch": + log(f"::notice::{REPO} is not {upstream}; external CI is only dispatched from upstream. " + "Use workflow_dispatch to run it here deliberately.") + return + carried, dispatch = select_repos(manifest, target) + state = {"target": target, "attempt": ATTEMPT, "repos": {}} + save_state(state) + for entry, prior in carried: + carry_forward(entry, prior, state) + ids = [e["id"] for e in dispatch] + if EVENT_NAME != "workflow_dispatch" and not relevant_change(changed_files(target), manifest): + log(f"no relevant files changed; marking {ids} as skipped") + skip_all(manifest, target, dispatch, state) + write_summary(target, state) + return + log(f"dispatch: {ids} carried forward: {[e['id'] for e, _ in carried]}") + for entry in dispatch: + dispatch_one(manifest, entry, target, state) + correlate(state) + poll_all(state) + for s in state["repos"].values(): + if s.get("carried_forward"): + continue + try: + finalize_one(target, s) + except Exception as e: # noqa: BLE001 - the check stays open and finalize() closes it + log(f"::error::[{s['id']}] finalize: {e}") + save_state(state) + write_summary(target, state) + blocking = [s["id"] for s in state["repos"].values() + if s["failure_policy"] != "notify" and (s.get("conclusion") or check_conclusion(s)) not in ("success", "skipped")] + if blocking: + log("::error::blocking external repos failed: " + ", ".join(blocking)) + sys.exit(1) + + +def finalize(): + """Runs under `if: always()`: whatever the run step left open (crash, cancellation, timeout) is + closed here so no Check Run stays in_progress and no remote run keeps burning runners.""" + state = load_state() + if not state: + log("no state file; nothing to finalize") + return + cancelled = env.get("JOB_STATUS") == "cancelled" + for s in state["repos"].values(): + if s.get("carried_forward") or s.get("skipped") or s.get("conclusion"): + continue + remote = s.get("remote_run_url") or "(not started)" + if s.get("remote_run_id") and not s.get("remote_conclusion"): + log(f"[{s['id']}] cancelling {remote}") + cancel_run(s) + remote += " (cancelled)" + if not s.get("check_run_id"): + continue + try: + current = own.get(f"/repos/{REPO}/check-runs/{s['check_run_id']}") + except ApiError: + current = None + if current and current["status"] == "completed": + continue + if s.get("remote_conclusion") and not cancelled: # reported, but the check update itself failed in run + conclusion, title = check_conclusion(s), f"{s['remote_conclusion']} ({s['repository']} @ {s.get('ext_ref') or '-'})" + why = "Closed by the finalize step after the check update failed" + elif cancelled: + conclusion, title, why = "cancelled", "Cancelled by orchestrator", "The orchestrator run was cancelled" + else: + conclusion = "neutral" if s["failure_policy"] == "notify" else "failure" + title = "Orchestrator error" + why = f"The orchestrator run ended before this repo was reported ({code(s.get('error') or 'orchestrator error')})" + try: + own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { + "status": "completed", "conclusion": conclusion, + "output": {"title": title, "summary": f"{why}; remote run {remote}.\n\n{ORCHESTRATOR_URL}"}, + }) + except ApiError as e: + log(f" {e}") + + +if __name__ == "__main__": + finalize() if MODE == "finalize" else run() diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index f911a8ef9b8..c1a1e1f7477 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -60,12 +60,12 @@ jobs: INPUT_FPRIME_REF: ${{ inputs.fprime_ref }} INPUT_FORCE_ALL: ${{ inputs.force_all }} POLL_INTERVAL_SECONDS: 60 - run: node .github/scripts/ext-ci-orchestrator.mjs run + run: python3 .github/scripts/ext-ci-orchestrator.py run - name: "Finalize (close open checks, cancel orphaned remote runs)" if: always() env: JOB_STATUS: ${{ job.status }} - run: node .github/scripts/ext-ci-orchestrator.mjs finalize + run: python3 .github/scripts/ext-ci-orchestrator.py finalize - uses: actions/upload-artifact@v4 if: always() with: From 77c7917968477a3d45f93dd7c0ca1eefb6f4eca4 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Wed, 4 Nov 2026 23:17:57 +0000 Subject: [PATCH 07/22] ci: ext-ci review fixes (carry-forward KeyError, manual fprime_ref manifest source, compare 404, concurrency key) - correlate/poll_all tolerate carried-forward entries (no remote_* keys) on a re-run - a manual run with fprime_ref reads the manifest and ext ref from the branch the run was started from, not from the ref under test (any PR SHA is readable via /contents?ref=) - a release push whose before SHA was rewritten runs everything instead of aborting - concurrency group includes the event name so a PR branch named release/* cannot cancel the push run --- .github/scripts/ext-ci-orchestrator.py | 15 +++++++++++---- .github/workflows/ext-ci.yml | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index 001b1cb9f02..f805e65a303 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -212,7 +212,9 @@ def resolve_target(): else: ref = (env.get("INPUT_FPRIME_REF") or "").strip() or env["GITHUB_REF_NAME"] commit = own.get(f"/repos/{REPO}/commits/{q(ref)}") - t.update(head_sha=commit["sha"], head_branch=ref, base_ref=ref) + # base_ref (manifest + ext ref lookup) is the branch this run was started from, never the + # possibly untrusted ref under test: any PR SHA is readable through /contents?ref=. + t.update(head_sha=commit["sha"], head_branch=ref, base_ref=env["GITHUB_REF_NAME"]) else: # push, schedule t.update(head_sha=env["GITHUB_SHA"], head_branch=env["GITHUB_REF_NAME"], base_ref=env["GITHUB_REF_NAME"]) before = EVENT.get("before") @@ -276,7 +278,12 @@ def changed_files(target): return None # unknown -> run everything files = [] for page in range(1, 31): - r = own.get(f"{url}&page={page}") + try: + r = own.get(f"{url}&page={page}") + except ApiError as e: + if e.status == 404 and not target["pr"]: + return None # before SHA no longer in history (force-push): run everything + raise batch = r if target["pr"] else r.get("files", []) files += [f["filename"] for f in batch] if len(batch) < 100: @@ -420,7 +427,7 @@ def cancel_run(s): def correlate(state): """For dispatches without a run id (lost response, or an API server that still answers 204): find the run whose title carries our dispatch_id (the ext workflow echoes it into run-name).""" - pending = [s for s in state["repos"].values() if not s["remote_run_id"] and not s["remote_conclusion"]] + pending = [s for s in state["repos"].values() if not s.get("remote_run_id") and not s.get("remote_conclusion")] deadline = time.time() + CORRELATE_TIMEOUT_S while pending and time.time() < deadline: for s in list(pending): @@ -452,7 +459,7 @@ def correlate(state): # ---------------------------------------------------------------- poll def poll_all(state): def active(): - return [s for s in state["repos"].values() if s["remote_run_id"] and not s["remote_conclusion"]] + return [s for s in state["repos"].values() if s.get("remote_run_id") and not s.get("remote_conclusion")] while active(): time.sleep(POLL_S) diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index c1a1e1f7477..1924a854b60 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -32,7 +32,7 @@ on: # One orchestrator per F´ head (repo + branch); a newer PR push cancels the older PR fan-out and # the finalize step below propagates that to the remote runs. Branch/nightly runs are never cancelled. concurrency: - group: ext-ci-${{ github.event.workflow_run.head_repository.full_name || github.repository }}-${{ github.event.workflow_run.head_branch || inputs.pr_number || inputs.fprime_ref || github.ref_name }} + group: ext-ci-${{ github.event_name }}-${{ github.event.workflow_run.head_repository.full_name || github.repository }}-${{ github.event.workflow_run.head_branch || inputs.pr_number || inputs.fprime_ref || github.ref_name }} cancel-in-progress: ${{ github.event_name == 'workflow_run' }} permissions: From 6052882e49f02a53ed5f3b94042fa283970aad79 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 01:34:13 +0000 Subject: [PATCH 08/22] ci: simplify the ext-ci orchestrator for the pilot Drop owner notifications (a notify repo is just a neutral check; no pull-requests/issues write needed any more), failed-only re-run carry-forward (a re-run re-dispatches every repo), the dispatch_id correlation fallback (the dispatch API returns the run id), manifest lookup from the base branch and ETag polling. --- .github/ext-ci-manifest.json | 3 - .github/scripts/ext-ci-orchestrator.py | 431 ++++++------------------- .github/workflows/ext-ci.yml | 7 - 3 files changed, 97 insertions(+), 344 deletions(-) diff --git a/.github/ext-ci-manifest.json b/.github/ext-ci-manifest.json index b20effe6f25..cbb7d869810 100644 --- a/.github/ext-ci-manifest.json +++ b/.github/ext-ci-manifest.json @@ -16,9 +16,6 @@ "default_ref": "devel", "timeout_minutes": 60, "failure_policy": "notify", - "owners": [ - "thomas-bc" - ], "on_pr": true } ] diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index f805e65a303..acd0534875c 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -4,12 +4,15 @@ python3 ext-ci-orchestrator.py run dispatch, poll, report python3 ext-ci-orchestrator.py finalize close anything still open (runs under if: always()) +For every repo in .github/ext-ci-manifest.json: create a Check Run on the F´ commit under test, +dispatch the repo's own ci.yml with that commit, poll the remote run, close the Check Run with +its result. A re-run of the orchestrator re-dispatches every repo. + Two credentials, never mixed: - GITHUB_TOKEN this repo only (checks/pull-requests/issues write) - reports results + GITHUB_TOKEN this repo only (checks: write) - reports results DISPATCH_TOKEN external repos only (actions: write; contents: read too if an external repo is private, for the branch lookup) - starts remote runs """ -import base64 import json import os import re @@ -19,6 +22,7 @@ import urllib.parse import urllib.request from datetime import datetime, timedelta, timezone +from fnmatch import fnmatch env = os.environ MODE = sys.argv[1] if len(sys.argv) > 1 else "run" @@ -28,9 +32,6 @@ RESULTS_FILE = "ext-ci-results.json" MANIFEST_PATH = ".github/ext-ci-manifest.json" POLL_S = int(env.get("POLL_INTERVAL_SECONDS") or 60) -CORRELATE_TIMEOUT_S = 5 * 60 -BOT_LOGIN = "github-actions[bot]" -NOTIFY_LABEL = "ext-ci" REPO = env.get("GITHUB_REPOSITORY") RUN_ID = env.get("GITHUB_RUN_ID") @@ -45,10 +46,6 @@ def log(msg): print(msg, flush=True) -def short(sha): - return sha[:7] - - def code(s): # Remote-controlled text (job/step names, branch names) is only ever rendered inside code spans. return "`" + re.sub(r"[`\r\n]", " ", str(s)) + "`" @@ -62,23 +59,15 @@ def now(): return datetime.now(timezone.utc) -def iso(dt): - return dt.strftime("%Y-%m-%dT%H:%M:%SZ") - - def parse_ts(s): return datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) def fprime_rev(target): - return f"{target['head_repo']}@{short(target['head_sha'])}" + return f"{target['head_repo']}@{target['head_sha'][:7]}" -def mentions(s): - return " ".join("@" + o for o in s["owners"]) - - -# ---------------------------------------------------------------- REST helpers +# ---------------------------------------------------------------- GitHub REST client class ApiError(Exception): def __init__(self, msg, status=0): super().__init__(msg) @@ -87,14 +76,13 @@ def __init__(self, msg, status=0): class Client: def __init__(self, token, label): - self.token, self.label, self.etags = token, label, {} + self.token, self.label = token, label - def request(self, method, url, body=None, conditional=False, idempotent=True): - """Returns (status, parsed body). Retries rate limits always, network errors and 5xx only + def request(self, method, url, body=None, idempotent=True): + """Returns (status, parsed body). Retries rate limits always; network errors and 5xx only when the call is idempotent (a workflow dispatch is not: the run may have started).""" if not self.token: raise ApiError(f"{self.label} token missing (secret not configured?)") - full = url if url.startswith("http") else API + url headers = { "Authorization": f"Bearer {self.token}", "Accept": "application/vnd.github+json", @@ -105,45 +93,35 @@ def request(self, method, url, body=None, conditional=False, idempotent=True): if body is not None: data = json.dumps(body).encode() headers["Content-Type"] = "application/json" - if conditional and full in self.etags: - headers["If-None-Match"] = self.etags[full][0] - i = 0 - while True: - req = urllib.request.Request(full, data=data, method=method, headers=headers) + for i in range(5): + req = urllib.request.Request(API + url, data=data, method=method, headers=headers) try: with urllib.request.urlopen(req, timeout=60) as res: status, hdrs, text = res.status, res.headers, res.read().decode() except urllib.error.HTTPError as e: status, hdrs, text = e.code, e.headers, e.read().decode(errors="replace") except (urllib.error.URLError, OSError) as e: - if not idempotent or i >= 4: + if not idempotent or i == 4: raise ApiError(f"{self.label}: {method} {url} -> network error {e}") time.sleep(2 * 2**i) - i += 1 continue - if status == 304 and conditional: - return 200, self.etags[full][1] - try: - parsed = json.loads(text) if text else None - except ValueError: - parsed = text limited = status == 429 or ( status == 403 and (hdrs.get("x-ratelimit-remaining") == "0" or re.search("rate limit", text, re.I)) ) if (limited or (status >= 500 and idempotent)) and i < 4: wait = 2 * 2**i - retry_after, reset = hdrs.get("retry-after"), hdrs.get("x-ratelimit-reset") - if retry_after: - wait = int(retry_after) - elif limited and reset: - wait = max(wait, int(reset) - int(time.time()) + 1) + if hdrs.get("retry-after"): + wait = int(hdrs["retry-after"]) + elif limited and hdrs.get("x-ratelimit-reset"): + wait = max(wait, int(hdrs["x-ratelimit-reset"]) - int(time.time()) + 1) log(f"::warning::{self.label}: {method} {url} -> {status}; retrying in {wait} s") time.sleep(min(wait, 15 * 60)) - i += 1 continue - if 200 <= status < 300 and conditional and hdrs.get("etag"): - self.etags[full] = (hdrs.get("etag"), parsed) - return status, parsed + try: + return status, json.loads(text) if text else None + except ValueError: + return status, text + raise AssertionError("unreachable") def api(self, method, url, body=None, **kw): status, data = self.request(method, url, body, **kw) @@ -151,8 +129,8 @@ def api(self, method, url, body=None, **kw): raise ApiError(f"{self.label}: {method} {url} -> {status} {json.dumps(data)[:300]}", status) return data - def get(self, url, **kw): - return self.api("GET", url, **kw) + def get(self, url): + return self.api("GET", url) def post(self, url, body=None, **kw): return self.api("POST", url, body, **kw) @@ -174,16 +152,15 @@ def exists(self, url): ext = Client(env.get("DISPATCH_TOKEN"), "DISPATCH_TOKEN") -# ---------------------------------------------------------------- target resolution +# ---------------------------------------------------------------- what to test def resolve_target(): - """What F´ revision are we testing, and which PR (if any) gets the Check Runs?""" + """Which F´ revision is under test, and which PR (if any) it belongs to.""" t = {"pr": None, "head_repo": REPO, "head_sha": None, "head_branch": None, "base_ref": None, "before_sha": None} if EVENT_NAME == "workflow_run": wr = EVENT["workflow_run"] # Trust only the event payload: nothing produced by the (untrusted) trigger run is read. - t["head_sha"] = wr["head_sha"] + t["head_sha"], t["head_branch"] = wr["head_sha"], wr["head_branch"] t["head_repo"] = wr["head_repository"]["full_name"] - t["head_branch"] = wr["head_branch"] if wr["event"] == "merge_group": # Merge-queue branch gh-readonly-queue//pr--: checks go on the queue SHA, no PR. m = re.match(r"^gh-readonly-queue/(.+?)/pr-\d+-", t["head_branch"] or "") @@ -212,8 +189,7 @@ def resolve_target(): else: ref = (env.get("INPUT_FPRIME_REF") or "").strip() or env["GITHUB_REF_NAME"] commit = own.get(f"/repos/{REPO}/commits/{q(ref)}") - # base_ref (manifest + ext ref lookup) is the branch this run was started from, never the - # possibly untrusted ref under test: any PR SHA is readable through /contents?ref=. + # base_ref (ext branch lookup) is the branch this run was started from, not the ref under test. t.update(head_sha=commit["sha"], head_branch=ref, base_ref=env["GITHUB_REF_NAME"]) else: # push, schedule t.update(head_sha=env["GITHUB_SHA"], head_branch=env["GITHUB_REF_NAME"], base_ref=env["GITHUB_REF_NAME"]) @@ -223,59 +199,14 @@ def resolve_target(): return t -def load_manifest(ref): - """Manifest is read from the base branch of the PR (the orchestrator itself always runs from - the default branch), so release/** PRs use their own branch's manifest. Only a missing file - falls back to the checked-out copy; any other error is fatal.""" - try: - file = own.get(f"/repos/{REPO}/contents/{MANIFEST_PATH}?ref={q(ref)}") - return json.loads(base64.b64decode(file["content"])) - except ApiError as e: - if e.status != 404: - raise - log(f"::warning::manifest missing at {ref}; using the checked-out copy") - with open(MANIFEST_PATH) as f: - return json.load(f) - - -def check_name(manifest, repo_id): - return f"{manifest.get('check_prefix', 'ext-ci')} / {repo_id}" - - -def our_external_id(repo_id): - return f"{RUN_ID}-{repo_id}" # same across attempts of one run - - -# ---------------------------------------------------------------- change filter -# paths-ignore lives here, not on the trigger workflow, so that filtered PRs still get a -# (skipped) Check Run per repo and required checks can be satisfied. -def glob_to_regex(glob): - out, i = "^", 0 - while i < len(glob): - c = glob[i] - if c == "*": - if glob[i + 1:i + 2] == "*": - out += ".*" - i += 1 - if glob[i + 1:i + 2] == "/": - i += 1 - else: - out += "[^/]*" - elif c == "?": - out += "[^/]" - else: - out += re.escape(c) - i += 1 - return re.compile(out + "$") - - def changed_files(target): + """Files of the PR / push, or None when unknown (then everything runs).""" if target["pr"]: url = f"/repos/{REPO}/pulls/{target['pr']}/files?per_page=100" elif target["before_sha"]: url = f"/repos/{REPO}/compare/{target['before_sha']}...{target['head_sha']}?per_page=100" else: - return None # unknown -> run everything + return None files = [] for page in range(1, 31): try: @@ -292,19 +223,12 @@ def changed_files(target): def relevant_change(files, manifest): - ignore = [glob_to_regex(g) for g in manifest.get("paths_ignore", [])] + # paths_ignore lives here, not on the trigger workflow, so that filtered PRs still get a + # (skipped) Check Run per repo and required checks can be satisfied. + ignore = manifest.get("paths_ignore", []) if files is None or not ignore: return True - return any(not any(rx.match(f) for rx in ignore) for f in files) - - -# ---------------------------------------------------------------- selection -def latest_own_check_run(name, repo_id, sha): - r = own.get(f"/repos/{REPO}/commits/{sha}/check-runs?check_name={q(name)}&filter=all&per_page=50") - # Only reuse a check this very orchestrator run created (external_id stamped with our run id). - ours = [c for c in r["check_runs"] - if c.get("external_id") == our_external_id(repo_id) and (c.get("app") or {}).get("slug") == "github-actions"] - return max(ours, key=lambda c: c["id"], default=None) + return any(not any(fnmatch(f, g) for g in ignore) for f in files) def select_repos(manifest, target): @@ -318,19 +242,7 @@ def select_repos(manifest, target): repos = [r for r in repos if r["id"] in ids] elif target["pr"]: repos = [r for r in repos if r.get("on_pr", True)] - # Failed-only re-run: on attempt > 1 keep every repo whose Check Run from attempt 1..n-1 of - # this same run already succeeded, and dispatch only the rest. - carried, dispatch = [], [] - force_all = (env.get("INPUT_FORCE_ALL") or "") == "true" - for entry in repos: - prior = None - if ATTEMPT > 1 and not force_all: - prior = latest_own_check_run(check_name(manifest, entry["id"]), entry["id"], target["head_sha"]) - if prior and prior["status"] == "completed" and prior["conclusion"] == "success": - carried.append((entry, prior)) - else: - dispatch.append(entry) - return carried, dispatch + return repos def resolve_ext_ref(entry, target): @@ -341,27 +253,22 @@ def resolve_ext_ref(entry, target): return entry.get("default_ref", "main") -# ---------------------------------------------------------------- state +def check_name(manifest, repo_id): + return f"{manifest.get('check_prefix', 'ext-ci')} / {repo_id}" + + def save_state(state): with open(STATE_FILE, "w") as f: json.dump(state, f, indent=2) -def load_state(): - if not os.path.exists(STATE_FILE): - return None - with open(STATE_FILE) as f: - return json.load(f) - - -# ---------------------------------------------------------------- dispatch +# ---------------------------------------------------------------- dispatch and poll def dispatch_one(manifest, entry, target, state): - dispatch_id = f"{RUN_ID}-{ATTEMPT}-{entry['id']}" s = state["repos"][entry["id"]] = { "id": entry["id"], "repository": entry["repository"], "workflow": entry["workflow"], "ext_ref": None, - "failure_policy": entry.get("failure_policy", "block"), "owners": entry.get("owners", []), - "timeout_minutes": entry.get("timeout_minutes", 60), "dispatch_id": dispatch_id, "dispatched_at": iso(now()), - "check_run_id": None, "remote_run_id": None, "remote_run_url": None, "remote_started_at": None, + "failure_policy": entry.get("failure_policy", "block"), "timeout_minutes": entry.get("timeout_minutes", 60), + "dispatched_at": now().strftime("%Y-%m-%dT%H:%M:%SZ"), "check_run_id": None, + "remote_run_id": None, "remote_run_url": None, "remote_started_at": None, "remote_conclusion": None, "conclusion": None, "error": None, } try: @@ -369,54 +276,38 @@ def dispatch_one(manifest, entry, target, state): # failed check rather than a required check that never appears. check = own.post(f"/repos/{REPO}/check-runs", { "name": check_name(manifest, entry["id"]), "head_sha": target["head_sha"], "status": "queued", - "external_id": our_external_id(entry["id"]), "details_url": ORCHESTRATOR_URL, - "output": { - "title": f"Dispatching {entry['repository']}", - "summary": f"Dispatching {code(entry['workflow'])} on {code(entry['repository'])} with F´ " - f"{code(fprime_rev(target))}.\n\nOrchestrator: {ORCHESTRATOR_URL}", - }, + "details_url": ORCHESTRATOR_URL, + "output": {"title": f"Dispatching {entry['repository']}", + "summary": f"Dispatching {code(entry['workflow'])} on {code(entry['repository'])} with F´ " + f"{code(fprime_rev(target))}.\n\nOrchestrator: {ORCHESTRATOR_URL}"}, }) s["check_run_id"] = check["id"] save_state(state) s["ext_ref"] = resolve_ext_ref(entry, target) + r = ext.post(f"/repos/{entry['repository']}/actions/workflows/{entry['workflow']}/dispatches", { + "ref": s["ext_ref"], + "inputs": {"fprime_repository": target["head_repo"], "fprime_ref": target["head_sha"], + "fprime_pr_number": str(target["pr"] or ""), "dispatch_id": f"{RUN_ID}-{ATTEMPT}-{entry['id']}"}, + "return_run_details": True, + }, idempotent=False) + if not r or not r.get("workflow_run_id"): + raise RuntimeError("dispatch accepted but no run id returned") + s["remote_run_id"], s["remote_run_url"] = r["workflow_run_id"], r.get("html_url") + log(f"[{entry['id']}] dispatched {entry['repository']}/{entry['workflow']} @ {s['ext_ref']} -> {s['remote_run_url']}") try: - r = ext.post(f"/repos/{entry['repository']}/actions/workflows/{entry['workflow']}/dispatches", { - "ref": s["ext_ref"], - "inputs": { - "fprime_repository": target["head_repo"], "fprime_ref": target["head_sha"], - "fprime_pr_number": str(target["pr"] or ""), "dispatch_id": dispatch_id, - }, - "return_run_details": True, - }, idempotent=False) - except ApiError as e: - if 0 < e.status < 500: - raise - # Lost response: the run may exist; correlate() finds it by dispatch_id or times out. - log(f"::warning::[{entry['id']}] dispatch outcome unknown ({e})") - r = None - if r and r.get("workflow_run_id"): - s["remote_run_id"], s["remote_run_url"] = r["workflow_run_id"], r.get("html_url") - mark_running(s) - where = f" -> {s['remote_run_url']}" if s["remote_run_url"] else f" (dispatch_id={dispatch_id})" - log(f"[{entry['id']}] dispatched {entry['repository']}/{entry['workflow']} @ {s['ext_ref']}{where}") + own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { + "status": "in_progress", "details_url": s["remote_run_url"], + "output": {"title": f"Running on {s['repository']} @ {s['ext_ref']}", + "summary": f"Remote run: {s['remote_run_url']}\n\nOrchestrator: {ORCHESTRATOR_URL}"}, + }) + except ApiError as e: # cosmetic: the run exists and is polled regardless + log(f"::warning::[{entry['id']}] check update: {e}") except Exception as e: # noqa: BLE001 - one repo's failure must not stop the fan-out s["error"], s["remote_conclusion"] = f"dispatch failed: {e}", "dispatch_failed" log(f"::error::[{entry['id']}] {s['error']}") save_state(state) -def mark_running(s): - # A failed status update is not a failed dispatch: the remote run exists and is polled anyway. - try: - own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { - "status": "in_progress", "details_url": s["remote_run_url"], - "output": {"title": f"Running on {s['repository']} @ {s['ext_ref']}", - "summary": f"Remote run: {s['remote_run_url']}\n\nOrchestrator: {ORCHESTRATOR_URL}"}, - }) - except ApiError as e: - log(f"::warning::[{s['id']}] check update: {e}") - - def cancel_run(s): try: ext.post(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}/cancel") @@ -424,48 +315,15 @@ def cancel_run(s): log(f" {e}") -def correlate(state): - """For dispatches without a run id (lost response, or an API server that still answers 204): - find the run whose title carries our dispatch_id (the ext workflow echoes it into run-name).""" - pending = [s for s in state["repos"].values() if not s.get("remote_run_id") and not s.get("remote_conclusion")] - deadline = time.time() + CORRELATE_TIMEOUT_S - while pending and time.time() < deadline: - for s in list(pending): - try: - since = iso(parse_ts(s["dispatched_at"]) - timedelta(seconds=60)) - runs = ext.get(f"/repos/{s['repository']}/actions/runs?event=workflow_dispatch" - f"&created={q('>=' + since)}&per_page=50") - run = next((r for r in runs["workflow_runs"] - if s["dispatch_id"] in (r.get("display_title") or r.get("name") or "").split()), None) - if not run: - continue - s["remote_run_id"], s["remote_run_url"] = run["id"], run["html_url"] - pending.remove(s) - log(f"[{s['id']}] remote run {run['html_url']}") - mark_running(s) - save_state(state) - except ApiError as e: - log(f"::warning::[{s['id']}] correlate: {e}") - if pending: - time.sleep(15) - for s in pending: - s["remote_conclusion"] = "not_found" - s["error"] = (f"no run with dispatch_id {s['dispatch_id']} appeared on {s['repository']} " - f"within {CORRELATE_TIMEOUT_S // 60} min") - log(f"::error::[{s['id']}] {s['error']}") - save_state(state) - - -# ---------------------------------------------------------------- poll def poll_all(state): def active(): - return [s for s in state["repos"].values() if s.get("remote_run_id") and not s.get("remote_conclusion")] + return [s for s in state["repos"].values() if s["remote_run_id"] and not s["remote_conclusion"]] while active(): time.sleep(POLL_S) for s in active(): try: - run = ext.get(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}", conditional=True) + run = ext.get(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}") if run.get("run_started_at") and not s["remote_started_at"]: s["remote_started_at"] = run["run_started_at"] timeout = timedelta(minutes=s["timeout_minutes"]) @@ -473,7 +331,7 @@ def active(): s["remote_conclusion"] = run["conclusion"] log(f"[{s['id']}] completed: {run['conclusion']}") elif s["remote_started_at"] and now() - parse_ts(s["remote_started_at"]) > timeout: - # Timeout counts from when the remote run started, not from dispatch (queue time is not the ext repo's fault). + # Counted from when the remote run started: queue time is not the ext repo's fault. log(f"::warning::[{s['id']}] timed out after {s['timeout_minutes']} min of running; cancelling remote run") cancel_run(s) s["remote_conclusion"] = "timed_out" @@ -513,7 +371,7 @@ def check_conclusion(s): if rc == "success": return "success" if s["failure_policy"] == "notify": - return "neutral" + return "neutral" # visible, but does not block the PR if rc == "cancelled": return "cancelled" if rc in ("timed_out", "queue_timeout"): @@ -528,103 +386,19 @@ def finalize_one(target, s): f"**{s['remote_conclusion']}**", ""] if s["remote_run_url"]: lines.append(f"Remote run: {s['remote_run_url']}") - if s.get("error"): + if s["error"]: lines.append(f"Error: {code(s['error'])}") if s["failure_policy"] == "notify" and s["remote_conclusion"] != "success": - lines += ["", f"This repo is {code('failure_policy: notify')}: the failure does not block the PR; " - f"owners ({mentions(s)}) have been notified."] - lines += ["", f'Orchestrator run (use "Re-run failed jobs" there to re-dispatch only the failed repos): {ORCHESTRATOR_URL}'] - if s["check_run_id"]: - output = {"title": f"{s['remote_conclusion']} ({s['repository']} @ {s['ext_ref'] or '-'})", "summary": "\n".join(lines)} - if details: - output["text"] = details - own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { - "status": "completed", "conclusion": conclusion, "details_url": s["remote_run_url"] or ORCHESTRATOR_URL, - "output": output, - }) + lines += ["", f"This repo is {code('failure_policy: notify')}: the failure does not block the PR."] + lines += ["", f"Orchestrator run: {ORCHESTRATOR_URL}"] + output = {"title": f"{s['remote_conclusion']} ({s['repository']} @ {s['ext_ref'] or '-'})", "summary": "\n".join(lines)} + if details: + output["text"] = details + own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { + "status": "completed", "conclusion": conclusion, "details_url": s["remote_run_url"] or ORCHESTRATOR_URL, + "output": output, + }) s["conclusion"] = conclusion # recorded only once the check is closed, so finalize() retries otherwise - if s["failure_policy"] == "notify": - try: - notify_owners(target, s, details) - except ApiError as e: - log(f"::warning::[{s['id']}] notify: {e}") - - -# notify policy: one PR comment per ext repo (updated in place), or one tracking issue when -# there is no PR (closed again once the repo passes). Only comments/issues authored by the -# Actions bot are considered ours, so nobody can plant a marker to suppress notifications. -def ensure_label(): - if own.exists(f"/repos/{REPO}/labels/{NOTIFY_LABEL}"): - return - try: - own.post(f"/repos/{REPO}/labels", - {"name": NOTIFY_LABEL, "color": "ededed", "description": "Opened by the external-CI orchestrator"}) - except ApiError: - pass - - -def notify_owners(target, s, details): - marker = f"" - failed = s["remote_conclusion"] != "success" - run_or_error = s["remote_run_url"] or code(s.get("error")) - - def ours(x): - return (x.get("user") or {}).get("login") == BOT_LOGIN and marker in (x.get("body") or "") - - if target["pr"]: - existing = None - for page in range(1, 11): - comments = own.get(f"/repos/{REPO}/issues/{target['pr']}/comments?per_page=100&page={page}") - existing = next((c for c in comments if ours(c)), None) - if existing or len(comments) < 100: - break - if not failed and not existing: - return - if failed: - body = (f"{marker}\n### External CI: {code(s['id'])} failed (non-blocking)\n\n" - f"{code(s['repository'])} @ {code(s['ext_ref'])} does not pass against {short(target['head_sha'])}: " - f"{run_or_error}\n\n{details}\n\n{mentions(s)} — this repo is {code('failure_policy: notify')}, " - "so the PR is not blocked, but it needs a look.") - else: - body = (f"{marker}\n### External CI: {code(s['id'])} passing again\n\n" - f"{code(s['repository'])} @ {code(s['ext_ref'])} passes against {short(target['head_sha'])}: " - f"{s['remote_run_url']}") - if existing: - own.patch(f"/repos/{REPO}/issues/comments/{existing['id']}", {"body": body}) - else: - own.post(f"/repos/{REPO}/issues/{target['pr']}/comments", {"body": body}) - return - title = f"[ext-ci] {s['id']} failing on {target['head_branch']}" - ensure_label() - issues = own.get(f"/repos/{REPO}/issues?state=open&labels={NOTIFY_LABEL}&creator={q(BOT_LOGIN)}&per_page=100") - existing = next((i for i in issues if not i.get("pull_request") and ours(i) and i["title"] == title), None) - if failed: - if existing: - own.post(f"/repos/{REPO}/issues/{existing['number']}/comments", - {"body": f"Still failing at {short(target['head_sha'])}: {run_or_error}"}) - else: - body = (f"{marker}\n{code(s['repository'])} @ {code(s['ext_ref'])} fails against {code(target['head_branch'])} " - f"({short(target['head_sha'])}): {run_or_error}\n\n{details}\n\n{mentions(s)}\n\n" - "This issue is closed automatically once the repo passes again.") - own.post(f"/repos/{REPO}/issues", {"title": title, "body": body, "labels": [NOTIFY_LABEL]}) - elif existing: - own.post(f"/repos/{REPO}/issues/{existing['number']}/comments", - {"body": f"Passing again at {short(target['head_sha'])}: {s['remote_run_url']}"}) - own.patch(f"/repos/{REPO}/issues/{existing['number']}", {"state": "closed", "state_reason": "completed"}) - - -def carry_forward(entry, prior, state): - state["repos"][entry["id"]] = { - "id": entry["id"], "repository": entry["repository"], "failure_policy": entry.get("failure_policy", "block"), - "carried_forward": True, "check_run_id": prior["id"], "remote_run_url": prior["details_url"], - "remote_conclusion": "success", "conclusion": "success", - } - own.patch(f"/repos/{REPO}/check-runs/{prior['id']}", {"output": { - "title": prior["output"]["title"], - "summary": f"{prior['output']['summary']}\n\n_Not re-run in attempt {ATTEMPT} of {ORCHESTRATOR_URL}: " - "previous success carried forward._", - }}) - log(f"[{entry['id']}] carried forward success from {prior['details_url']}") def skip_all(manifest, target, repos, state): @@ -632,24 +406,22 @@ def skip_all(manifest, target, repos, state): for entry in repos: check = own.post(f"/repos/{REPO}/check-runs", { "name": check_name(manifest, entry["id"]), "head_sha": target["head_sha"], - "status": "completed", "conclusion": "skipped", - "external_id": our_external_id(entry["id"]), "details_url": ORCHESTRATOR_URL, + "status": "completed", "conclusion": "skipped", "details_url": ORCHESTRATOR_URL, "output": {"title": "Not dispatched: no relevant files changed", "summary": f"Only paths matching {code(ignored)} changed.\n\nOrchestrator: {ORCHESTRATOR_URL}"}, }) state["repos"][entry["id"]] = { "id": entry["id"], "repository": entry["repository"], "failure_policy": entry.get("failure_policy", "block"), - "skipped": True, "check_run_id": check["id"], "remote_conclusion": "skipped", "conclusion": "skipped", + "check_run_id": check["id"], "remote_conclusion": "skipped", "conclusion": "skipped", } def write_summary(target, state): rows = [] for s in state["repos"].values(): - remote = "carried forward" if s.get("carried_forward") else s.get("remote_conclusion") link = f"[run]({s['remote_run_url']})" if s.get("remote_run_url") else (code(s["error"]) if s.get("error") else "") rows.append(f"| {code(s['id'])} | {s['repository']} | {code(s.get('ext_ref') or '-')} | {s['failure_policy']} " - f"| {remote} | {s.get('conclusion')} | {link} |") + f"| {s['remote_conclusion']} | {s['conclusion']} | {link} |") where = f" (PR #{target['pr']})" if target["pr"] else f" ({target['head_branch']})" md = "\n".join([f"### External CI for {code(fprime_rev(target))}{where} — attempt {ATTEMPT}", "", "| id | repository | ext ref | policy | remote | check | link |", @@ -667,32 +439,26 @@ def write_summary(target, state): def run(): target = resolve_target() log(f"target: {json.dumps(target)}") - manifest = load_manifest(target["base_ref"]) + with open(MANIFEST_PATH) as f: + manifest = json.load(f) # Fork preflight: a fork of F´ that copies this workflow must not dispatch into the ext repos. upstream = manifest.get("upstream_repository") if upstream and upstream != REPO and EVENT_NAME != "workflow_dispatch": log(f"::notice::{REPO} is not {upstream}; external CI is only dispatched from upstream. " "Use workflow_dispatch to run it here deliberately.") return - carried, dispatch = select_repos(manifest, target) + repos = select_repos(manifest, target) state = {"target": target, "attempt": ATTEMPT, "repos": {}} save_state(state) - for entry, prior in carried: - carry_forward(entry, prior, state) - ids = [e["id"] for e in dispatch] if EVENT_NAME != "workflow_dispatch" and not relevant_change(changed_files(target), manifest): - log(f"no relevant files changed; marking {ids} as skipped") - skip_all(manifest, target, dispatch, state) + log("no relevant files changed; marking all repos as skipped") + skip_all(manifest, target, repos, state) write_summary(target, state) return - log(f"dispatch: {ids} carried forward: {[e['id'] for e, _ in carried]}") - for entry in dispatch: + for entry in repos: dispatch_one(manifest, entry, target, state) - correlate(state) poll_all(state) for s in state["repos"].values(): - if s.get("carried_forward"): - continue try: finalize_one(target, s) except Exception as e: # noqa: BLE001 - the check stays open and finalize() closes it @@ -700,7 +466,7 @@ def run(): save_state(state) write_summary(target, state) blocking = [s["id"] for s in state["repos"].values() - if s["failure_policy"] != "notify" and (s.get("conclusion") or check_conclusion(s)) not in ("success", "skipped")] + if s["failure_policy"] != "notify" and (s["conclusion"] or check_conclusion(s)) not in ("success", "skipped")] if blocking: log("::error::blocking external repos failed: " + ", ".join(blocking)) sys.exit(1) @@ -709,27 +475,24 @@ def run(): def finalize(): """Runs under `if: always()`: whatever the run step left open (crash, cancellation, timeout) is closed here so no Check Run stays in_progress and no remote run keeps burning runners.""" - state = load_state() - if not state: - log("no state file; nothing to finalize") - return + if not os.path.exists(STATE_FILE): + return log("no state file; nothing to finalize") + with open(STATE_FILE) as f: + state = json.load(f) cancelled = env.get("JOB_STATUS") == "cancelled" for s in state["repos"].values(): - if s.get("carried_forward") or s.get("skipped") or s.get("conclusion"): + if s.get("conclusion") or not s.get("check_run_id"): continue remote = s.get("remote_run_url") or "(not started)" if s.get("remote_run_id") and not s.get("remote_conclusion"): log(f"[{s['id']}] cancelling {remote}") cancel_run(s) remote += " (cancelled)" - if not s.get("check_run_id"): - continue try: - current = own.get(f"/repos/{REPO}/check-runs/{s['check_run_id']}") + if own.get(f"/repos/{REPO}/check-runs/{s['check_run_id']}")["status"] == "completed": + continue except ApiError: - current = None - if current and current["status"] == "completed": - continue + pass if s.get("remote_conclusion") and not cancelled: # reported, but the check update itself failed in run conclusion, title = check_conclusion(s), f"{s['remote_conclusion']} ({s['repository']} @ {s.get('ext_ref') or '-'})" why = "Closed by the finalize step after the check update failed" diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index 1924a854b60..760db88981c 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -24,10 +24,6 @@ on: description: "Branch/SHA to test when no PR is given (default: the branch this run was started from)" type: string default: "" - force_all: - description: "On a re-run, re-dispatch every selected repo instead of only the failed ones" - type: boolean - default: false # One orchestrator per F´ head (repo + branch); a newer PR push cancels the older PR fan-out and # the finalize step below propagates that to the remote runs. Branch/nightly runs are never cancelled. @@ -38,8 +34,6 @@ concurrency: permissions: contents: read # manifest checks: write # per-repo Check Runs - pull-requests: write # notify-policy PR comments - issues: write # notify-policy tracking issues jobs: orchestrate: @@ -58,7 +52,6 @@ jobs: INPUT_REPOS: ${{ inputs.repos }} INPUT_PR_NUMBER: ${{ inputs.pr_number }} INPUT_FPRIME_REF: ${{ inputs.fprime_ref }} - INPUT_FORCE_ALL: ${{ inputs.force_all }} POLL_INTERVAL_SECONDS: 60 run: python3 .github/scripts/ext-ci-orchestrator.py run - name: "Finalize (close open checks, cancel orphaned remote runs)" From 4f6b54c3107c1248fe3151160e365e83f4a87de1 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 02:36:02 +0000 Subject: [PATCH 09/22] ci: docstrings and type hints for the ext-ci orchestrator --- .github/scripts/ext-ci-orchestrator.py | 143 +++++++++++++++++-------- 1 file changed, 101 insertions(+), 42 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index acd0534875c..95f9dcc10d8 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -42,45 +42,61 @@ ORCHESTRATOR_URL = f"{SERVER}/{REPO}/actions/runs/{RUN_ID}" + (f"/attempts/{ATTEMPT}" if ATTEMPT > 1 else "") -def log(msg): +def log(msg: str) -> None: + """Print one line to the job log, unbuffered so it interleaves correctly with GitHub's step output.""" print(msg, flush=True) -def code(s): - # Remote-controlled text (job/step names, branch names) is only ever rendered inside code spans. +def code(s: object) -> str: + """Render untrusted text (job/step names, branch names, errors) as a markdown code span. + + Backticks and newlines are replaced so remote-controlled text cannot inject markdown or links into + Check Run output. Returns the quoted string.""" return "`" + re.sub(r"[`\r\n]", " ", str(s)) + "`" -def q(s): +def q(s: object) -> str: + """Percent-encode one URL path segment (branch names may contain '/').""" return urllib.parse.quote(str(s), safe="") -def now(): +def now() -> datetime: + """Current time as a timezone-aware UTC datetime.""" return datetime.now(timezone.utc) -def parse_ts(s): +def parse_ts(s: str) -> datetime: + """Parse a GitHub API timestamp such as '2026-01-01T12:00:00Z' into an aware UTC datetime.""" return datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) -def fprime_rev(target): +def fprime_rev(target: dict) -> str: + """Short 'owner/repo@sha7' label of the F´ revision under test, for messages.""" return f"{target['head_repo']}@{target['head_sha'][:7]}" # ---------------------------------------------------------------- GitHub REST client class ApiError(Exception): - def __init__(self, msg, status=0): + """A GitHub API call failed. `status` is the HTTP status, or 0 for a network error / missing token.""" + + def __init__(self, msg: str, status: int = 0) -> None: super().__init__(msg) self.status = status class Client: - def __init__(self, token, label): + """Minimal GitHub REST client bound to one credential; `label` names it in logs and errors.""" + + def __init__(self, token: str | None, label: str) -> None: self.token, self.label = token, label - def request(self, method, url, body=None, idempotent=True): - """Returns (status, parsed body). Retries rate limits always; network errors and 5xx only - when the call is idempotent (a workflow dispatch is not: the run may have started).""" + def request(self, method: str, url: str, body=None, idempotent: bool = True) -> tuple: + """Perform one API call and return (HTTP status, parsed body). + + Never raises on a 4xx/5xx status: callers decide what a status means (see `api`, `exists`). + Retries rate limits always; network errors and 5xx only when the call is idempotent (a + workflow dispatch is not: the run may have started). Raises ApiError when the token is + missing or a non-idempotent call hits a network error.""" if not self.token: raise ApiError(f"{self.label} token missing (secret not configured?)") headers = { @@ -123,23 +139,27 @@ def request(self, method, url, body=None, idempotent=True): return status, text raise AssertionError("unreachable") - def api(self, method, url, body=None, **kw): + def api(self, method: str, url: str, body=None, **kw): + """`request` that raises ApiError on any non-2xx status; returns the parsed body.""" status, data = self.request(method, url, body, **kw) if not 200 <= status < 300: raise ApiError(f"{self.label}: {method} {url} -> {status} {json.dumps(data)[:300]}", status) return data - def get(self, url): + def get(self, url: str): + """GET; returns the parsed body or raises ApiError.""" return self.api("GET", url) - def post(self, url, body=None, **kw): + def post(self, url: str, body=None, **kw): + """POST a JSON body; returns the parsed response or raises ApiError.""" return self.api("POST", url, body, **kw) - def patch(self, url, body): + def patch(self, url: str, body): + """PATCH a JSON body; returns the parsed response or raises ApiError.""" return self.api("PATCH", url, body) - def exists(self, url): - # True/False for 200/404; anything else is an error, never silently "absent". + def exists(self, url: str) -> bool: + """True on 200, False on 404; any other status raises ApiError, never silently "absent".""" status, _ = self.request("GET", url) if status == 200: return True @@ -153,8 +173,14 @@ def exists(self, url): # ---------------------------------------------------------------- what to test -def resolve_target(): - """Which F´ revision is under test, and which PR (if any) it belongs to.""" +def resolve_target() -> dict: + """Decide which F´ revision is under test from the triggering event. + + Returns a dict with: `pr` (number or None), `head_repo` / `head_sha` / `head_branch` (the revision + to test; PR-author-controlled, only ever used as the SHA to dispatch and as display text), + `base_ref` (trusted branch the run was started from; drives the external branch lookup) and + `before_sha` (previous tip for pushes, else None). Raises RuntimeError when a PR run no longer + matches an open PR or a manual input is invalid.""" t = {"pr": None, "head_repo": REPO, "head_sha": None, "head_branch": None, "base_ref": None, "before_sha": None} if EVENT_NAME == "workflow_run": wr = EVENT["workflow_run"] @@ -199,8 +225,11 @@ def resolve_target(): return t -def changed_files(target): - """Files of the PR / push, or None when unknown (then everything runs).""" +def changed_files(target: dict) -> list[str] | None: + """List the files touched by the PR or push. + + Returns None when the set is unknown (schedule / manual runs, or a push whose `before` SHA is + gone after a force-push); callers then treat every repo as relevant.""" if target["pr"]: url = f"/repos/{REPO}/pulls/{target['pr']}/files?per_page=100" elif target["before_sha"]: @@ -222,16 +251,20 @@ def changed_files(target): return files -def relevant_change(files, manifest): - # paths_ignore lives here, not on the trigger workflow, so that filtered PRs still get a - # (skipped) Check Run per repo and required checks can be satisfied. +def relevant_change(files: list[str] | None, manifest) -> bool: + """True unless every changed file matches a `paths_ignore` glob (or the file list is unknown). + + paths_ignore lives here, not on the trigger workflow, so that filtered PRs still get a (skipped) + Check Run per repo and required checks can be satisfied.""" ignore = manifest.get("paths_ignore", []) if files is None or not ignore: return True return any(not any(fnmatch(f, g) for g in ignore) for f in files) -def select_repos(manifest, target): +def select_repos(manifest, target: dict) -> list[dict]: + """Return the manifest entries to run: the `repos` input ids on a manual run (unknown id raises + RuntimeError), entries with `on_pr: true` on a PR, otherwise all of them.""" repos = manifest["repos"] wanted = (env.get("INPUT_REPOS") or "all").strip() if EVENT_NAME == "workflow_dispatch" and wanted and wanted != "all": @@ -245,25 +278,34 @@ def select_repos(manifest, target): return repos -def resolve_ext_ref(entry, target): - """pr- branch on the ext repo, else the F´ base branch name, else the manifest default.""" +def resolve_ext_ref(entry, target: dict) -> str: + """Return the external repo branch to dispatch: its `pr-` branch if it exists, else a branch + named like the F´ base branch (`devel`, `release/x`), else the entry's `default_ref`.""" for c in (f"pr-{target['pr']}" if target["pr"] else None, target["base_ref"]): if c and ext.exists(f"/repos/{entry['repository']}/branches/{q(c)}"): return c return entry.get("default_ref", "main") -def check_name(manifest, repo_id): +def check_name(manifest, repo_id: str) -> str: + """Name of the Check Run for one manifest entry: ' / '.""" return f"{manifest.get('check_prefix', 'ext-ci')} / {repo_id}" -def save_state(state): +def save_state(state: dict) -> None: + """Write the run state to STATE_FILE; called after every step so `finalize` can recover it.""" with open(STATE_FILE, "w") as f: json.dump(state, f, indent=2) # ---------------------------------------------------------------- dispatch and poll -def dispatch_one(manifest, entry, target, state): +def dispatch_one(manifest, entry, target: dict, state: dict) -> None: + """Start one external repo: create its Check Run (queued), dispatch its workflow with the F´ + revision, mark the check in_progress with the remote run link. + + Records progress in state["repos"][id] (keys: check_run_id, ext_ref, remote_run_id/_url, + remote_started_at, remote_conclusion, conclusion, error, ...). Never raises: any failure ends + this repo as `remote_conclusion: dispatch_failed` so the other repos still run.""" s = state["repos"][entry["id"]] = { "id": entry["id"], "repository": entry["repository"], "workflow": entry["workflow"], "ext_ref": None, "failure_policy": entry.get("failure_policy", "block"), "timeout_minutes": entry.get("timeout_minutes", 60), @@ -308,15 +350,22 @@ def dispatch_one(manifest, entry, target, state): save_state(state) -def cancel_run(s): +def cancel_run(s: dict) -> None: + """Cancel the remote run of one repo; API errors are logged, not raised.""" try: ext.post(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}/cancel") except ApiError as e: log(f" {e}") -def poll_all(state): - def active(): +def poll_all(state: dict) -> None: + """Poll every dispatched run every POLL_S seconds until each has a `remote_conclusion`. + + Sets it to the run's conclusion, `timed_out` (running longer than timeout_minutes, remote run + cancelled), `queue_timeout` (not started after 2x timeout) or `error` (401/404 while polling). + Other poll errors are retried on the next tick.""" + + def active() -> list[dict]: return [s for s in state["repos"].values() if s["remote_run_id"] and not s["remote_conclusion"]] while active(): @@ -349,7 +398,8 @@ def active(): # ---------------------------------------------------------------- report -def failure_details(s): +def failure_details(s: dict) -> str: + """Markdown list of the remote run's failed jobs and steps (empty when there is no remote run).""" if not s["remote_run_id"]: return "" try: @@ -366,7 +416,9 @@ def failure_details(s): return "\n".join(lines) -def check_conclusion(s): +def check_conclusion(s: dict) -> str: + """Map the remote result to the Check Run conclusion: success, neutral (any failure on a + `failure_policy: notify` repo), cancelled, timed_out or failure.""" rc = s["remote_conclusion"] if rc == "success": return "success" @@ -379,7 +431,10 @@ def check_conclusion(s): return "failure" -def finalize_one(target, s): +def finalize_one(target: dict, s: dict) -> None: + """Close one repo's Check Run with its conclusion, summary and failed-job details. + + Sets s["conclusion"] only after the update succeeded, so `finalize` retries it otherwise.""" conclusion = check_conclusion(s) details = "" if s["remote_conclusion"] == "success" else failure_details(s) lines = [f"{code(s['repository'])} @ {code(s['ext_ref'] or '-')} with F´ {code(fprime_rev(target))}: " @@ -401,7 +456,8 @@ def finalize_one(target, s): s["conclusion"] = conclusion # recorded only once the check is closed, so finalize() retries otherwise -def skip_all(manifest, target, repos, state): +def skip_all(manifest, target: dict, repos: list[dict], state: dict) -> None: + """Create every repo's Check Run directly as completed/skipped (only `paths_ignore` files changed).""" ignored = ", ".join(manifest.get("paths_ignore", [])) for entry in repos: check = own.post(f"/repos/{REPO}/check-runs", { @@ -416,7 +472,8 @@ def skip_all(manifest, target, repos, state): } -def write_summary(target, state): +def write_summary(target: dict, state: dict) -> None: + """Write the per-repo results table to the job summary and the log, and RESULTS_FILE as JSON.""" rows = [] for s in state["repos"].values(): link = f"[run]({s['remote_run_url']})" if s.get("remote_run_url") else (code(s["error"]) if s.get("error") else "") @@ -436,7 +493,9 @@ def write_summary(target, state): # ---------------------------------------------------------------- modes -def run(): +def run() -> None: + """The `run` mode: resolve target, load manifest, fork preflight, select repos, dispatch, poll, + close checks, write summary. Exits 1 when a `failure_policy: block` repo did not succeed.""" target = resolve_target() log(f"target: {json.dumps(target)}") with open(MANIFEST_PATH) as f: @@ -472,8 +531,8 @@ def run(): sys.exit(1) -def finalize(): - """Runs under `if: always()`: whatever the run step left open (crash, cancellation, timeout) is +def finalize() -> None: + """The `finalize` mode, run under `if: always()`: whatever the run step left open (crash, cancellation, timeout) is closed here so no Check Run stays in_progress and no remote run keeps burning runners.""" if not os.path.exists(STATE_FILE): return log("no state file; nothing to finalize") From 19cbe9485d3ff197e22510a2ac9794eb3733eeef Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 17:46:09 +0000 Subject: [PATCH 10/22] ci: ext-ci review follow-ups: step-scoped dispatch token, on_pr on manual runs, readability --- .github/scripts/ext-ci-orchestrator.py | 97 +++++++++++++++----------- .github/workflows/ext-ci.yml | 3 +- 2 files changed, 60 insertions(+), 40 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index 95f9dcc10d8..fa952df0cca 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -47,17 +47,17 @@ def log(msg: str) -> None: print(msg, flush=True) -def code(s: object) -> str: +def code(text: object) -> str: """Render untrusted text (job/step names, branch names, errors) as a markdown code span. Backticks and newlines are replaced so remote-controlled text cannot inject markdown or links into Check Run output. Returns the quoted string.""" - return "`" + re.sub(r"[`\r\n]", " ", str(s)) + "`" + return "`" + re.sub(r"[`\r\n]", " ", str(text)) + "`" -def q(s: object) -> str: +def q(text: object) -> str: """Percent-encode one URL path segment (branch names may contain '/').""" - return urllib.parse.quote(str(s), safe="") + return urllib.parse.quote(str(text), safe="") def now() -> datetime: @@ -96,7 +96,7 @@ def request(self, method: str, url: str, body=None, idempotent: bool = True) -> Never raises on a 4xx/5xx status: callers decide what a status means (see `api`, `exists`). Retries rate limits always; network errors and 5xx only when the call is idempotent (a workflow dispatch is not: the run may have started). Raises ApiError when the token is - missing or a non-idempotent call hits a network error.""" + missing, when a non-idempotent call hits a network error, or after five failed attempts.""" if not self.token: raise ApiError(f"{self.label} token missing (secret not configured?)") headers = { @@ -139,9 +139,9 @@ def request(self, method: str, url: str, body=None, idempotent: bool = True) -> return status, text raise AssertionError("unreachable") - def api(self, method: str, url: str, body=None, **kw): + def api(self, method: str, url: str, body=None, idempotent: bool = True): """`request` that raises ApiError on any non-2xx status; returns the parsed body.""" - status, data = self.request(method, url, body, **kw) + status, data = self.request(method, url, body, idempotent) if not 200 <= status < 300: raise ApiError(f"{self.label}: {method} {url} -> {status} {json.dumps(data)[:300]}", status) return data @@ -150,9 +150,9 @@ def get(self, url: str): """GET; returns the parsed body or raises ApiError.""" return self.api("GET", url) - def post(self, url: str, body=None, **kw): + def post(self, url: str, body=None, idempotent: bool = True): """POST a JSON body; returns the parsed response or raises ApiError.""" - return self.api("POST", url, body, **kw) + return self.api("POST", url, body, idempotent) def patch(self, url: str, body): """PATCH a JSON body; returns the parsed response or raises ApiError.""" @@ -177,10 +177,11 @@ def resolve_target() -> dict: """Decide which F´ revision is under test from the triggering event. Returns a dict with: `pr` (number or None), `head_repo` / `head_sha` / `head_branch` (the revision - to test; PR-author-controlled, only ever used as the SHA to dispatch and as display text), + to test; PR-author-controlled: forwarded verbatim as the dispatch inputs the external `ci.yml` + checks out, and otherwise used only to find the PR and as display text), `base_ref` (trusted branch the run was started from; drives the external branch lookup) and `before_sha` (previous tip for pushes, else None). Raises RuntimeError when a PR run no longer - matches an open PR or a manual input is invalid.""" + matches an open PR or `pr_number` is not numeric, ApiError when a manual PR/ref does not exist.""" t = {"pr": None, "head_repo": REPO, "head_sha": None, "head_branch": None, "base_ref": None, "before_sha": None} if EVENT_NAME == "workflow_run": wr = EVENT["workflow_run"] @@ -251,43 +252,50 @@ def changed_files(target: dict) -> list[str] | None: return files -def relevant_change(files: list[str] | None, manifest) -> bool: - """True unless every changed file matches a `paths_ignore` glob (or the file list is unknown). +def relevant_change(files: list[str] | None, manifest: dict) -> bool: + """False only when every changed file matches a `paths_ignore` glob; an unknown list (None) counts + as relevant. paths_ignore lives here, not on the trigger workflow, so that filtered PRs still get a (skipped) Check Run per repo and required checks can be satisfied.""" ignore = manifest.get("paths_ignore", []) if files is None or not ignore: return True - return any(not any(fnmatch(f, g) for g in ignore) for f in files) + # fnmatch's '*' also matches '/', so 'docs/**' behaves like 'docs/*' (good enough here). + ignored = lambda f: any(fnmatch(f, g) for g in ignore) # noqa: E731 + return not all(ignored(f) for f in files) -def select_repos(manifest, target: dict) -> list[dict]: - """Return the manifest entries to run: the `repos` input ids on a manual run (unknown id raises - RuntimeError), entries with `on_pr: true` on a PR, otherwise all of them.""" +def select_repos(manifest: dict, target: dict) -> list[dict]: + """Return the manifest entries to run: all of them, narrowed to the `repos` input ids on a manual run + (unknown id raises RuntimeError) and to entries with `on_pr` true (the default) when a PR is tested, + manual runs included.""" repos = manifest["repos"] wanted = (env.get("INPUT_REPOS") or "all").strip() if EVENT_NAME == "workflow_dispatch" and wanted and wanted != "all": ids = [s.strip() for s in wanted.split(",") if s.strip()] - unknown = [i for i in ids if not any(r["id"] == i for r in repos)] + unknown = [i for i in ids if not any(entry["id"] == i for entry in repos)] if unknown: raise RuntimeError("unknown manifest ids: " + ", ".join(unknown)) - repos = [r for r in repos if r["id"] in ids] - elif target["pr"]: - repos = [r for r in repos if r.get("on_pr", True)] + repos = [entry for entry in repos if entry["id"] in ids] + if target["pr"]: + repos = [entry for entry in repos if entry.get("on_pr", True)] return repos -def resolve_ext_ref(entry, target: dict) -> str: +def resolve_ext_ref(entry: dict, target: dict) -> str: """Return the external repo branch to dispatch: its `pr-` branch if it exists, else a branch - named like the F´ base branch (`devel`, `release/x`), else the entry's `default_ref`.""" - for c in (f"pr-{target['pr']}" if target["pr"] else None, target["base_ref"]): - if c and ext.exists(f"/repos/{entry['repository']}/branches/{q(c)}"): - return c + named like the F´ base branch (`devel`, `release/x`), else the entry's `default_ref` (`main`).""" + candidates = [target["base_ref"]] + if target["pr"]: + candidates.insert(0, f"pr-{target['pr']}") + for candidate in candidates: + if candidate and ext.exists(f"/repos/{entry['repository']}/branches/{q(candidate)}"): + return candidate return entry.get("default_ref", "main") -def check_name(manifest, repo_id: str) -> str: +def check_name(manifest: dict, repo_id: str) -> str: """Name of the Check Run for one manifest entry: ' / '.""" return f"{manifest.get('check_prefix', 'ext-ci')} / {repo_id}" @@ -299,7 +307,7 @@ def save_state(state: dict) -> None: # ---------------------------------------------------------------- dispatch and poll -def dispatch_one(manifest, entry, target: dict, state: dict) -> None: +def dispatch_one(manifest: dict, entry: dict, target: dict, state: dict) -> None: """Start one external repo: create its Check Run (queued), dispatch its workflow with the F´ revision, mark the check in_progress with the remote run link. @@ -355,7 +363,7 @@ def cancel_run(s: dict) -> None: try: ext.post(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}/cancel") except ApiError as e: - log(f" {e}") + log(f"::warning::[{s['id']}] cancel: {e}") def poll_all(state: dict) -> None: @@ -453,10 +461,10 @@ def finalize_one(target: dict, s: dict) -> None: "status": "completed", "conclusion": conclusion, "details_url": s["remote_run_url"] or ORCHESTRATOR_URL, "output": output, }) - s["conclusion"] = conclusion # recorded only once the check is closed, so finalize() retries otherwise + s["conclusion"] = conclusion -def skip_all(manifest, target: dict, repos: list[dict], state: dict) -> None: +def skip_all(manifest: dict, target: dict, repos: list[dict], state: dict) -> None: """Create every repo's Check Run directly as completed/skipped (only `paths_ignore` files changed).""" ignored = ", ".join(manifest.get("paths_ignore", [])) for entry in repos: @@ -476,10 +484,15 @@ def write_summary(target: dict, state: dict) -> None: """Write the per-repo results table to the job summary and the log, and RESULTS_FILE as JSON.""" rows = [] for s in state["repos"].values(): - link = f"[run]({s['remote_run_url']})" if s.get("remote_run_url") else (code(s["error"]) if s.get("error") else "") + if s.get("remote_run_url"): + link = f"[run]({s['remote_run_url']})" + elif s.get("error"): + link = code(s["error"]) + else: + link = "" rows.append(f"| {code(s['id'])} | {s['repository']} | {code(s.get('ext_ref') or '-')} | {s['failure_policy']} " f"| {s['remote_conclusion']} | {s['conclusion']} | {link} |") - where = f" (PR #{target['pr']})" if target["pr"] else f" ({target['head_branch']})" + where = f" (PR #{target['pr']})" if target["pr"] else f" ({code(target['head_branch'])})" md = "\n".join([f"### External CI for {code(fprime_rev(target))}{where} — attempt {ATTEMPT}", "", "| id | repository | ext ref | policy | remote | check | link |", "|---|---|---|---|---|---|---|", *rows, ""]) @@ -524,8 +537,9 @@ def run() -> None: log(f"::error::[{s['id']}] finalize: {e}") save_state(state) write_summary(target, state) + # conclusion is None when finalize_one failed: judge on what it would have been. blocking = [s["id"] for s in state["repos"].values() - if s["failure_policy"] != "notify" and (s["conclusion"] or check_conclusion(s)) not in ("success", "skipped")] + if s["failure_policy"] != "notify" and (s["conclusion"] or check_conclusion(s)) != "success"] if blocking: log("::error::blocking external repos failed: " + ", ".join(blocking)) sys.exit(1) @@ -552,11 +566,11 @@ def finalize() -> None: continue except ApiError: pass - if s.get("remote_conclusion") and not cancelled: # reported, but the check update itself failed in run + if cancelled: + conclusion, title, why = "cancelled", "Cancelled by orchestrator", "The orchestrator run was cancelled" + elif s.get("remote_conclusion"): # reported, but the check update itself failed in run conclusion, title = check_conclusion(s), f"{s['remote_conclusion']} ({s['repository']} @ {s.get('ext_ref') or '-'})" why = "Closed by the finalize step after the check update failed" - elif cancelled: - conclusion, title, why = "cancelled", "Cancelled by orchestrator", "The orchestrator run was cancelled" else: conclusion = "neutral" if s["failure_policy"] == "notify" else "failure" title = "Orchestrator error" @@ -567,8 +581,13 @@ def finalize() -> None: "output": {"title": title, "summary": f"{why}; remote run {remote}.\n\n{ORCHESTRATOR_URL}"}, }) except ApiError as e: - log(f" {e}") + log(f"::warning::[{s['id']}] check update: {e}") if __name__ == "__main__": - finalize() if MODE == "finalize" else run() + if MODE == "finalize": + finalize() + elif MODE == "run": + run() + else: + sys.exit(f"usage: {sys.argv[0]} run|finalize") diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index 760db88981c..cecdf9d680c 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -44,11 +44,11 @@ jobs: timeout-minutes: 180 env: GITHUB_TOKEN: ${{ github.token }} - DISPATCH_TOKEN: ${{ secrets.EXT_CI_DISPATCH_TOKEN }} steps: - uses: actions/checkout@v4 - name: "Dispatch, poll and report" env: + DISPATCH_TOKEN: ${{ secrets.EXT_CI_DISPATCH_TOKEN }} # external repos only; not exposed to other steps INPUT_REPOS: ${{ inputs.repos }} INPUT_PR_NUMBER: ${{ inputs.pr_number }} INPUT_FPRIME_REF: ${{ inputs.fprime_ref }} @@ -57,6 +57,7 @@ jobs: - name: "Finalize (close open checks, cancel orphaned remote runs)" if: always() env: + DISPATCH_TOKEN: ${{ secrets.EXT_CI_DISPATCH_TOKEN }} # external repos only; not exposed to other steps JOB_STATUS: ${{ job.status }} run: python3 .github/scripts/ext-ci-orchestrator.py finalize - uses: actions/upload-artifact@v4 From 646ce8fb7509f916a4f18a96eb12a6fcccd11efe Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 17:52:21 +0000 Subject: [PATCH 11/22] ci: ext-ci runs for PRs, merge queue and manual dispatch only; orchestrator faults fail the job --- .github/scripts/ext-ci-orchestrator.py | 45 ++++++++++---------------- .github/workflows/ext-ci.yml | 6 +--- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index fa952df0cca..204f3309b65 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -32,6 +32,7 @@ RESULTS_FILE = "ext-ci-results.json" MANIFEST_PATH = ".github/ext-ci-manifest.json" POLL_S = int(env.get("POLL_INTERVAL_SECONDS") or 60) +ORCHESTRATOR_FAULTS = ("dispatch_failed", "queue_timeout", "error") # remote_conclusion values we made up REPO = env.get("GITHUB_REPOSITORY") RUN_ID = env.get("GITHUB_RUN_ID") @@ -180,9 +181,9 @@ def resolve_target() -> dict: to test; PR-author-controlled: forwarded verbatim as the dispatch inputs the external `ci.yml` checks out, and otherwise used only to find the PR and as display text), `base_ref` (trusted branch the run was started from; drives the external branch lookup) and - `before_sha` (previous tip for pushes, else None). Raises RuntimeError when a PR run no longer - matches an open PR or `pr_number` is not numeric, ApiError when a manual PR/ref does not exist.""" - t = {"pr": None, "head_repo": REPO, "head_sha": None, "head_branch": None, "base_ref": None, "before_sha": None} + Raises RuntimeError when a PR run no longer matches an open PR or `pr_number` is not numeric, + ApiError when a manual PR/ref does not exist.""" + t = {"pr": None, "head_repo": REPO, "head_sha": None, "head_branch": None, "base_ref": None} if EVENT_NAME == "workflow_run": wr = EVENT["workflow_run"] # Trust only the event payload: nothing produced by the (untrusted) trigger run is read. @@ -218,34 +219,19 @@ def resolve_target() -> dict: commit = own.get(f"/repos/{REPO}/commits/{q(ref)}") # base_ref (ext branch lookup) is the branch this run was started from, not the ref under test. t.update(head_sha=commit["sha"], head_branch=ref, base_ref=env["GITHUB_REF_NAME"]) - else: # push, schedule - t.update(head_sha=env["GITHUB_SHA"], head_branch=env["GITHUB_REF_NAME"], base_ref=env["GITHUB_REF_NAME"]) - before = EVENT.get("before") - if EVENT_NAME == "push" and before and not re.fullmatch("0+", before): - t["before_sha"] = before + else: + raise RuntimeError(f"unsupported event: {EVENT_NAME}") return t def changed_files(target: dict) -> list[str] | None: - """List the files touched by the PR or push. - - Returns None when the set is unknown (schedule / manual runs, or a push whose `before` SHA is - gone after a force-push); callers then treat every repo as relevant.""" - if target["pr"]: - url = f"/repos/{REPO}/pulls/{target['pr']}/files?per_page=100" - elif target["before_sha"]: - url = f"/repos/{REPO}/compare/{target['before_sha']}...{target['head_sha']}?per_page=100" - else: + """List the files touched by the PR; None when there is no PR (manual ref runs), which callers + treat as "every repo is relevant".""" + if not target["pr"]: return None files = [] - for page in range(1, 31): - try: - r = own.get(f"{url}&page={page}") - except ApiError as e: - if e.status == 404 and not target["pr"]: - return None # before SHA no longer in history (force-push): run everything - raise - batch = r if target["pr"] else r.get("files", []) + for page in range(1, 31): # GitHub lists at most 3000 files per PR + batch = own.get(f"/repos/{REPO}/pulls/{target['pr']}/files?per_page=100&page={page}") files += [f["filename"] for f in batch] if len(batch) < 100: break @@ -508,7 +494,8 @@ def write_summary(target: dict, state: dict) -> None: # ---------------------------------------------------------------- modes def run() -> None: """The `run` mode: resolve target, load manifest, fork preflight, select repos, dispatch, poll, - close checks, write summary. Exits 1 when a `failure_policy: block` repo did not succeed.""" + close checks, write summary. Exits 1 when a `failure_policy: block` repo did not succeed or the + orchestrator itself failed on any repo.""" target = resolve_target() log(f"target: {json.dumps(target)}") with open(MANIFEST_PATH) as f: @@ -537,9 +524,11 @@ def run() -> None: log(f"::error::[{s['id']}] finalize: {e}") save_state(state) write_summary(target, state) - # conclusion is None when finalize_one failed: judge on what it would have been. + # conclusion is None when finalize_one failed: judge on what it would have been. `notify` only + # softens a remote build failure; an orchestrator-side fault always fails this job. blocking = [s["id"] for s in state["repos"].values() - if s["failure_policy"] != "notify" and (s["conclusion"] or check_conclusion(s)) != "success"] + if (s["conclusion"] or check_conclusion(s)) != "success" + and (s["failure_policy"] != "notify" or s["remote_conclusion"] in ORCHESTRATOR_FAULTS)] if blocking: log("::error::blocking external repos failed: " + ", ".join(blocking)) sys.exit(1) diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index cecdf9d680c..ebd8ec15e56 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -6,10 +6,6 @@ on: workflow_run: workflows: [ ext-ci-trigger ] types: [ completed ] - push: - branches: [ release/** ] - schedule: - - cron: "0 21 * * *" # Nightly at 2:00 PM US/Pacific (PDT), as the ext-*.yml workflows today workflow_dispatch: inputs: repos: @@ -26,7 +22,7 @@ on: default: "" # One orchestrator per F´ head (repo + branch); a newer PR push cancels the older PR fan-out and -# the finalize step below propagates that to the remote runs. Branch/nightly runs are never cancelled. +# the finalize step below propagates that to the remote runs. Manual runs are never cancelled. concurrency: group: ext-ci-${{ github.event_name }}-${{ github.event.workflow_run.head_repository.full_name || github.repository }}-${{ github.event.workflow_run.head_branch || inputs.pr_number || inputs.fprime_ref || github.ref_name }} cancel-in-progress: ${{ github.event_name == 'workflow_run' }} From 219317eb414fc288acbb6977734a2c347715280e Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 17:53:19 +0000 Subject: [PATCH 12/22] ci: ext-ci re-runs dispatch only the repos whose check is not green --- .github/scripts/ext-ci-orchestrator.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index 204f3309b65..ace47633a79 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -286,6 +286,13 @@ def check_name(manifest: dict, repo_id: str) -> str: return f"{manifest.get('check_prefix', 'ext-ci')} / {repo_id}" +def check_succeeded(manifest: dict, entry: dict, target: dict) -> bool: + """True when this entry's latest Check Run on the F´ commit (from an earlier attempt) succeeded.""" + runs = own.get(f"/repos/{REPO}/commits/{target['head_sha']}/check-runs" + f"?check_name={q(check_name(manifest, entry['id']))}&filter=latest")["check_runs"] + return any(r["conclusion"] == "success" for r in runs) + + def save_state(state: dict) -> None: """Write the run state to STATE_FILE; called after every step so `finalize` can recover it.""" with open(STATE_FILE, "w") as f: @@ -507,6 +514,9 @@ def run() -> None: "Use workflow_dispatch to run it here deliberately.") return repos = select_repos(manifest, target) + if ATTEMPT > 1: # re-run: only the repos whose check is not green yet + repos = [e for e in repos if not check_succeeded(manifest, e, target)] + log(f"attempt {ATTEMPT}: re-dispatching {[e['id'] for e in repos] or 'nothing'}") state = {"target": target, "attempt": ATTEMPT, "repos": {}} save_state(state) if EVENT_NAME != "workflow_dispatch" and not relevant_change(changed_files(target), manifest): From d2045d385280b786ea654a3f60b2b2d4b6df800c Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 17:55:44 +0000 Subject: [PATCH 13/22] ci: ext-ci header documents the manifest and external ci.yml contract; notify repos never cancelled-red --- .github/scripts/ext-ci-orchestrator.py | 19 +++++++++++++++++-- .github/workflows/ext-ci-trigger.yml | 5 +++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index ace47633a79..ebdee38da80 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -6,7 +6,21 @@ For every repo in .github/ext-ci-manifest.json: create a Check Run on the F´ commit under test, dispatch the repo's own ci.yml with that commit, poll the remote run, close the Check Run with -its result. A re-run of the orchestrator re-dispatches every repo. +its result. A re-run of the orchestrator re-dispatches only the repos whose check is not green. + +Manifest (.github/ext-ci-manifest.json): + upstream_repository only this repo dispatches automatically; forks only on workflow_dispatch + check_prefix Check Run name is " / " + paths_ignore globs; a PR touching only these gets "skipped" checks and no dispatch + repos[]: id, repository (owner/name), workflow (file name, must accept the inputs below), + default_ref (branch to dispatch when no pr- / base-named branch exists), + timeout_minutes (of remote running time; 2x while queued), failure_policy + ("block": failure is a red check and fails this job; "notify": neutral check), + on_pr (false: never dispatched for a PR, only merge queue / manual) + +Contract for the external ci.yml: `on: workflow_dispatch` with inputs fprime_repository, +fprime_ref (exact SHA to overlay as the F´ dependency), fprime_pr_number, dispatch_id; no secrets, +read-only permissions, no cache writes on dispatched runs. Two credentials, never mixed: GITHUB_TOKEN this repo only (checks: write) - reports results @@ -566,7 +580,8 @@ def finalize() -> None: except ApiError: pass if cancelled: - conclusion, title, why = "cancelled", "Cancelled by orchestrator", "The orchestrator run was cancelled" + conclusion = "neutral" if s["failure_policy"] == "notify" else "cancelled" + title, why = "Cancelled by orchestrator", "The orchestrator run was cancelled" elif s.get("remote_conclusion"): # reported, but the check update itself failed in run conclusion, title = check_conclusion(s), f"{s['remote_conclusion']} ({s['repository']} @ {s.get('ext_ref') or '-'})" why = "Closed by the finalize step after the check update failed" diff --git a/.github/workflows/ext-ci-trigger.yml b/.github/workflows/ext-ci-trigger.yml index fd8bd281977..4c949fb507e 100644 --- a/.github/workflows/ext-ci-trigger.yml +++ b/.github/workflows/ext-ci-trigger.yml @@ -1,8 +1,9 @@ name: ext-ci-trigger # Deliberately trivial. This workflow runs under the repository's normal `pull_request` -# rules (including "require approval for workflows from forks"), and its completion is what -# starts the privileged orchestrator in ext-ci.yml via `workflow_run`. It must never need +# rules, and its completion is what starts the privileged orchestrator in ext-ci.yml via +# `workflow_run`. The design assumes the repository setting "Require approval for all outside +# collaborators" (GitHub's default gates first-time contributors only). It must never need # secrets and must never produce anything the orchestrator trusts. No paths filter here: the # orchestrator applies the manifest's paths_ignore itself and emits skipped Check Runs, so # docs-only PRs still satisfy required checks. From e447fd1135e181f0b12291fda4fa8a38053fe195 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 18:22:54 +0000 Subject: [PATCH 14/22] ci: black-format the ext-ci orchestrator --- .github/scripts/ext-ci-orchestrator.py | 404 ++++++++++++++++++------- 1 file changed, 302 insertions(+), 102 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index ebdee38da80..59ca6792752 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -27,6 +27,7 @@ DISPATCH_TOKEN external repos only (actions: write; contents: read too if an external repo is private, for the branch lookup) - starts remote runs """ + import json import os import re @@ -46,7 +47,11 @@ RESULTS_FILE = "ext-ci-results.json" MANIFEST_PATH = ".github/ext-ci-manifest.json" POLL_S = int(env.get("POLL_INTERVAL_SECONDS") or 60) -ORCHESTRATOR_FAULTS = ("dispatch_failed", "queue_timeout", "error") # remote_conclusion values we made up +ORCHESTRATOR_FAULTS = ( + "dispatch_failed", + "queue_timeout", + "error", +) # remote_conclusion values we made up REPO = env.get("GITHUB_REPOSITORY") RUN_ID = env.get("GITHUB_RUN_ID") @@ -54,7 +59,9 @@ EVENT_NAME = env.get("GITHUB_EVENT_NAME") with open(env["GITHUB_EVENT_PATH"]) as _f: EVENT = json.load(_f) -ORCHESTRATOR_URL = f"{SERVER}/{REPO}/actions/runs/{RUN_ID}" + (f"/attempts/{ATTEMPT}" if ATTEMPT > 1 else "") +ORCHESTRATOR_URL = f"{SERVER}/{REPO}/actions/runs/{RUN_ID}" + ( + f"/attempts/{ATTEMPT}" if ATTEMPT > 1 else "" +) def log(msg: str) -> None: @@ -105,13 +112,16 @@ class Client: def __init__(self, token: str | None, label: str) -> None: self.token, self.label = token, label - def request(self, method: str, url: str, body=None, idempotent: bool = True) -> tuple: + def request( + self, method: str, url: str, body=None, idempotent: bool = True + ) -> tuple: """Perform one API call and return (HTTP status, parsed body). Never raises on a 4xx/5xx status: callers decide what a status means (see `api`, `exists`). Retries rate limits always; network errors and 5xx only when the call is idempotent (a workflow dispatch is not: the run may have started). Raises ApiError when the token is - missing, when a non-idempotent call hits a network error, or after five failed attempts.""" + missing, when a non-idempotent call hits a network error, or after five failed attempts. + """ if not self.token: raise ApiError(f"{self.label} token missing (secret not configured?)") headers = { @@ -125,27 +135,41 @@ def request(self, method: str, url: str, body=None, idempotent: bool = True) -> data = json.dumps(body).encode() headers["Content-Type"] = "application/json" for i in range(5): - req = urllib.request.Request(API + url, data=data, method=method, headers=headers) + req = urllib.request.Request( + API + url, data=data, method=method, headers=headers + ) try: with urllib.request.urlopen(req, timeout=60) as res: status, hdrs, text = res.status, res.headers, res.read().decode() except urllib.error.HTTPError as e: - status, hdrs, text = e.code, e.headers, e.read().decode(errors="replace") + status, hdrs, text = ( + e.code, + e.headers, + e.read().decode(errors="replace"), + ) except (urllib.error.URLError, OSError) as e: if not idempotent or i == 4: raise ApiError(f"{self.label}: {method} {url} -> network error {e}") time.sleep(2 * 2**i) continue limited = status == 429 or ( - status == 403 and (hdrs.get("x-ratelimit-remaining") == "0" or re.search("rate limit", text, re.I)) + status == 403 + and ( + hdrs.get("x-ratelimit-remaining") == "0" + or re.search("rate limit", text, re.I) + ) ) if (limited or (status >= 500 and idempotent)) and i < 4: wait = 2 * 2**i if hdrs.get("retry-after"): wait = int(hdrs["retry-after"]) elif limited and hdrs.get("x-ratelimit-reset"): - wait = max(wait, int(hdrs["x-ratelimit-reset"]) - int(time.time()) + 1) - log(f"::warning::{self.label}: {method} {url} -> {status}; retrying in {wait} s") + wait = max( + wait, int(hdrs["x-ratelimit-reset"]) - int(time.time()) + 1 + ) + log( + f"::warning::{self.label}: {method} {url} -> {status}; retrying in {wait} s" + ) time.sleep(min(wait, 15 * 60)) continue try: @@ -158,7 +182,10 @@ def api(self, method: str, url: str, body=None, idempotent: bool = True): """`request` that raises ApiError on any non-2xx status; returns the parsed body.""" status, data = self.request(method, url, body, idempotent) if not 200 <= status < 300: - raise ApiError(f"{self.label}: {method} {url} -> {status} {json.dumps(data)[:300]}", status) + raise ApiError( + f"{self.label}: {method} {url} -> {status} {json.dumps(data)[:300]}", + status, + ) return data def get(self, url: str): @@ -197,7 +224,13 @@ def resolve_target() -> dict: `base_ref` (trusted branch the run was started from; drives the external branch lookup) and Raises RuntimeError when a PR run no longer matches an open PR or `pr_number` is not numeric, ApiError when a manual PR/ref does not exist.""" - t = {"pr": None, "head_repo": REPO, "head_sha": None, "head_branch": None, "base_ref": None} + t = { + "pr": None, + "head_repo": REPO, + "head_sha": None, + "head_branch": None, + "base_ref": None, + } if EVENT_NAME == "workflow_run": wr = EVENT["workflow_run"] # Trust only the event payload: nothing produced by the (untrusted) trigger run is read. @@ -210,15 +243,27 @@ def resolve_target() -> dict: return t # workflow_run.pull_requests is empty for fork PRs; find the PR by commit, then by head ref. prs = own.get(f"/repos/{REPO}/commits/{t['head_sha']}/pulls?per_page=20") - pr = next((p for p in prs if p["state"] == "open" and p["head"]["sha"] == t["head_sha"] - and p["base"]["repo"]["full_name"] == REPO), None) + pr = next( + ( + p + for p in prs + if p["state"] == "open" + and p["head"]["sha"] == t["head_sha"] + and p["base"]["repo"]["full_name"] == REPO + ), + None, + ) if not pr: owner = t["head_repo"].split("/")[0] - prs = own.get(f"/repos/{REPO}/pulls?state=open&head={q(owner + ':' + t['head_branch'])}") + prs = own.get( + f"/repos/{REPO}/pulls?state=open&head={q(owner + ':' + t['head_branch'])}" + ) pr = next((p for p in prs if p["head"]["sha"] == t["head_sha"]), None) if not pr: - raise RuntimeError(f"no open PR found for {t['head_repo']}@{t['head_sha']} ({t['head_branch']}); " - "the PR may have been updated or closed") + raise RuntimeError( + f"no open PR found for {t['head_repo']}@{t['head_sha']} ({t['head_branch']}); " + "the PR may have been updated or closed" + ) t["pr"], t["base_ref"] = pr["number"], pr["base"]["ref"] elif EVENT_NAME == "workflow_dispatch": pr_number = (env.get("INPUT_PR_NUMBER") or "").strip() @@ -226,13 +271,20 @@ def resolve_target() -> dict: if not pr_number.isdigit(): raise RuntimeError(f"pr_number must be numeric, got {pr_number!r}") pr = own.get(f"/repos/{REPO}/pulls/{pr_number}") - t.update(pr=pr["number"], head_sha=pr["head"]["sha"], head_repo=pr["head"]["repo"]["full_name"], - head_branch=pr["head"]["ref"], base_ref=pr["base"]["ref"]) + t.update( + pr=pr["number"], + head_sha=pr["head"]["sha"], + head_repo=pr["head"]["repo"]["full_name"], + head_branch=pr["head"]["ref"], + base_ref=pr["base"]["ref"], + ) else: ref = (env.get("INPUT_FPRIME_REF") or "").strip() or env["GITHUB_REF_NAME"] commit = own.get(f"/repos/{REPO}/commits/{q(ref)}") # base_ref (ext branch lookup) is the branch this run was started from, not the ref under test. - t.update(head_sha=commit["sha"], head_branch=ref, base_ref=env["GITHUB_REF_NAME"]) + t.update( + head_sha=commit["sha"], head_branch=ref, base_ref=env["GITHUB_REF_NAME"] + ) else: raise RuntimeError(f"unsupported event: {EVENT_NAME}") return t @@ -245,7 +297,9 @@ def changed_files(target: dict) -> list[str] | None: return None files = [] for page in range(1, 31): # GitHub lists at most 3000 files per PR - batch = own.get(f"/repos/{REPO}/pulls/{target['pr']}/files?per_page=100&page={page}") + batch = own.get( + f"/repos/{REPO}/pulls/{target['pr']}/files?per_page=100&page={page}" + ) files += [f["filename"] for f in batch] if len(batch) < 100: break @@ -285,12 +339,15 @@ def select_repos(manifest: dict, target: dict) -> list[dict]: def resolve_ext_ref(entry: dict, target: dict) -> str: """Return the external repo branch to dispatch: its `pr-` branch if it exists, else a branch - named like the F´ base branch (`devel`, `release/x`), else the entry's `default_ref` (`main`).""" + named like the F´ base branch (`devel`, `release/x`), else the entry's `default_ref` (`main`). + """ candidates = [target["base_ref"]] if target["pr"]: candidates.insert(0, f"pr-{target['pr']}") for candidate in candidates: - if candidate and ext.exists(f"/repos/{entry['repository']}/branches/{q(candidate)}"): + if candidate and ext.exists( + f"/repos/{entry['repository']}/branches/{q(candidate)}" + ): return candidate return entry.get("default_ref", "main") @@ -302,8 +359,10 @@ def check_name(manifest: dict, repo_id: str) -> str: def check_succeeded(manifest: dict, entry: dict, target: dict) -> bool: """True when this entry's latest Check Run on the F´ commit (from an earlier attempt) succeeded.""" - runs = own.get(f"/repos/{REPO}/commits/{target['head_sha']}/check-runs" - f"?check_name={q(check_name(manifest, entry['id']))}&filter=latest")["check_runs"] + runs = own.get( + f"/repos/{REPO}/commits/{target['head_sha']}/check-runs" + f"?check_name={q(check_name(manifest, entry['id']))}&filter=latest" + )["check_runs"] return any(r["conclusion"] == "success" for r in runs) @@ -322,44 +381,80 @@ def dispatch_one(manifest: dict, entry: dict, target: dict, state: dict) -> None remote_started_at, remote_conclusion, conclusion, error, ...). Never raises: any failure ends this repo as `remote_conclusion: dispatch_failed` so the other repos still run.""" s = state["repos"][entry["id"]] = { - "id": entry["id"], "repository": entry["repository"], "workflow": entry["workflow"], "ext_ref": None, - "failure_policy": entry.get("failure_policy", "block"), "timeout_minutes": entry.get("timeout_minutes", 60), - "dispatched_at": now().strftime("%Y-%m-%dT%H:%M:%SZ"), "check_run_id": None, - "remote_run_id": None, "remote_run_url": None, "remote_started_at": None, - "remote_conclusion": None, "conclusion": None, "error": None, + "id": entry["id"], + "repository": entry["repository"], + "workflow": entry["workflow"], + "ext_ref": None, + "failure_policy": entry.get("failure_policy", "block"), + "timeout_minutes": entry.get("timeout_minutes", 60), + "dispatched_at": now().strftime("%Y-%m-%dT%H:%M:%SZ"), + "check_run_id": None, + "remote_run_id": None, + "remote_run_url": None, + "remote_started_at": None, + "remote_conclusion": None, + "conclusion": None, + "error": None, } try: # Check Run first, so a credential or API problem on the ext side still ends in a visible # failed check rather than a required check that never appears. - check = own.post(f"/repos/{REPO}/check-runs", { - "name": check_name(manifest, entry["id"]), "head_sha": target["head_sha"], "status": "queued", - "details_url": ORCHESTRATOR_URL, - "output": {"title": f"Dispatching {entry['repository']}", - "summary": f"Dispatching {code(entry['workflow'])} on {code(entry['repository'])} with F´ " - f"{code(fprime_rev(target))}.\n\nOrchestrator: {ORCHESTRATOR_URL}"}, - }) + check = own.post( + f"/repos/{REPO}/check-runs", + { + "name": check_name(manifest, entry["id"]), + "head_sha": target["head_sha"], + "status": "queued", + "details_url": ORCHESTRATOR_URL, + "output": { + "title": f"Dispatching {entry['repository']}", + "summary": f"Dispatching {code(entry['workflow'])} on {code(entry['repository'])} with F´ " + f"{code(fprime_rev(target))}.\n\nOrchestrator: {ORCHESTRATOR_URL}", + }, + }, + ) s["check_run_id"] = check["id"] save_state(state) s["ext_ref"] = resolve_ext_ref(entry, target) - r = ext.post(f"/repos/{entry['repository']}/actions/workflows/{entry['workflow']}/dispatches", { - "ref": s["ext_ref"], - "inputs": {"fprime_repository": target["head_repo"], "fprime_ref": target["head_sha"], - "fprime_pr_number": str(target["pr"] or ""), "dispatch_id": f"{RUN_ID}-{ATTEMPT}-{entry['id']}"}, - "return_run_details": True, - }, idempotent=False) + r = ext.post( + f"/repos/{entry['repository']}/actions/workflows/{entry['workflow']}/dispatches", + { + "ref": s["ext_ref"], + "inputs": { + "fprime_repository": target["head_repo"], + "fprime_ref": target["head_sha"], + "fprime_pr_number": str(target["pr"] or ""), + "dispatch_id": f"{RUN_ID}-{ATTEMPT}-{entry['id']}", + }, + "return_run_details": True, + }, + idempotent=False, + ) if not r or not r.get("workflow_run_id"): raise RuntimeError("dispatch accepted but no run id returned") - s["remote_run_id"], s["remote_run_url"] = r["workflow_run_id"], r.get("html_url") - log(f"[{entry['id']}] dispatched {entry['repository']}/{entry['workflow']} @ {s['ext_ref']} -> {s['remote_run_url']}") + s["remote_run_id"], s["remote_run_url"] = r["workflow_run_id"], r.get( + "html_url" + ) + log( + f"[{entry['id']}] dispatched {entry['repository']}/{entry['workflow']} @ {s['ext_ref']} -> {s['remote_run_url']}" + ) try: - own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { - "status": "in_progress", "details_url": s["remote_run_url"], - "output": {"title": f"Running on {s['repository']} @ {s['ext_ref']}", - "summary": f"Remote run: {s['remote_run_url']}\n\nOrchestrator: {ORCHESTRATOR_URL}"}, - }) + own.patch( + f"/repos/{REPO}/check-runs/{s['check_run_id']}", + { + "status": "in_progress", + "details_url": s["remote_run_url"], + "output": { + "title": f"Running on {s['repository']} @ {s['ext_ref']}", + "summary": f"Remote run: {s['remote_run_url']}\n\nOrchestrator: {ORCHESTRATOR_URL}", + }, + }, + ) except ApiError as e: # cosmetic: the run exists and is polled regardless log(f"::warning::[{entry['id']}] check update: {e}") - except Exception as e: # noqa: BLE001 - one repo's failure must not stop the fan-out + except ( + Exception + ) as e: # noqa: BLE001 - one repo's failure must not stop the fan-out s["error"], s["remote_conclusion"] = f"dispatch failed: {e}", "dispatch_failed" log(f"::error::[{entry['id']}] {s['error']}") save_state(state) @@ -381,28 +476,47 @@ def poll_all(state: dict) -> None: Other poll errors are retried on the next tick.""" def active() -> list[dict]: - return [s for s in state["repos"].values() if s["remote_run_id"] and not s["remote_conclusion"]] + return [ + s + for s in state["repos"].values() + if s["remote_run_id"] and not s["remote_conclusion"] + ] while active(): time.sleep(POLL_S) for s in active(): try: - run = ext.get(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}") + run = ext.get( + f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}" + ) if run.get("run_started_at") and not s["remote_started_at"]: s["remote_started_at"] = run["run_started_at"] timeout = timedelta(minutes=s["timeout_minutes"]) if run["status"] == "completed": s["remote_conclusion"] = run["conclusion"] log(f"[{s['id']}] completed: {run['conclusion']}") - elif s["remote_started_at"] and now() - parse_ts(s["remote_started_at"]) > timeout: + elif ( + s["remote_started_at"] + and now() - parse_ts(s["remote_started_at"]) > timeout + ): # Counted from when the remote run started: queue time is not the ext repo's fault. - log(f"::warning::[{s['id']}] timed out after {s['timeout_minutes']} min of running; cancelling remote run") + log( + f"::warning::[{s['id']}] timed out after {s['timeout_minutes']} min of running; cancelling remote run" + ) cancel_run(s) s["remote_conclusion"] = "timed_out" - elif not s["remote_started_at"] and now() - parse_ts(s["dispatched_at"]) > 2 * timeout: - log(f"::warning::[{s['id']}] still queued after {2 * s['timeout_minutes']} min; giving up") + elif ( + not s["remote_started_at"] + and now() - parse_ts(s["dispatched_at"]) > 2 * timeout + ): + log( + f"::warning::[{s['id']}] still queued after {2 * s['timeout_minutes']} min; giving up" + ) cancel_run(s) - s["remote_conclusion"], s["error"] = "queue_timeout", "remote run never started (no runner available?)" + s["remote_conclusion"], s["error"] = ( + "queue_timeout", + "remote run never started (no runner available?)", + ) except ApiError as e: # A single bad poll must not take the whole fan-out down; give up on this repo only # when the credential is gone or the run vanished. @@ -418,7 +532,9 @@ def failure_details(s: dict) -> str: if not s["remote_run_id"]: return "" try: - jobs = ext.get(f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}/jobs?per_page=100")["jobs"] + jobs = ext.get( + f"/repos/{s['repository']}/actions/runs/{s['remote_run_id']}/jobs?per_page=100" + )["jobs"] except ApiError as e: return f"_(job details unavailable: {code(e)})_" lines = [] @@ -426,8 +542,11 @@ def failure_details(s: dict) -> str: if j.get("conclusion") in (None, "success", "skipped"): continue lines.append(f"- [{code(j['name'])}]({j['html_url']}): **{j['conclusion']}**") - lines += [f" - step {code(st['name'])}: {st['conclusion']}" - for st in j.get("steps", []) if st.get("conclusion") not in (None, "success", "skipped")] + lines += [ + f" - step {code(st['name'])}: {st['conclusion']}" + for st in j.get("steps", []) + if st.get("conclusion") not in (None, "success", "skipped") + ] return "\n".join(lines) @@ -449,25 +568,40 @@ def check_conclusion(s: dict) -> str: def finalize_one(target: dict, s: dict) -> None: """Close one repo's Check Run with its conclusion, summary and failed-job details. - Sets s["conclusion"] only after the update succeeded, so `finalize` retries it otherwise.""" + Sets s["conclusion"] only after the update succeeded, so `finalize` retries it otherwise. + """ conclusion = check_conclusion(s) details = "" if s["remote_conclusion"] == "success" else failure_details(s) - lines = [f"{code(s['repository'])} @ {code(s['ext_ref'] or '-')} with F´ {code(fprime_rev(target))}: " - f"**{s['remote_conclusion']}**", ""] + lines = [ + f"{code(s['repository'])} @ {code(s['ext_ref'] or '-')} with F´ {code(fprime_rev(target))}: " + f"**{s['remote_conclusion']}**", + "", + ] if s["remote_run_url"]: lines.append(f"Remote run: {s['remote_run_url']}") if s["error"]: lines.append(f"Error: {code(s['error'])}") if s["failure_policy"] == "notify" and s["remote_conclusion"] != "success": - lines += ["", f"This repo is {code('failure_policy: notify')}: the failure does not block the PR."] + lines += [ + "", + f"This repo is {code('failure_policy: notify')}: the failure does not block the PR.", + ] lines += ["", f"Orchestrator run: {ORCHESTRATOR_URL}"] - output = {"title": f"{s['remote_conclusion']} ({s['repository']} @ {s['ext_ref'] or '-'})", "summary": "\n".join(lines)} + output = { + "title": f"{s['remote_conclusion']} ({s['repository']} @ {s['ext_ref'] or '-'})", + "summary": "\n".join(lines), + } if details: output["text"] = details - own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { - "status": "completed", "conclusion": conclusion, "details_url": s["remote_run_url"] or ORCHESTRATOR_URL, - "output": output, - }) + own.patch( + f"/repos/{REPO}/check-runs/{s['check_run_id']}", + { + "status": "completed", + "conclusion": conclusion, + "details_url": s["remote_run_url"] or ORCHESTRATOR_URL, + "output": output, + }, + ) s["conclusion"] = conclusion @@ -475,15 +609,27 @@ def skip_all(manifest: dict, target: dict, repos: list[dict], state: dict) -> No """Create every repo's Check Run directly as completed/skipped (only `paths_ignore` files changed).""" ignored = ", ".join(manifest.get("paths_ignore", [])) for entry in repos: - check = own.post(f"/repos/{REPO}/check-runs", { - "name": check_name(manifest, entry["id"]), "head_sha": target["head_sha"], - "status": "completed", "conclusion": "skipped", "details_url": ORCHESTRATOR_URL, - "output": {"title": "Not dispatched: no relevant files changed", - "summary": f"Only paths matching {code(ignored)} changed.\n\nOrchestrator: {ORCHESTRATOR_URL}"}, - }) + check = own.post( + f"/repos/{REPO}/check-runs", + { + "name": check_name(manifest, entry["id"]), + "head_sha": target["head_sha"], + "status": "completed", + "conclusion": "skipped", + "details_url": ORCHESTRATOR_URL, + "output": { + "title": "Not dispatched: no relevant files changed", + "summary": f"Only paths matching {code(ignored)} changed.\n\nOrchestrator: {ORCHESTRATOR_URL}", + }, + }, + ) state["repos"][entry["id"]] = { - "id": entry["id"], "repository": entry["repository"], "failure_policy": entry.get("failure_policy", "block"), - "check_run_id": check["id"], "remote_conclusion": "skipped", "conclusion": "skipped", + "id": entry["id"], + "repository": entry["repository"], + "failure_policy": entry.get("failure_policy", "block"), + "check_run_id": check["id"], + "remote_conclusion": "skipped", + "conclusion": "skipped", } @@ -497,19 +643,40 @@ def write_summary(target: dict, state: dict) -> None: link = code(s["error"]) else: link = "" - rows.append(f"| {code(s['id'])} | {s['repository']} | {code(s.get('ext_ref') or '-')} | {s['failure_policy']} " - f"| {s['remote_conclusion']} | {s['conclusion']} | {link} |") - where = f" (PR #{target['pr']})" if target["pr"] else f" ({code(target['head_branch'])})" - md = "\n".join([f"### External CI for {code(fprime_rev(target))}{where} — attempt {ATTEMPT}", "", - "| id | repository | ext ref | policy | remote | check | link |", - "|---|---|---|---|---|---|---|", *rows, ""]) + rows.append( + f"| {code(s['id'])} | {s['repository']} | {code(s.get('ext_ref') or '-')} | {s['failure_policy']} " + f"| {s['remote_conclusion']} | {s['conclusion']} | {link} |" + ) + where = ( + f" (PR #{target['pr']})" + if target["pr"] + else f" ({code(target['head_branch'])})" + ) + md = "\n".join( + [ + f"### External CI for {code(fprime_rev(target))}{where} — attempt {ATTEMPT}", + "", + "| id | repository | ext ref | policy | remote | check | link |", + "|---|---|---|---|---|---|---|", + *rows, + "", + ] + ) if env.get("GITHUB_STEP_SUMMARY"): with open(env["GITHUB_STEP_SUMMARY"], "a") as f: f.write(md) log(md) with open(RESULTS_FILE, "w") as f: - json.dump({"target": target, "attempt": ATTEMPT, "orchestrator_url": ORCHESTRATOR_URL, "repos": state["repos"]}, - f, indent=2) + json.dump( + { + "target": target, + "attempt": ATTEMPT, + "orchestrator_url": ORCHESTRATOR_URL, + "repos": state["repos"], + }, + f, + indent=2, + ) # ---------------------------------------------------------------- modes @@ -524,16 +691,22 @@ def run() -> None: # Fork preflight: a fork of F´ that copies this workflow must not dispatch into the ext repos. upstream = manifest.get("upstream_repository") if upstream and upstream != REPO and EVENT_NAME != "workflow_dispatch": - log(f"::notice::{REPO} is not {upstream}; external CI is only dispatched from upstream. " - "Use workflow_dispatch to run it here deliberately.") + log( + f"::notice::{REPO} is not {upstream}; external CI is only dispatched from upstream. " + "Use workflow_dispatch to run it here deliberately." + ) return repos = select_repos(manifest, target) if ATTEMPT > 1: # re-run: only the repos whose check is not green yet repos = [e for e in repos if not check_succeeded(manifest, e, target)] - log(f"attempt {ATTEMPT}: re-dispatching {[e['id'] for e in repos] or 'nothing'}") + log( + f"attempt {ATTEMPT}: re-dispatching {[e['id'] for e in repos] or 'nothing'}" + ) state = {"target": target, "attempt": ATTEMPT, "repos": {}} save_state(state) - if EVENT_NAME != "workflow_dispatch" and not relevant_change(changed_files(target), manifest): + if EVENT_NAME != "workflow_dispatch" and not relevant_change( + changed_files(target), manifest + ): log("no relevant files changed; marking all repos as skipped") skip_all(manifest, target, repos, state) write_summary(target, state) @@ -544,15 +717,23 @@ def run() -> None: for s in state["repos"].values(): try: finalize_one(target, s) - except Exception as e: # noqa: BLE001 - the check stays open and finalize() closes it + except ( + Exception + ) as e: # noqa: BLE001 - the check stays open and finalize() closes it log(f"::error::[{s['id']}] finalize: {e}") save_state(state) write_summary(target, state) # conclusion is None when finalize_one failed: judge on what it would have been. `notify` only # softens a remote build failure; an orchestrator-side fault always fails this job. - blocking = [s["id"] for s in state["repos"].values() - if (s["conclusion"] or check_conclusion(s)) != "success" - and (s["failure_policy"] != "notify" or s["remote_conclusion"] in ORCHESTRATOR_FAULTS)] + blocking = [ + s["id"] + for s in state["repos"].values() + if (s["conclusion"] or check_conclusion(s)) != "success" + and ( + s["failure_policy"] != "notify" + or s["remote_conclusion"] in ORCHESTRATOR_FAULTS + ) + ] if blocking: log("::error::blocking external repos failed: " + ", ".join(blocking)) sys.exit(1) @@ -560,7 +741,8 @@ def run() -> None: def finalize() -> None: """The `finalize` mode, run under `if: always()`: whatever the run step left open (crash, cancellation, timeout) is - closed here so no Check Run stays in_progress and no remote run keeps burning runners.""" + closed here so no Check Run stays in_progress and no remote run keeps burning runners. + """ if not os.path.exists(STATE_FILE): return log("no state file; nothing to finalize") with open(STATE_FILE) as f: @@ -575,25 +757,43 @@ def finalize() -> None: cancel_run(s) remote += " (cancelled)" try: - if own.get(f"/repos/{REPO}/check-runs/{s['check_run_id']}")["status"] == "completed": + if ( + own.get(f"/repos/{REPO}/check-runs/{s['check_run_id']}")["status"] + == "completed" + ): continue except ApiError: pass if cancelled: conclusion = "neutral" if s["failure_policy"] == "notify" else "cancelled" - title, why = "Cancelled by orchestrator", "The orchestrator run was cancelled" - elif s.get("remote_conclusion"): # reported, but the check update itself failed in run - conclusion, title = check_conclusion(s), f"{s['remote_conclusion']} ({s['repository']} @ {s.get('ext_ref') or '-'})" + title, why = ( + "Cancelled by orchestrator", + "The orchestrator run was cancelled", + ) + elif s.get( + "remote_conclusion" + ): # reported, but the check update itself failed in run + conclusion, title = ( + check_conclusion(s), + f"{s['remote_conclusion']} ({s['repository']} @ {s.get('ext_ref') or '-'})", + ) why = "Closed by the finalize step after the check update failed" else: conclusion = "neutral" if s["failure_policy"] == "notify" else "failure" title = "Orchestrator error" why = f"The orchestrator run ended before this repo was reported ({code(s.get('error') or 'orchestrator error')})" try: - own.patch(f"/repos/{REPO}/check-runs/{s['check_run_id']}", { - "status": "completed", "conclusion": conclusion, - "output": {"title": title, "summary": f"{why}; remote run {remote}.\n\n{ORCHESTRATOR_URL}"}, - }) + own.patch( + f"/repos/{REPO}/check-runs/{s['check_run_id']}", + { + "status": "completed", + "conclusion": conclusion, + "output": { + "title": title, + "summary": f"{why}; remote run {remote}.\n\n{ORCHESTRATOR_URL}", + }, + }, + ) except ApiError as e: log(f"::warning::[{s['id']}] check update: {e}") From ca346a39c0c167d93d0479428426f0298b0f8968 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 18:23:46 +0000 Subject: [PATCH 15/22] ci: ext-ci orchestrator formatting compatible with black 21.6b0 (CI pin) --- .github/scripts/ext-ci-orchestrator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/scripts/ext-ci-orchestrator.py index 59ca6792752..70cd7e7f30f 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/scripts/ext-ci-orchestrator.py @@ -150,7 +150,7 @@ def request( except (urllib.error.URLError, OSError) as e: if not idempotent or i == 4: raise ApiError(f"{self.label}: {method} {url} -> network error {e}") - time.sleep(2 * 2**i) + time.sleep(2 << i) continue limited = status == 429 or ( status == 403 @@ -160,7 +160,7 @@ def request( ) ) if (limited or (status >= 500 and idempotent)) and i < 4: - wait = 2 * 2**i + wait = 2 << i if hdrs.get("retry-after"): wait = int(hdrs["retry-after"]) elif limited and hdrs.get("x-ratelimit-reset"): From c5982f7a671fd866fce37f1c9a821b25d3d84435 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 18:31:02 +0000 Subject: [PATCH 16/22] ci: ext-ci files under .github/actions/ext-ci with a README; ubuntu-latest; 60-min job / 30-min remote timeouts --- .github/actions/ext-ci/README.md | 40 +++++++++++++++++++ .../ext-ci/manifest.json} | 2 +- .../ext-ci/orchestrator.py} | 22 +++------- .github/workflows/ext-ci-trigger.yml | 2 +- .github/workflows/ext-ci.yml | 10 ++--- 5 files changed, 52 insertions(+), 24 deletions(-) create mode 100644 .github/actions/ext-ci/README.md rename .github/{ext-ci-manifest.json => actions/ext-ci/manifest.json} (93%) rename .github/{scripts/ext-ci-orchestrator.py => actions/ext-ci/orchestrator.py} (96%) diff --git a/.github/actions/ext-ci/README.md b/.github/actions/ext-ci/README.md new file mode 100644 index 00000000000..3e550c69b8d --- /dev/null +++ b/.github/actions/ext-ci/README.md @@ -0,0 +1,40 @@ +# External CI (`ext-ci`) + +F´ does not build the `fprime-community` projects itself any more: `.github/workflows/ext-ci.yml` +dispatches each project's own `ci.yml` with the F´ revision under test and reports the result as a +Check Run named `ext-ci / ` on the F´ commit. `ext-ci-trigger.yml` is the trivial `pull_request` +workflow whose completion (after GitHub's fork-approval gate) starts the orchestrator; the orchestrator +also runs for merge groups and on manual dispatch (`repos`, `pr_number` or `fprime_ref` inputs). + +## manifest.json + +| key | meaning | +|---|---| +| `upstream_repository` | only this repository dispatches automatically; forks run only on manual dispatch | +| `check_prefix` | Check Run name is ` / ` | +| `paths_ignore` | globs; a PR touching only these gets `skipped` checks and no dispatch | +| `repos[].id` | Check Run suffix and value for the manual `repos` input | +| `repos[].repository` | external repository (`owner/name`) | +| `repos[].workflow` | workflow file in that repository; must accept the inputs below | +| `repos[].default_ref` | branch to dispatch when neither `pr-` nor a branch named like the F´ base branch exists there | +| `repos[].timeout_minutes` | remote running time allowed (2x while queued); the remote run is cancelled on timeout | +| `repos[].failure_policy` | `block`: a failure is a red check and fails the orchestrator job; `notify`: neutral (non-blocking) check | +| `repos[].on_pr` | `false`: never dispatched for a PR head (merge queue and manual `fprime_ref` runs only) | + +Orchestrator-side faults (dispatch error, run never started, polling lost the run) fail the job +whatever the policy. A re-run of the orchestrator re-dispatches only the repos whose check is not green. + +## Contract for an external `ci.yml` + +- `on: workflow_dispatch` with inputs `fprime_repository`, `fprime_ref` (exact SHA to use as the F´ + dependency), `fprime_pr_number`, `dispatch_id`. +- `permissions: contents: read` (or `{}`), no `${{ secrets.* }}`, no Actions cache writes on dispatched + runs: the dispatched run executes untrusted F´ PR code in the external repository's own context. +- Repositories with self-hosted or hardware runners: `on_pr: false` in the manifest, ephemeral runners, + SHA-pinned actions, and a job-level guard on `inputs.fprime_repository` in the workflow. + +## Credentials + +`GITHUB_TOKEN` (`checks: write`, this repository only) reports results; the repository secret +`EXT_CI_DISPATCH_TOKEN` (`actions: write` on the external repositories, no permission here) starts the +remote runs and is exposed only to the two orchestrator steps. diff --git a/.github/ext-ci-manifest.json b/.github/actions/ext-ci/manifest.json similarity index 93% rename from .github/ext-ci-manifest.json rename to .github/actions/ext-ci/manifest.json index cbb7d869810..0971b8042e0 100644 --- a/.github/ext-ci-manifest.json +++ b/.github/actions/ext-ci/manifest.json @@ -14,7 +14,7 @@ "repository": "fprime-community/fprime-workshop-led-blinker", "workflow": "ci.yml", "default_ref": "devel", - "timeout_minutes": 60, + "timeout_minutes": 30, "failure_policy": "notify", "on_pr": true } diff --git a/.github/scripts/ext-ci-orchestrator.py b/.github/actions/ext-ci/orchestrator.py similarity index 96% rename from .github/scripts/ext-ci-orchestrator.py rename to .github/actions/ext-ci/orchestrator.py index 70cd7e7f30f..f8874826d6f 100644 --- a/.github/scripts/ext-ci-orchestrator.py +++ b/.github/actions/ext-ci/orchestrator.py @@ -1,26 +1,14 @@ #!/usr/bin/env python3 """External-CI orchestrator (standard library only, Python >= 3.10). - python3 ext-ci-orchestrator.py run dispatch, poll, report - python3 ext-ci-orchestrator.py finalize close anything still open (runs under if: always()) + python3 orchestrator.py run dispatch, poll, report + python3 orchestrator.py finalize close anything still open (runs under if: always()) -For every repo in .github/ext-ci-manifest.json: create a Check Run on the F´ commit under test, +For every repo in manifest.json (same directory): create a Check Run on the F´ commit under test, dispatch the repo's own ci.yml with that commit, poll the remote run, close the Check Run with its result. A re-run of the orchestrator re-dispatches only the repos whose check is not green. -Manifest (.github/ext-ci-manifest.json): - upstream_repository only this repo dispatches automatically; forks only on workflow_dispatch - check_prefix Check Run name is " / " - paths_ignore globs; a PR touching only these gets "skipped" checks and no dispatch - repos[]: id, repository (owner/name), workflow (file name, must accept the inputs below), - default_ref (branch to dispatch when no pr- / base-named branch exists), - timeout_minutes (of remote running time; 2x while queued), failure_policy - ("block": failure is a red check and fails this job; "notify": neutral check), - on_pr (false: never dispatched for a PR, only merge queue / manual) - -Contract for the external ci.yml: `on: workflow_dispatch` with inputs fprime_repository, -fprime_ref (exact SHA to overlay as the F´ dependency), fprime_pr_number, dispatch_id; no secrets, -read-only permissions, no cache writes on dispatched runs. +Manifest keys and the contract an external ci.yml must satisfy: see README.md next to this file. Two credentials, never mixed: GITHUB_TOKEN this repo only (checks: write) - reports results @@ -45,7 +33,7 @@ SERVER = env.get("GITHUB_SERVER_URL", "https://github.com") STATE_FILE = "ext-ci-state.json" RESULTS_FILE = "ext-ci-results.json" -MANIFEST_PATH = ".github/ext-ci-manifest.json" +MANIFEST_PATH = ".github/actions/ext-ci/manifest.json" POLL_S = int(env.get("POLL_INTERVAL_SECONDS") or 60) ORCHESTRATOR_FAULTS = ( "dispatch_failed", diff --git a/.github/workflows/ext-ci-trigger.yml b/.github/workflows/ext-ci-trigger.yml index 4c949fb507e..6fa7dacce5e 100644 --- a/.github/workflows/ext-ci-trigger.yml +++ b/.github/workflows/ext-ci-trigger.yml @@ -17,6 +17,6 @@ permissions: {} jobs: approved: name: "Gate passed" - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - run: echo "${GITHUB_EVENT_NAME} workflow ran for ${GITHUB_SHA}; ext-ci may proceed" diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index ebd8ec15e56..c0c3916d280 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -28,7 +28,7 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'workflow_run' }} permissions: - contents: read # manifest + contents: read # manifest + script checks: write # per-repo Check Runs jobs: @@ -36,8 +36,8 @@ jobs: name: "External CI" # Only chain from a trigger that actually ran (not skipped/cancelled) for a PR or merge group. if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) - runs-on: ubuntu-22.04 - timeout-minutes: 180 + runs-on: ubuntu-latest + timeout-minutes: 60 env: GITHUB_TOKEN: ${{ github.token }} steps: @@ -49,13 +49,13 @@ jobs: INPUT_PR_NUMBER: ${{ inputs.pr_number }} INPUT_FPRIME_REF: ${{ inputs.fprime_ref }} POLL_INTERVAL_SECONDS: 60 - run: python3 .github/scripts/ext-ci-orchestrator.py run + run: python3 .github/actions/ext-ci/orchestrator.py run - name: "Finalize (close open checks, cancel orphaned remote runs)" if: always() env: DISPATCH_TOKEN: ${{ secrets.EXT_CI_DISPATCH_TOKEN }} # external repos only; not exposed to other steps JOB_STATUS: ${{ job.status }} - run: python3 .github/scripts/ext-ci-orchestrator.py finalize + run: python3 .github/actions/ext-ci/orchestrator.py finalize - uses: actions/upload-artifact@v4 if: always() with: From a363fa349501dcc669447a7bbbb6808ffa69bd40 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 18:33:39 +0000 Subject: [PATCH 17/22] ci: ext-ci queue allowance = timeout_minutes (20 min), so queue + run stay inside the 60-min job --- .github/actions/ext-ci/README.md | 2 +- .github/actions/ext-ci/manifest.json | 2 +- .github/actions/ext-ci/orchestrator.py | 6 +++--- .github/workflows/ext-ci.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/ext-ci/README.md b/.github/actions/ext-ci/README.md index 3e550c69b8d..32a45ff6b61 100644 --- a/.github/actions/ext-ci/README.md +++ b/.github/actions/ext-ci/README.md @@ -17,7 +17,7 @@ also runs for merge groups and on manual dispatch (`repos`, `pr_number` or `fpri | `repos[].repository` | external repository (`owner/name`) | | `repos[].workflow` | workflow file in that repository; must accept the inputs below | | `repos[].default_ref` | branch to dispatch when neither `pr-` nor a branch named like the F´ base branch exists there | -| `repos[].timeout_minutes` | remote running time allowed (2x while queued); the remote run is cancelled on timeout | +| `repos[].timeout_minutes` | remote running time allowed, and separately the time it may wait in queue; the remote run is cancelled on timeout. Keep `2 * timeout_minutes` under the job's `timeout-minutes` in ext-ci.yml | | `repos[].failure_policy` | `block`: a failure is a red check and fails the orchestrator job; `notify`: neutral (non-blocking) check | | `repos[].on_pr` | `false`: never dispatched for a PR head (merge queue and manual `fprime_ref` runs only) | diff --git a/.github/actions/ext-ci/manifest.json b/.github/actions/ext-ci/manifest.json index 0971b8042e0..c2198286347 100644 --- a/.github/actions/ext-ci/manifest.json +++ b/.github/actions/ext-ci/manifest.json @@ -14,7 +14,7 @@ "repository": "fprime-community/fprime-workshop-led-blinker", "workflow": "ci.yml", "default_ref": "devel", - "timeout_minutes": 30, + "timeout_minutes": 20, "failure_policy": "notify", "on_pr": true } diff --git a/.github/actions/ext-ci/orchestrator.py b/.github/actions/ext-ci/orchestrator.py index f8874826d6f..2ae6d4995c1 100644 --- a/.github/actions/ext-ci/orchestrator.py +++ b/.github/actions/ext-ci/orchestrator.py @@ -460,7 +460,7 @@ def poll_all(state: dict) -> None: """Poll every dispatched run every POLL_S seconds until each has a `remote_conclusion`. Sets it to the run's conclusion, `timed_out` (running longer than timeout_minutes, remote run - cancelled), `queue_timeout` (not started after 2x timeout) or `error` (401/404 while polling). + cancelled), `queue_timeout` (not started after another timeout_minutes) or `error` (401/404 while polling). Other poll errors are retried on the next tick.""" def active() -> list[dict]: @@ -495,10 +495,10 @@ def active() -> list[dict]: s["remote_conclusion"] = "timed_out" elif ( not s["remote_started_at"] - and now() - parse_ts(s["dispatched_at"]) > 2 * timeout + and now() - parse_ts(s["dispatched_at"]) > timeout ): log( - f"::warning::[{s['id']}] still queued after {2 * s['timeout_minutes']} min; giving up" + f"::warning::[{s['id']}] still queued after {s['timeout_minutes']} min; giving up" ) cancel_run(s) s["remote_conclusion"], s["error"] = ( diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index c0c3916d280..97c6bfd0bf9 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -37,7 +37,7 @@ jobs: # Only chain from a trigger that actually ran (not skipped/cancelled) for a PR or merge group. if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 60 # > 2 * max manifest timeout_minutes (queue + run) + overhead env: GITHUB_TOKEN: ${{ github.token }} steps: From ba785215dedf9c9333f97a7f1384893788e8d4ee Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 18:38:48 +0000 Subject: [PATCH 18/22] ci: ext-ci job display name "Orchestrate" --- .github/workflows/ext-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index 97c6bfd0bf9..d887d3ab46e 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -33,7 +33,7 @@ permissions: jobs: orchestrate: - name: "External CI" + name: "Orchestrate" # Only chain from a trigger that actually ran (not skipped/cancelled) for a PR or merge group. if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) runs-on: ubuntu-latest From a7c70204085cbabeb55b0b8b216f3dd9a8e23869 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 18:43:14 +0000 Subject: [PATCH 19/22] ci: ext-ci display names: "External CI / Approval Gate", "External CI / Orchestrate", "External CI / " --- .github/actions/ext-ci/README.md | 4 ++-- .github/actions/ext-ci/manifest.json | 2 +- .github/workflows/ext-ci-trigger.yml | 4 ++-- .github/workflows/ext-ci.yml | 12 +++++++----- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/actions/ext-ci/README.md b/.github/actions/ext-ci/README.md index 32a45ff6b61..446e7447ce6 100644 --- a/.github/actions/ext-ci/README.md +++ b/.github/actions/ext-ci/README.md @@ -1,8 +1,8 @@ -# External CI (`ext-ci`) +# External CI F´ does not build the `fprime-community` projects itself any more: `.github/workflows/ext-ci.yml` dispatches each project's own `ci.yml` with the F´ revision under test and reports the result as a -Check Run named `ext-ci / ` on the F´ commit. `ext-ci-trigger.yml` is the trivial `pull_request` +Check Run named `External CI / ` on the F´ commit. `ext-ci-trigger.yml` (`External CI / Approval Gate`) is the trivial `pull_request` workflow whose completion (after GitHub's fork-approval gate) starts the orchestrator; the orchestrator also runs for merge groups and on manual dispatch (`repos`, `pr_number` or `fprime_ref` inputs). diff --git a/.github/actions/ext-ci/manifest.json b/.github/actions/ext-ci/manifest.json index c2198286347..9d5395332cb 100644 --- a/.github/actions/ext-ci/manifest.json +++ b/.github/actions/ext-ci/manifest.json @@ -1,6 +1,6 @@ { "upstream_repository": "nasa/fprime", - "check_prefix": "ext-ci", + "check_prefix": "External CI", "paths_ignore": [ "docs/**", "**.md", diff --git a/.github/workflows/ext-ci-trigger.yml b/.github/workflows/ext-ci-trigger.yml index 6fa7dacce5e..bf6d5a56857 100644 --- a/.github/workflows/ext-ci-trigger.yml +++ b/.github/workflows/ext-ci-trigger.yml @@ -1,4 +1,4 @@ -name: ext-ci-trigger +name: External CI # Deliberately trivial. This workflow runs under the repository's normal `pull_request` # rules, and its completion is what starts the privileged orchestrator in ext-ci.yml via @@ -16,7 +16,7 @@ permissions: {} jobs: approved: - name: "Gate passed" + name: "Approval Gate" runs-on: ubuntu-latest steps: - run: echo "${GITHUB_EVENT_NAME} workflow ran for ${GITHUB_SHA}; ext-ci may proceed" diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index d887d3ab46e..be618d3012c 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -1,10 +1,11 @@ -name: ext-ci +name: External CI on: - # PR path: fires after ext-ci-trigger completed, i.e. after GitHub's own approval gate. + # PR path: fires after the ext-ci-trigger.yml workflow (also named "External CI", so the checks read + # "External CI / ") completed, i.e. after GitHub's own approval gate. # The workflow file and the checked-out scripts come from the default branch, never from the PR. workflow_run: - workflows: [ ext-ci-trigger ] + workflows: [ "External CI" ] types: [ completed ] workflow_dispatch: inputs: @@ -34,8 +35,9 @@ permissions: jobs: orchestrate: name: "Orchestrate" - # Only chain from a trigger that actually ran (not skipped/cancelled) for a PR or merge group. - if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) + # Only chain from the trigger workflow (same name as this one, hence the path check) when it + # actually ran (not skipped/cancelled) for a PR or merge group. + if: github.event_name != 'workflow_run' || (github.event.workflow_run.path == '.github/workflows/ext-ci-trigger.yml' && github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) runs-on: ubuntu-latest timeout-minutes: 60 # > 2 * max manifest timeout_minutes (queue + run) + overhead env: From cf2f06ecc0423033ebc5da31f9999be8c792157a Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 18:44:49 +0000 Subject: [PATCH 20/22] ci: trigger workflow named "External CI Trigger" so workflow_run cannot match the orchestrator itself --- .github/actions/ext-ci/README.md | 2 +- .github/workflows/ext-ci-trigger.yml | 2 +- .github/workflows/ext-ci.yml | 11 +++++------ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/actions/ext-ci/README.md b/.github/actions/ext-ci/README.md index 446e7447ce6..3b35d658840 100644 --- a/.github/actions/ext-ci/README.md +++ b/.github/actions/ext-ci/README.md @@ -2,7 +2,7 @@ F´ does not build the `fprime-community` projects itself any more: `.github/workflows/ext-ci.yml` dispatches each project's own `ci.yml` with the F´ revision under test and reports the result as a -Check Run named `External CI / ` on the F´ commit. `ext-ci-trigger.yml` (`External CI / Approval Gate`) is the trivial `pull_request` +Check Run named `External CI / ` on the F´ commit. `ext-ci-trigger.yml` (`External CI Trigger / Approval Gate`) is the trivial `pull_request` workflow whose completion (after GitHub's fork-approval gate) starts the orchestrator; the orchestrator also runs for merge groups and on manual dispatch (`repos`, `pr_number` or `fprime_ref` inputs). diff --git a/.github/workflows/ext-ci-trigger.yml b/.github/workflows/ext-ci-trigger.yml index bf6d5a56857..ad5cf3ad116 100644 --- a/.github/workflows/ext-ci-trigger.yml +++ b/.github/workflows/ext-ci-trigger.yml @@ -1,4 +1,4 @@ -name: External CI +name: External CI Trigger # Deliberately trivial. This workflow runs under the repository's normal `pull_request` # rules, and its completion is what starts the privileged orchestrator in ext-ci.yml via diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index be618d3012c..363622a9485 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -1,11 +1,11 @@ name: External CI on: - # PR path: fires after the ext-ci-trigger.yml workflow (also named "External CI", so the checks read - # "External CI / ") completed, i.e. after GitHub's own approval gate. + # PR path: fires after the "External CI Trigger" workflow (ext-ci-trigger.yml) completed, i.e. after + # GitHub's own approval gate. # The workflow file and the checked-out scripts come from the default branch, never from the PR. workflow_run: - workflows: [ "External CI" ] + workflows: [ "External CI Trigger" ] types: [ completed ] workflow_dispatch: inputs: @@ -35,9 +35,8 @@ permissions: jobs: orchestrate: name: "Orchestrate" - # Only chain from the trigger workflow (same name as this one, hence the path check) when it - # actually ran (not skipped/cancelled) for a PR or merge group. - if: github.event_name != 'workflow_run' || (github.event.workflow_run.path == '.github/workflows/ext-ci-trigger.yml' && github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) + # Only chain from a trigger that actually ran (not skipped/cancelled) for a PR or merge group. + if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && contains(fromJSON('["pull_request","merge_group"]'), github.event.workflow_run.event)) runs-on: ubuntu-latest timeout-minutes: 60 # > 2 * max manifest timeout_minutes (queue + run) + overhead env: From 5dbbb94d1d0f9a172dbc9163b0bf9caaf0e2db56 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 19:18:01 +0000 Subject: [PATCH 21/22] ci: ext-ci uses GitHub App installation tokens (dispatch + Check Runs) instead of a PAT Check Runs created with GITHUB_TOKEN are filed into an arbitrary existing Actions check suite on the PR head, so the PR shows them as ' / ext-ci / led-blinker'. Creating them with the External CI App's own installation token gives them their own suite. The same App's fprime-community installation replaces EXT_CI_DISPATCH_TOKEN; secrets are EXT_CI_APP_ID and EXT_CI_APP_PRIVATE_KEY. Falls back to GITHUB_TOKEN for the checks while the App is not installed on this repo. --- .github/actions/ext-ci/README.md | 14 +++++++++++--- .github/actions/ext-ci/orchestrator.py | 17 ++++++++++++----- .github/workflows/ext-ci.yml | 24 ++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/.github/actions/ext-ci/README.md b/.github/actions/ext-ci/README.md index 3b35d658840..d472c8db42d 100644 --- a/.github/actions/ext-ci/README.md +++ b/.github/actions/ext-ci/README.md @@ -35,6 +35,14 @@ whatever the policy. A re-run of the orchestrator re-dispatches only the repos w ## Credentials -`GITHUB_TOKEN` (`checks: write`, this repository only) reports results; the repository secret -`EXT_CI_DISPATCH_TOKEN` (`actions: write` on the external repositories, no permission here) starts the -remote runs and is exposed only to the two orchestrator steps. +One GitHub App ("External CI", registered under fprime-community; permissions Actions read/write, +Checks read/write, Contents read, Pull requests read; no webhook) installed twice: on the external +repositories listed in the manifest and on this repository. The workflow mints two 1-hour installation +tokens from the repository secrets `EXT_CI_APP_ID` and `EXT_CI_APP_PRIVATE_KEY`, exposed only to the two +orchestrator steps: + +- `DISPATCH_TOKEN` (fprime-community installation) starts and cancels the remote runs. Adding an + external repository = adding it to that installation and to the manifest. +- `CHECKS_TOKEN` (this repository's installation) creates the Check Runs, so they appear under their own + "External CI" group in the PR checks list. If the App is not installed here the orchestrator falls + back to `GITHUB_TOKEN` (`checks: write`) and GitHub files the checks under another workflow's group. diff --git a/.github/actions/ext-ci/orchestrator.py b/.github/actions/ext-ci/orchestrator.py index 2ae6d4995c1..3d8bf300307 100644 --- a/.github/actions/ext-ci/orchestrator.py +++ b/.github/actions/ext-ci/orchestrator.py @@ -10,10 +10,13 @@ Manifest keys and the contract an external ci.yml must satisfy: see README.md next to this file. -Two credentials, never mixed: - GITHUB_TOKEN this repo only (checks: write) - reports results - DISPATCH_TOKEN external repos only (actions: write; contents: read too if an external repo - is private, for the branch lookup) - starts remote runs +Two credentials, never mixed (both are installation tokens of the External CI GitHub App, minted by +the workflow): + CHECKS_TOKEN this repo only (checks: write) - reports results; the App's own check suite makes the + checks show as "External CI / " instead of inside another workflow's group. + Falls back to GITHUB_TOKEN when the App is not installed on this repository. + DISPATCH_TOKEN external repos only (actions: write; contents: read for the branch lookup) - starts + remote runs """ import json @@ -198,7 +201,11 @@ def exists(self, url: str) -> bool: raise ApiError(f"{self.label}: GET {url} -> {status}", status) -own = Client(env.get("GITHUB_TOKEN"), "GITHUB_TOKEN") +own = ( + Client(env["CHECKS_TOKEN"], "CHECKS_TOKEN") + if env.get("CHECKS_TOKEN") + else Client(env.get("GITHUB_TOKEN"), "GITHUB_TOKEN") +) ext = Client(env.get("DISPATCH_TOKEN"), "DISPATCH_TOKEN") diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index 363622a9485..a801b5bdfd1 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -43,9 +43,28 @@ jobs: GITHUB_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@v4 + # One GitHub App, two installation tokens (1 h, revoked after the job): the external one can only + # start/cancel runs in the external repositories; the checks one can only write Check Runs here. + - name: "App token for the external repositories" + id: ext-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.EXT_CI_APP_ID }} + private-key: ${{ secrets.EXT_CI_APP_PRIVATE_KEY }} + owner: fprime-community + - name: "App token for the Check Runs" + id: checks-token + continue-on-error: true # App not installed here yet -> GITHUB_TOKEN (checks then land in another workflow's suite) + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.EXT_CI_APP_ID }} + private-key: ${{ secrets.EXT_CI_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} - name: "Dispatch, poll and report" env: - DISPATCH_TOKEN: ${{ secrets.EXT_CI_DISPATCH_TOKEN }} # external repos only; not exposed to other steps + DISPATCH_TOKEN: ${{ steps.ext-token.outputs.token }} # external repos only; not exposed to other steps + CHECKS_TOKEN: ${{ steps.checks-token.outputs.token }} INPUT_REPOS: ${{ inputs.repos }} INPUT_PR_NUMBER: ${{ inputs.pr_number }} INPUT_FPRIME_REF: ${{ inputs.fprime_ref }} @@ -54,7 +73,8 @@ jobs: - name: "Finalize (close open checks, cancel orphaned remote runs)" if: always() env: - DISPATCH_TOKEN: ${{ secrets.EXT_CI_DISPATCH_TOKEN }} # external repos only; not exposed to other steps + DISPATCH_TOKEN: ${{ steps.ext-token.outputs.token }} # external repos only; not exposed to other steps + CHECKS_TOKEN: ${{ steps.checks-token.outputs.token }} JOB_STATUS: ${{ job.status }} run: python3 .github/actions/ext-ci/orchestrator.py finalize - uses: actions/upload-artifact@v4 From c9951b11d2ee7e02f7655a7d0d301d06fd201081 Mon Sep 17 00:00:00 2001 From: "thomas.boyer.chammard" Date: Thu, 5 Nov 2026 19:19:51 +0000 Subject: [PATCH 22/22] ci: ext-ci scopes each App token to the permissions its client uses --- .github/workflows/ext-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ext-ci.yml b/.github/workflows/ext-ci.yml index a801b5bdfd1..fcbb173b647 100644 --- a/.github/workflows/ext-ci.yml +++ b/.github/workflows/ext-ci.yml @@ -52,6 +52,8 @@ jobs: app-id: ${{ secrets.EXT_CI_APP_ID }} private-key: ${{ secrets.EXT_CI_APP_PRIVATE_KEY }} owner: fprime-community + permission-actions: write + permission-contents: read - name: "App token for the Check Runs" id: checks-token continue-on-error: true # App not installed here yet -> GITHUB_TOKEN (checks then land in another workflow's suite) @@ -61,6 +63,10 @@ jobs: private-key: ${{ secrets.EXT_CI_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: ${{ github.event.repository.name }} + permission-checks: write + permission-actions: read + permission-contents: read + permission-pull-requests: read - name: "Dispatch, poll and report" env: DISPATCH_TOKEN: ${{ steps.ext-token.outputs.token }} # external repos only; not exposed to other steps