Allow completed Codecov failures during automerge

README.md

@@ -1,6 +1,6 @@
 # Automerge
 
-GitHub App that merges an eligible pull request after every reported check run and commit status passes. Add either the `automerge` or
+GitHub App that merges an eligible pull request after checks finish and blocking checks pass. Add either the `automerge` or
 `tag: automerge` label to opt a pull request in.
 
 - Website: <https://automerge.python-templates.dev>
@@ -22,8 +22,9 @@ The App evaluates labeled pull requests when:
 - a legacy commit status changes
 - a review is submitted or dismissed
 
-It ignores closed pull requests, drafts, and pull requests without an automerge label. It waits unless every latest check run has a `success`
-conclusion and every latest commit status is `success`. Once all reported work passes, it requests a SHA-pinned merge.
+It ignores closed pull requests, drafts, and pull requests without an automerge label. It waits for every latest check run to be `completed` and every
+latest commit status to leave `pending`. Completed `codecov/patch` and `codecov/project` results are non-blocking. All other check-run conclusions and
+commit statuses must be `success` before it requests a SHA-pinned merge.
 
 The App also waits up to 10 seconds for GitHub to calculate whether the current head can merge cleanly with the latest base branch. This calculation
 can be pending after another pull request merges. Pull requests with merge conflicts are left open.

src/automerge.ts

@@ -1,6 +1,7 @@
 import type { CheckRun, CommitStatus, IssueEvent, MergeMethod, PullRequest, RepositorySettings } from "./github";
 
 export const AUTOMERGE_LABELS = new Set(["automerge", "tag: automerge"]);
+export const NON_BLOCKING_CHECKS = new Set(["codecov/patch", "codecov/project"]);
 export const AUTOMERGE_GRACE_PERIOD_MS = 10_000;
 export const MERGE_ATTEMPT_LIMIT = 3;
 export const MERGEABILITY_POLL_INTERVAL_MS = 1_000;
@@ -95,8 +96,14 @@ export async function evaluatePullRequest(
       github.statuses(pullRequest.head.sha),
     ]);
     if (
-      checks.some((check) => check.status !== "completed" || check.conclusion !== "success") ||
-      statuses.some((status) => status.state !== "success")
+      checks.some(
+        (check) =>
+          check.status !== "completed" || (check.conclusion !== "success" && !NON_BLOCKING_CHECKS.has(check.name)),
+      ) ||
+      statuses.some(
+        (status) =>
+          status.state === "pending" || (status.state !== "success" && !NON_BLOCKING_CHECKS.has(status.context)),
+      )
     ) {
       console.log(`Checks have not passed on ${repository}#${number}`);
       return false;

src/github.ts

@@ -12,6 +12,7 @@ export interface PullRequest {
 }
 
 export interface CheckRun {
+  name: string;
   status: string;
   conclusion: string | null;
 }

src/pages.ts

@@ -28,8 +28,8 @@ function layout(title: string, description: string, content: string): Response {
 export function landingPage(): Response {
   return layout(
     "Automerge — Label-driven pull request merging",
-    "Automerge merges labeled pull requests after all reported GitHub checks and commit statuses pass.",
-    `<section class="hero"><div class="eyebrow">GitHub App</div><h1>Merge when GitHub says it is ready.</h1><p>Add an <code>automerge</code> or <code>tag: automerge</code> label. The App waits for all reported work to pass, rechecks the pull request, and asks GitHub to merge the exact checked commit.</p><div class="actions"><a class="button primary" href="${INSTALL_URL}">Install Automerge</a><a class="button" href="${REPOSITORY_URL}#readme">View documentation</a></div></section><section class="grid" aria-label="Benefits"><article class="card"><div class="eyebrow">Explicit</div><h2>Labels opt pull requests in</h2><p>Nothing merges until a maintainer adds one of the two recognized labels.</p></article><article class="card"><div class="eyebrow">Patient</div><h2>Reported work passes first</h2><p>Automerge requires successful check-run conclusions and commit statuses.</p></article><article class="card"><div class="eyebrow">Protected</div><h2>GitHub keeps final control</h2><p>Required reviews, rulesets, branch protection, and merge policies still apply.</p></article></section><section id="how-it-works" class="steps" aria-label="How Automerge works"><div class="step">Add an automerge label to an open, non-draft pull request.</div><div class="step">All reported checks and commit statuses pass.</div><div class="step">A 10-second grace period passes and eligibility is checked again.</div><div class="step">GitHub receives a merge request pinned to the checked head commit.</div></section>`,
+    "Automerge merges labeled pull requests after checks finish and blocking checks pass.",
+    `<section class="hero"><div class="eyebrow">GitHub App</div><h1>Merge when GitHub says it is ready.</h1><p>Add an <code>automerge</code> or <code>tag: automerge</code> label. The App waits for checks to finish and blocking checks to pass, rechecks the pull request, and asks GitHub to merge the exact checked commit.</p><div class="actions"><a class="button primary" href="${INSTALL_URL}">Install Automerge</a><a class="button" href="${REPOSITORY_URL}#readme">View documentation</a></div></section><section class="grid" aria-label="Benefits"><article class="card"><div class="eyebrow">Explicit</div><h2>Labels opt pull requests in</h2><p>Nothing merges until a maintainer adds one of the two recognized labels.</p></article><article class="card"><div class="eyebrow">Patient</div><h2>Checks finish before merging</h2><p>Completed <code>codecov/patch</code> and <code>codecov/project</code> results are non-blocking. All other checks must succeed.</p></article><article class="card"><div class="eyebrow">Protected</div><h2>GitHub keeps final control</h2><p>Required reviews, rulesets, branch protection, and merge policies still apply.</p></article></section><section id="how-it-works" class="steps" aria-label="How Automerge works"><div class="step">Add an automerge label to an open, non-draft pull request.</div><div class="step">All checks finish, and blocking checks and commit statuses pass.</div><div class="step">A 10-second grace period passes and eligibility is checked again.</div><div class="step">GitHub receives a merge request pinned to the checked head commit.</div></section>`,
   );
 }
 

tests/automerge.test.ts

@@ -21,7 +21,7 @@ class FakeGitHub implements AutomergeGitHub {
     labels: [{ name: "automerge" }],
     head: { sha: "abc123" },
   };
-  checks: CheckRun[] = [{ status: "completed", conclusion: "success" }];
+  checks: CheckRun[] = [{ name: "build", status: "completed", conclusion: "success" }];
   commitStatuses: CommitStatus[] = [{ context: "legacy", state: "success" }];
   events: IssueEvent[] = [{ event: "labeled", created_at: "2000-01-01T00:00:00Z", label: { name: "automerge" } }];
   settings: RepositorySettings = {
@@ -184,8 +184,8 @@ test("does not attempt to merge a conflicting pull request", async () => {
 test("waits until check runs and commit statuses finish", async () => {
   const github = new FakeGitHub();
   github.checks = [
-    { status: "completed", conclusion: "success" },
-    { status: "in_progress", conclusion: null },
+    { name: "build", status: "completed", conclusion: "success" },
+    { name: "build", status: "in_progress", conclusion: null },
   ];
   github.commitStatuses = [{ context: "legacy", state: "pending" }];
 
@@ -197,7 +197,7 @@ test.each(["action_required", "cancelled", "failure", "neutral", "skipped", "sta
   "does not merge when a check run concludes %s",
   async (conclusion) => {
     const github = new FakeGitHub();
-    github.checks = [{ status: "completed", conclusion }];
+    github.checks = [{ name: "build", status: "completed", conclusion }];
 
     expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(false);
     expect(github.merges).toEqual([]);
@@ -215,8 +215,8 @@ test.each(["error", "failure"] as const)("does not merge when a commit status is
 test("asks GitHub to merge after all checks pass", async () => {
   const github = new FakeGitHub();
   github.checks = [
-    { status: "completed", conclusion: "success" },
-    { status: "completed", conclusion: "success" },
+    { name: "build", status: "completed", conclusion: "success" },
+    { name: "build", status: "completed", conclusion: "success" },
   ];
   github.commitStatuses = [
     { context: "required", state: "success" },
@@ -288,3 +288,70 @@ test("selects the first enabled merge method", () => {
   );
   expect(mergeMethod({ allow_squash_merge: false, allow_merge_commit: false, allow_rebase_merge: false })).toBeNull();
 });
+
+test.each(["codecov/patch", "codecov/project"])("merges after completed %s failures", async (name) => {
+  const github = new FakeGitHub();
+  github.checks.push({ name, status: "completed", conclusion: "failure" });
+
+  expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(true);
+  expect(github.merges).toEqual([{ number: 12, sha: "abc123", method: "squash" }]);
+});
+
+test.each([
+  { context: "codecov/patch", state: "failure" },
+  { context: "codecov/patch", state: "error" },
+  { context: "codecov/project", state: "failure" },
+  { context: "codecov/project", state: "error" },
+] as const)("merges after $context commit status $state", async (status) => {
+  const github = new FakeGitHub();
+  github.commitStatuses.push(status);
+
+  expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(true);
+  expect(github.merges).toEqual([{ number: 12, sha: "abc123", method: "squash" }]);
+});
+
+test.each(["codecov/patch", "codecov/project"])("waits for pending %s check runs", async (name) => {
+  const github = new FakeGitHub();
+  github.checks.push({ name, status: "in_progress", conclusion: null });
+
+  expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(false);
+  expect(github.merges).toEqual([]);
+});
+
+test.each(["codecov/patch", "codecov/project"])("waits for pending %s commit statuses", async (context) => {
+  const github = new FakeGitHub();
+  github.commitStatuses.push({ context, state: "pending" });
+
+  expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(false);
+  expect(github.merges).toEqual([]);
+});
+
+test("failed build checks still block merges with failed Codecov results", async () => {
+  const github = new FakeGitHub();
+  github.checks = [
+    { name: "build", status: "completed", conclusion: "failure" },
+    { name: "codecov/patch", status: "completed", conclusion: "failure" },
+  ];
+  github.commitStatuses.push({ context: "codecov/project", state: "failure" });
+
+  expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(false);
+  expect(github.merges).toEqual([]);
+});
+
+test("failed build statuses still block merges with failed Codecov results", async () => {
+  const github = new FakeGitHub();
+  github.checks.push({ name: "codecov/patch", status: "completed", conclusion: "failure" });
+  github.commitStatuses.push({ context: "build", state: "failure" });
+
+  expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(false);
+  expect(github.merges).toEqual([]);
+});
+
+test.each(["codecov/build", "codecov/patch-build"])("does not exempt an unlisted check named %s", async (name) => {
+  const github = new FakeGitHub();
+  github.checks.push({ name, status: "completed", conclusion: "failure" });
+  github.commitStatuses.push({ context: name, state: "failure" });
+
+  expect(await evaluatePullRequest(github, 12, "owner/repository")).toBe(false);
+  expect(github.merges).toEqual([]);
+});