From 4af6e729345d4686de327a87f93dbfa75a1cff28 Mon Sep 17 00:00:00 2001 From: Sian Ford Date: Wed, 13 May 2026 00:10:26 +0100 Subject: [PATCH] ci(accessibility-report): generate accessibility report --- .github/workflows/playwright.yml | 10 +- playwright-ct.config.ts | 11 +- playwright/README.md | 27 + playwright/support/accessibility-reporter.ts | 532 +++++++++++++++++++ playwright/support/helper.ts | 55 +- 5 files changed, 614 insertions(+), 21 deletions(-) create mode 100644 playwright/support/accessibility-reporter.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index adc8365db3..a5ae7a170d 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -79,7 +79,7 @@ jobs: merge-multiple: true - name: Merge into HTML Report - run: npx playwright merge-reports --config=playwright-ct.config.ts --reporter=html ./all-blob-reports + run: npx playwright merge-reports --config=playwright-ct.config.ts --reporter=html --reporter=./playwright/support/accessibility-reporter.ts ./all-blob-reports - name: Upload HTML report uses: actions/upload-artifact@v6 @@ -87,3 +87,11 @@ jobs: name: html-report--attempt-${{ github.run_attempt }} path: playwright-report retention-days: 14 + + - name: Upload Accessibility report + if: always() + uses: actions/upload-artifact@v6 + with: + name: accessibility-report--attempt-${{ github.run_attempt }} + path: playwright/test-report/accessibility-report.* + retention-days: 14 diff --git a/playwright-ct.config.ts b/playwright-ct.config.ts index e289ed7bf3..01987bb344 100644 --- a/playwright-ct.config.ts +++ b/playwright-ct.config.ts @@ -23,8 +23,15 @@ export default defineConfig({ maxFailures: process.env.CI ? 10 : undefined, reporter: process.env.CI - ? "blob" - : [["html", { outputFolder: resolve(playwrightDir, "./test-report") }]], + ? [ + ["blob"], + [resolve(playwrightDir, "./support/accessibility-reporter.ts")], + ] + : [ + ["list"], + ["html", { outputFolder: resolve(playwrightDir, "./test-report") }], + [resolve(playwrightDir, "./support/accessibility-reporter.ts")], + ], use: { trace: "retain-on-failure", diff --git a/playwright/README.md b/playwright/README.md index f3b284f834..c1c0c51766 100644 --- a/playwright/README.md +++ b/playwright/README.md @@ -30,6 +30,33 @@ To run the tests locally: 4. To run specific Playwright tests at the command line run `npm run test:ct -- ./src/components/[component]/*.pw.tsx`. Test results can be seen in the console run summary. 5. We have specified three browsers to run tests on. So to run Playwright tests in a specific browser run `npm run test:ct -- --project=chromium` or `npm run test:ct -- --project=firefox`. If you want to run Playwright tests on a specific browser using `UI` runner you need to manually select which browser you want to use (or all available in `playwright-ct.config.ts`). +### Accessibility Report + +When tests are run, an **Accessibility Report** is automatically generated alongside the standard Playwright test report. This report consolidates all accessibility violations and incomplete checks detected during the test run. + +**Accessing the Report:** + +- **Locally**: After running `npm run test:ct`, view the report at `playwright/test-report/accessibility-report.html` +- **Via command**: Run `npm run test:ct:report` to open the standard Playwright HTML report. The accessibility report is in the same directory (`playwright/test-report/`) +- **In CI**: The accessibility report is included in the HTML report artifacts uploaded to GitHub Actions + +**Report Contents:** + +The accessibility report includes: + +- **Summary statistics**: Count of incomplete checks and violations by component +- **Filtering**: Search by component name, test name, or issue description; filter by type (incomplete/violation) or severity (critical/serious/moderate/minor) +- **Detailed issue information**: Each issue includes the rule ID, impact level, description, help text, and a link to detailed documentation +- **Affected elements**: For each issue, see which DOM elements are affected with their selectors and HTML +- **Export options**: Download the data as CSV or JSON for bug tracking systems + +**Understanding the Results:** + +- **Incomplete Checks** (Primary Focus): Accessibility rules that require manual verification because they cannot be fully automated (e.g., color contrast checks that need visual confirmation, ARIA attribute values that need contextual validation). These are displayed as console warnings during local test runs and captured in the report. They do NOT cause test failures but should be reviewed and logged as bugs if needed. +- **Violations**: Accessibility rules that definitively failed automated testing. These WILL cause test failures and must be fixed before merging. + +The report is particularly valuable for tracking **incomplete checks** that require manual review by Test Analysts. While tests continue to pass with incomplete checks, the report provides a centralized view for logging and tracking these issues. + > If you use VSCode as your code editor, you can also run tests via the official [Playwright Test for VSCode](https://playwright.dev/docs/getting-started-vscode) extension. ## Continuous Integration (CI) diff --git a/playwright/support/accessibility-reporter.ts b/playwright/support/accessibility-reporter.ts new file mode 100644 index 0000000000..f59c3392c7 --- /dev/null +++ b/playwright/support/accessibility-reporter.ts @@ -0,0 +1,532 @@ +/* eslint-disable no-console */ +import type { + Reporter, + TestCase, + TestResult, + FullResult, +} from "@playwright/test/reporter"; +import * as fs from "fs"; +import * as path from "path"; + +interface A11yIssue { + id: string; + impact: string; + description: string; + help: string; + helpUrl: string; + tags: string[]; + nodes?: Array<{ + html: string; + target: string[]; + failureSummary?: string; + }>; +} + +interface A11yRecord { + component: string; + testFile: string; + testTitle: string; + status: "incomplete" | "violation"; + issues: A11yIssue[]; + timestamp: string; +} + +class AccessibilityReporter implements Reporter { + private records: A11yRecord[] = []; + private outputDir: string; + + constructor() { + this.outputDir = path.resolve(__dirname, "../test-report"); + } + + onTestEnd(test: TestCase, result: TestResult): void { + // Extract accessibility data from attachments + const incompleteAttachment = result.attachments.find( + (a) => a.name === "accessibility-incomplete.json", + ); + const violationsAttachment = result.attachments.find( + (a) => a.name === "accessibility-violations.json", + ); + + const componentName = this.extractComponentName(test.location.file); + const testFile = test.location.file.replace(process.cwd(), ""); + + if (incompleteAttachment?.body) { + const issues = JSON.parse(incompleteAttachment.body.toString()); + this.records.push({ + component: componentName, + testFile, + testTitle: test.title, + status: "incomplete", + issues, + timestamp: new Date().toISOString(), + }); + } + + if (violationsAttachment?.body) { + const issues = JSON.parse(violationsAttachment.body.toString()); + this.records.push({ + component: componentName, + testFile, + testTitle: test.title, + status: "violation", + issues, + timestamp: new Date().toISOString(), + }); + } + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async onEnd(_result: FullResult): Promise { + if (this.records.length === 0) { + console.log("āœ“ No accessibility issues detected"); + return; + } + + // Ensure output directory exists + if (!fs.existsSync(this.outputDir)) { + fs.mkdirSync(this.outputDir, { recursive: true }); + } + + // Write JSON data + const jsonPath = path.join(this.outputDir, "accessibility-report.json"); + fs.writeFileSync(jsonPath, JSON.stringify(this.records, null, 2)); + + // Generate HTML report + const htmlPath = path.join(this.outputDir, "accessibility-report.html"); + fs.writeFileSync(htmlPath, this.generateHTML()); + + console.log( + `\nšŸ“Š Accessibility Report: ${this.records.length} issue(s) found`, + ); + console.log(` HTML: ${htmlPath}`); + console.log(` JSON: ${jsonPath}\n`); + } + + private extractComponentName(filePath: string): string { + const match = filePath.match(/components\/([^/]+)\//); + return match ? match[1] : "Unknown"; + } + + private generateHTML(): string { + const incompleteCount = this.records.filter( + (r) => r.status === "incomplete", + ).length; + const violationCount = this.records.filter( + (r) => r.status === "violation", + ).length; + + return ` + + + + + + Carbon Accessibility Report + + + +
+
+

Carbon Accessibility Report

+

Generated: ${new Date().toLocaleString()}

+
+
+
${incompleteCount}
+
Incomplete Checks
+
+
+
${violationCount}
+
Violations
+
+
+
+ + +
+
+ +
+
+ + + + + + + + +
+
+ +
+ ${this.records.map((record, idx) => this.generateRecordHTML(record, idx)).join("")} +
+ + +
+ + + +`; + } + + private generateRecordHTML(record: A11yRecord, idx: number): string { + return ` +
+
+
+
${this.escapeHtml(record.component)}
+
${this.escapeHtml(record.testTitle)}
+
${this.escapeHtml(record.testFile)}
+
+ ${record.status} +
+ ${record.issues.map((issue) => this.generateIssueHTML(issue)).join("")} +
`; + } + + private generateIssueHTML(issue: A11yIssue): string { + const impactClass = issue.impact || "minor"; + return ` +
+
+ ${this.escapeHtml(issue.id)} + ${issue.impact ? `${this.escapeHtml(issue.impact)}` : ""} +
+
${this.escapeHtml(issue.description)}
+
${this.escapeHtml(issue.help)}
+
+ + More information ā†’ + +
+ ${issue.nodes && issue.nodes.length > 0 ? this.generateNodesHTML(issue.nodes) : ""} +
`; + } + + private generateNodesHTML( + nodes: Array<{ html: string; target: string[]; failureSummary?: string }>, + ): string { + return ` +
+
Affected Elements (${nodes.length}):
+ ${nodes + .map( + (node) => ` +
+
Target: ${this.escapeHtml(node.target.join(", "))}
+ ${node.failureSummary ? `
${this.escapeHtml(node.failureSummary)}
` : ""} +
${this.escapeHtml(node.html.substring(0, 300))}${node.html.length > 300 ? "..." : ""}
+
`, + ) + .join("")} +
`; + } + + private escapeHtml(text: string): string { + const map: { [key: string]: string } = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'", + }; + return text.replace(/[&<>"']/g, (m) => map[m]); + } +} + +export default AccessibilityReporter; diff --git a/playwright/support/helper.ts b/playwright/support/helper.ts index 70a89d1335..10ddd0ec60 100644 --- a/playwright/support/helper.ts +++ b/playwright/support/helper.ts @@ -1,6 +1,6 @@ import type { Locator, Page } from "@playwright/test"; import AxeBuilder from "@axe-core/playwright"; -import { expect } from "@playwright/experimental-ct-react"; +import { expect, test } from "@playwright/experimental-ct-react"; import { label, legend } from "../components/index"; /** @@ -62,24 +62,43 @@ export const checkAccessibility = async ( .analyze(); const isCI = process.env.CI === "true"; + const testInfo = test.info(); + + // Always attach accessibility data for the report (both local and CI) + if (accessibilityScanResults.incomplete.length > 0) { + await testInfo.attach("accessibility-incomplete.json", { + body: JSON.stringify(accessibilityScanResults.incomplete, null, 2), + contentType: "application/json", + }); + + // Also show console warnings locally + if (!isCI) { + // Capture the calling stack + const { stack } = new Error(); + + // Parse stack to find the test file name + const testFileMatch = stack?.match( + /at .*?(\/[^\s]+\.pw\.tsx):(\d+):(\d+)/, + ); + const componentName = testFileMatch + ? testFileMatch[1].split("/").slice(-2, -1)[0] + : "Unknown component"; + + // eslint-disable-next-line no-console + console.warn( + `\nACCESSIBILITY SCAN INCOMPLETE. Incomplete rules: ${accessibilityScanResults.incomplete.map( + (rule) => + `\n\t- ${rule.id}: this is a ${rule.impact} accessibility issue. ${rule.description}`, + )}\nPlease check and ensure that the "${componentName}" component meets accessibility criteria manually.\n`, + ); + } + } - if (accessibilityScanResults.incomplete.length > 0 && !isCI) { - // Capture the calling stack - const { stack } = new Error(); - - // Parse stack to find the test file name - const testFileMatch = stack?.match(/at .*?(\/[^\s]+\.pw\.tsx):(\d+):(\d+)/); - const componentName = testFileMatch - ? testFileMatch[1].split("/").slice(-2, -1)[0] - : "Unknown component"; - - // eslint-disable-next-line no-console - console.warn( - `\nACCESSIBILITY SCAN INCOMPLETE. Incomplete rules: ${accessibilityScanResults.incomplete.map( - (rule) => - `\n\t- ${rule.id}: this is a ${rule.impact} accessibility issue. ${rule.description}`, - )}\nPlease check and ensure that the "${componentName}" component meets accessibility criteria manually.\n`, - ); + if (accessibilityScanResults.violations.length > 0) { + await testInfo.attach("accessibility-violations.json", { + body: JSON.stringify(accessibilityScanResults.violations, null, 2), + contentType: "application/json", + }); } expect(accessibilityScanResults.violations).toEqual([]);