Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ option(GREENFLAME_ENABLE_TIME_TRACE
"Emit Clang -ftime-trace JSON files during compilation"
OFF
)
option(GREENFLAME_ENABLE_COVERAGE
"Instrument greenflame_core and tests for LLVM source coverage (Clang only)"
OFF
)

# cmake-scripts (pulled by Catch2/CPM) sets CMAKE_COMPILE_WARNING_AS_ERROR=ON
# which adds /WX globally, breaking builds on warnings in third-party headers.
Expand Down Expand Up @@ -270,6 +274,9 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
if(GREENFLAME_ENABLE_TIME_TRACE)
target_compile_options(greenflame_core PRIVATE -ftime-trace)
endif()
if(GREENFLAME_ENABLE_COVERAGE)
target_compile_options(greenflame_core PRIVATE -fprofile-instr-generate -fcoverage-mapping)
endif()
elseif(MSVC)
target_compile_options(greenflame_core PRIVATE
/Wall
Expand Down
14 changes: 13 additions & 1 deletion CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,24 @@
"CMAKE_C_COMPILER": "clang-cl.exe",
"CMAKE_CXX_COMPILER": "clang-cl.exe"
}
},
{
"name": "x64-coverage-clang",
"generator": "Ninja",
"binaryDir": "${sourceDir}/build/x64-coverage-clang",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Debug",
"CMAKE_C_COMPILER": "clang-cl.exe",
"CMAKE_CXX_COMPILER": "clang-cl.exe",
"GREENFLAME_ENABLE_COVERAGE": "ON"
}
}
],
"buildPresets": [
{ "name": "x64-debug", "configurePreset": "x64-debug" },
{ "name": "x64-release", "configurePreset": "x64-release" },
{ "name": "x64-debug-clang", "configurePreset": "x64-debug-clang" },
{ "name": "x64-release-clang", "configurePreset": "x64-release-clang" }
{ "name": "x64-release-clang", "configurePreset": "x64-release-clang" },
{ "name": "x64-coverage-clang", "configurePreset": "x64-coverage-clang" }
]
}
131 changes: 131 additions & 0 deletions docs/coverage.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
---
title: Coverage Guide
summary: How to measure LLVM source coverage for greenflame_core.
audience: contributors
status: authoritative
owners:
- core-team
last_updated: 2026-04-06
tags:
- coverage
- llvm-cov
- clang
---

# Coverage Guide

This document describes how to measure source coverage for `greenflame_core`
using LLVM's instrumentation-based coverage (`llvm-cov`).

Coverage is a diagnostic tool, not a gate. It helps identify untested paths in
`greenflame_core`. It is not required before considering a task complete.

---

## Prerequisites

In addition to the standard build prerequisites in [build.md](build.md):

- **clang-cl.exe** — provided by the Visual Studio 2026 "C++ Clang compiler
for Windows" component (already required for the `x64-debug-clang` preset).
- **llvm-profdata.exe** and **llvm-cov.exe** — these ship with the VS
"C++ Clang compiler for Windows" component at:
```
<VS install>\VC\Tools\Llvm\x64\bin\
```
`scripts\coverage.ps1` adds this directory to `PATH` automatically when
`VSINSTALLDIR` is set (i.e. when run from a VS Developer Command Prompt or
after running `VsDevCmd.bat`). In a plain shell, set `VSINSTALLDIR` or add
the directory to `PATH` manually.

---

## Running coverage

A helper script handles the full workflow:

```powershell
.\scripts\coverage.ps1
```

The script:
1. Configures with the `x64-coverage-clang` CMake preset (Clang + coverage
instrumentation enabled).
2. Builds `greenflame_tests`.
3. Runs the test binary with `LLVM_PROFILE_FILE` set to capture raw profile
data.
4. Merges the raw profile with `llvm-profdata`.
5. Prints a summary coverage table scoped to `src\greenflame_core\`.
6. Writes an HTML line-level report to
`build\x64-coverage-clang\coverage\report\index.html`.

To open the HTML report after the run:

```powershell
start build\x64-coverage-clang\coverage\report\index.html
```

---

## How it works

The `x64-coverage-clang` CMake preset sets `GREENFLAME_ENABLE_COVERAGE=ON`.
When enabled, CMake adds the following flags to `greenflame_core` and
`greenflame_tests`:

| Target | Compile flags | Link flags |
|---------------------|------------------------------------------------------|-------------------------------|
| `greenflame_core` | `-fprofile-instr-generate -fcoverage-mapping` | — |
| `greenflame_tests` | `-fprofile-instr-generate -fcoverage-mapping` | `-fprofile-instr-generate` |

Because `greenflame_core` is a static library, its instrumented object files
are linked into `greenflame_tests.exe`, which is the binary passed to
`llvm-cov`.

---

## Manual workflow

If you prefer to run the steps yourself:

```powershell
# 1. Configure and build
cmake --preset x64-coverage-clang
cmake --build --preset x64-coverage-clang

# 2. Run tests (generates coverage.profraw)
$env:LLVM_PROFILE_FILE = "$PWD\build\x64-coverage-clang\coverage\coverage.profraw"
.\build\x64-coverage-clang\bin\greenflame_tests.exe
Remove-Item Env:\LLVM_PROFILE_FILE

# 3. Merge profile data
llvm-profdata merge -sparse `
build\x64-coverage-clang\coverage\coverage.profraw `
-o build\x64-coverage-clang\coverage\coverage.profdata

# 4. Summary report (scoped to greenflame_core)
llvm-cov report `
.\build\x64-coverage-clang\bin\greenflame_tests.exe `
"--instr-profile=build\x64-coverage-clang\coverage\coverage.profdata" `
src\greenflame_core

# 5. HTML report
llvm-cov show `
.\build\x64-coverage-clang\bin\greenflame_tests.exe `
"--instr-profile=build\x64-coverage-clang\coverage\coverage.profdata" `
"--output-dir=build\x64-coverage-clang\coverage\report" `
-format=html `
src\greenflame_core
```

---

## Scope

The report is scoped to `src\greenflame_core\`. GoogleTest internals and test
source files under `tests\` are excluded from the summary by passing the core
source directory as a positional filter to `llvm-cov`.

Coverage of `src\greenflame\` (the Win32 GUI layer) is intentionally excluded:
that code cannot run in the headless test binary and is not a target for
automated coverage.
10 changes: 10 additions & 0 deletions docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ Test must be run and must pass before any task is considered complete. This is a
- Register them in `tests/CMakeLists.txt` as sources of `greenflame_tests`
- Tests must only link against `greenflame_core` and the testable logic library — never against `greenflame` directly

## Source coverage

LLVM-based source coverage for `greenflame_core` can be generated with:

```powershell
.\scripts\coverage.ps1
```

See [docs/coverage.md](coverage.md) for prerequisites and details.

## Manual verification coverage

Some Win32 overlay behaviors cannot be exercised in the unit-test binary because `greenflame_tests`
Expand Down
121 changes: 121 additions & 0 deletions scripts/coverage.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
<#
.SYNOPSIS
Build greenflame_core with LLVM coverage instrumentation, run tests, and
produce a coverage report scoped to src/greenflame_core/.

.DESCRIPTION
Requires:
- clang-cl.exe, llvm-profdata.exe, and llvm-cov.exe
These ship with the Visual Studio "C++ Clang compiler for Windows"
component at:
<VS install>\VC\Tools\Llvm\x64\bin\
The script adds that directory to PATH automatically when $env:VSINSTALLDIR
is set (i.e. when run from a VS Developer Command Prompt or after
invoking VsDevCmd.bat). If you run from a plain shell, either set
VSINSTALLDIR or add the Llvm\x64\bin directory to PATH manually.

Produces (all under build\x64-coverage-clang\coverage\):
- coverage.profraw raw instrumentation profile
- coverage.profdata merged profile
- report\ HTML line-level coverage report
A summary table scoped to src/greenflame_core/ is printed to stdout.

.EXAMPLE
# From the repo root:
.\scripts\coverage.ps1

# Open the HTML report after the script finishes:
start build\x64-coverage-clang\coverage\report\index.html
#>

$ErrorActionPreference = "Stop"

# ── resolve repo root ──────────────────────────────────────────────────────────
$RepoRoot = Split-Path -Parent $PSScriptRoot
Set-Location $RepoRoot

# ── add VS LLVM bin to PATH if needed ─────────────────────────────────────────
# llvm-cov and llvm-profdata ship with the VS "C++ Clang" component at:
# <VS install>\VC\Tools\Llvm\x64\bin\
# Prepend that directory when $env:VSINSTALLDIR is set and the tools aren't
# already on PATH.
if ($env:VSINSTALLDIR -and
-not (Get-Command "llvm-cov" -ErrorAction SilentlyContinue)) {
Comment on lines +42 to +43
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Add VS LLVM bin when either coverage tool is missing

The PATH bootstrap only runs when llvm-cov is absent, but the script later requires both llvm-cov and llvm-profdata. On machines where llvm-cov is already on PATH (for example from another LLVM install) but llvm-profdata is missing or version-mismatched, this block skips adding Visual Studio's LLVM bin and the run fails or mixes incompatible tool versions. Gate this on both tools (or always prepend VS LLVM when VSINSTALLDIR is set) so the toolchain is consistent.

Useful? React with 👍 / 👎.

$vsLlvmBin = Join-Path $env:VSINSTALLDIR "VC\Tools\Llvm\x64\bin"
if (Test-Path $vsLlvmBin) {
$env:PATH = "$vsLlvmBin;$env:PATH"
}
}

# ── verify required tools ──────────────────────────────────────────────────────
$missing = @()
foreach ($tool in @("cmake", "ninja", "llvm-profdata", "llvm-cov")) {
if (-not (Get-Command $tool -ErrorAction SilentlyContinue)) {
$missing += $tool
}
}
if ($missing.Count -gt 0) {
Write-Error @"
The following tools were not found on PATH: $($missing -join ', ')
See docs/coverage.md for setup instructions.
"@
}

# ── configure ─────────────────────────────────────────────────────────────────
Write-Host "=== Configure (x64-coverage-clang) ==="
cmake --preset x64-coverage-clang
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

# ── build ─────────────────────────────────────────────────────────────────────
Write-Host ""
Write-Host "=== Build (x64-coverage-clang) ==="
cmake --build --preset x64-coverage-clang
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

# ── paths ─────────────────────────────────────────────────────────────────────
$TestExe = "build\x64-coverage-clang\bin\greenflame_tests.exe"
$CovDir = "build\x64-coverage-clang\coverage"
$ProfRaw = "$CovDir\coverage.profraw"
$ProfData = "$CovDir\coverage.profdata"
$HtmlDir = "$CovDir\report"
$CoreSrc = "src\greenflame_core"

New-Item -ItemType Directory -Force -Path $CovDir | Out-Null

# ── run tests with LLVM instrumentation ───────────────────────────────────────
Write-Host ""
Write-Host "=== Run tests ==="
$env:LLVM_PROFILE_FILE = (Resolve-Path -LiteralPath $CovDir).Path + "\coverage.profraw"
& $TestExe
$TestExitCode = $LASTEXITCODE
Remove-Item Env:\LLVM_PROFILE_FILE -ErrorAction SilentlyContinue

if ($TestExitCode -ne 0) {
Write-Warning "Tests reported failures (exit $TestExitCode). Coverage data was still captured."
}

# ── merge profile data ────────────────────────────────────────────────────────
Write-Host ""
Write-Host "=== Merge profile data ==="
llvm-profdata merge -sparse $ProfRaw -o $ProfData
Comment on lines +97 to +100
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Prevent stale profraw reuse before merge

The merge step unconditionally consumes coverage.profraw without ensuring it was produced by the current test run. If a previous run left coverage.profraw behind and the current run crashes or exits before flushing profile data, llvm-profdata merge will still succeed on stale data and produce an incorrect coverage report for this invocation. Remove any existing raw profile before running tests (or verify fresh timestamp/existence after test execution) to avoid reporting outdated coverage.

Useful? React with 👍 / 👎.

if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

# ── summary report (stdout, scoped to greenflame_core) ────────────────────────
Write-Host ""
Write-Host "=== Coverage summary (src\greenflame_core) ==="
llvm-cov report $TestExe "--instr-profile=$ProfData" $CoreSrc
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

# ── HTML report ───────────────────────────────────────────────────────────────
Write-Host ""
Write-Host "=== HTML report -> $HtmlDir ==="
llvm-cov show $TestExe `
"--instr-profile=$ProfData" `
"--output-dir=$HtmlDir" `
-format=html `
$CoreSrc
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

Write-Host ""
Write-Host "Done. Open: $HtmlDir\index.html"
exit $TestExitCode
4 changes: 4 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,10 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
if(GREENFLAME_ENABLE_TIME_TRACE)
target_compile_options(greenflame_tests PRIVATE -ftime-trace)
endif()
if(GREENFLAME_ENABLE_COVERAGE)
target_compile_options(greenflame_tests PRIVATE -fprofile-instr-generate -fcoverage-mapping)
target_link_options(greenflame_tests PRIVATE -fprofile-instr-generate)
endif()
elseif(MSVC)
target_compile_options(greenflame_tests PRIVATE
/Wall
Expand Down
Loading