diff --git a/.bazelignore b/.bazelignore index 83c51ebd..fbcb4bd3 100644 --- a/.bazelignore +++ b/.bazelignore @@ -13,6 +13,9 @@ python_basics/integration_tests starpls/integration_tests +# Separate local Bazel module: consumer-style integration test for the +# LLVM coverage pipeline (run from its own directory) +coverage/integration_tests # Should be run from its own directory (separate Module) bazel/rules/rules_score/test # Separate local Bazel modules (use @seooc// and @some_other_library// instead) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67676623..2575cabe 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,6 +87,9 @@ jobs: run: | cd cr_checker/tests bazel test //... + - name: coverage + run: | + coverage/integration_tests/run_integration_test.sh steps: - name: Checkout repository uses: actions/checkout@v7.0.0 diff --git a/.gitignore b/.gitignore index e5bb7211..c2df75cc 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,8 @@ user.bazelrc __pycache__ .ruff_cache/ coverage-html/ + +# Outputs of coverage/integration_tests/run_integration_test.sh +coverage/integration_tests/coverage_artifacts.zip +coverage/integration_tests/lcov.dat +coverage/integration_tests/coverage_linux/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac7eb751..3c9bf678 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: check-executables-have-shebangs - id: check-added-large-files args: [--maxkb=100, --enforce-all] # increase or add git lfs if too strict - exclude: org.eclipse.dash.licenses-1.1.0.jar|blanket_index.html + exclude: org.eclipse.dash.licenses-1.1.0.jar - repo: local hooks: - id: copyright diff --git a/MODULE.bazel b/MODULE.bazel index 9cbb87df..5e5c3277 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -22,8 +22,10 @@ module( ############################################################################### bazel_dep(name = "aspect_rules_py", version = "1.4.0") # py_binary/py_library loads in cli_helper.bzl:13, cr_checker.bzl:16, defs.bzl:16, etc. -bazel_dep(name = "aspect_rules_lint", version = "2.5.0", dev_dependency = True) - +# NOT a dev_dependency: third_party/format/macros.bzl (use_format_targets, +# public consumer API) load()s @aspect_rules_lint//format:defs.bzl, which +# must resolve from score_tooling's own repo mapping in CONSUMER builds too. +bazel_dep(name = "aspect_rules_lint", version = "2.5.0") bazel_dep(name = "rules_multitool", version = "1.9.0") bazel_dep(name = "rules_java", version = "8.15.1") # required for dash @@ -215,6 +217,24 @@ pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") # Lobster Bazel use_repo(pip, "pip_tooling") +# Coverage pipeline Python deps (pyyaml for justify.py), kept in its own hub +# so the reusable coverage component stays independent of the other tools. +[ + pip.parse( + envsubst = ["PIP_INDEX_URL"], + extra_pip_args = ["--index-url=${PIP_INDEX_URL:-https://pypi.org/simple/}"], + hub_name = "pip_coverage", + python_version = "3.{}".format(version), + requirements_lock = "//coverage:requirements_3_{}.txt".format(version), + ) + for version in [ + "11", + "12", + ] +] + +use_repo(pip, "pip_coverage") + [ pip.parse( envsubst = ["PIP_INDEX_URL"], diff --git a/README.md b/README.md index 2e151be3..d5d05a0f 100644 --- a/README.md +++ b/README.md @@ -38,20 +38,29 @@ See the individual README files for detailed usage instructions and configuratio | **python_basics** | Python development utilities and testing | [README](python_basics/README.md) | | **starpls** | Starlark language server support | [README](starpls/README.md) | | **tools** | Formatters & Linters | [README](tools/README.md) | -| **coverage** | Rust + Python coverage reports | [README](coverage/README.md) | +| **coverage** | Unified LLVM source-based coverage (C++ + Rust) | [README](coverage/README.md) | ## Coverage -Generate a combined Rust + Python HTML coverage report for `plantuml`, `validation`, -and `manual_analysis`: +The `coverage/` module provides the reusable LLVM source-based coverage pipeline +used across S-CORE repositories: one report covering C++ and Rust (line + branch), +exact 0% entries for untested in-scope files, a justification system +(`COV_JUSTIFIED` markers + YAML), and effective-coverage gating. See the +[adoption guide](coverage/README.md) and the +[mechanism deep-dive](coverage/COVERAGE_GUIDE.md). + +> **Breaking change:** the former Ferrocene `symbol-report`/`blanket` workflow +> (`rust_coverage_report`, `//coverage:ferrocene_report`) was removed. The LLVM +> pipeline replaces it with unified C++ + Rust reports; see the +> [adoption guide](coverage/README.md) for migration. + +Generate a combined Rust + Python HTML coverage report for this repository's own +tools (`plantuml`, `validation`, `manual_analysis`): ```bash bazel run //coverage:combined_report ``` -See [coverage/README.md](coverage/README.md) for full details, options, and the -Ferrocene Rust coverage workflow. - ## Usage Examples Load tools in your `BUILD` files: @@ -59,27 +68,28 @@ Load tools in your `BUILD` files: ```starlark load("@score_tooling//:defs.bzl", "score_py_pytest") load("@score_tooling//:defs.bzl", "cli_tool") -load("@score_tooling//coverage:coverage.bzl", "rust_coverage_report") +load("@score_tooling//coverage:defs.bzl", "score_coverage_reporter", "score_coverage_scope") ``` -Create a repo-local coverage target: +Declare the coverage scope and reporter for your repository (see the +[coverage adoption guide](coverage/README.md) for the full setup, including the +required `.bazelrc` configuration and toolchains): ```starlark -rust_coverage_report( - name = "rust_coverage", - bazel_configs = [ - "ferrocene-x86_64-linux", - "ferrocene-coverage", - ], - query = 'kind("rust_test", //...)', - min_line_coverage = "80", +score_coverage_scope( + name = "coverage_scope", + testonly = True, + deps = ["//src/mylib"], ) -``` -Then run: - -```bash -bazel run //:rust_coverage -- --min-line-coverage 80 +score_coverage_reporter( + name = "reporter_wrapper", + testonly = True, + coverage_scope = ":coverage_scope", + llvm_cov = "@llvm_toolchain//:llvm-cov", + llvm_profdata = "@llvm_toolchain//:llvm-profdata", + llvm_cxxfilt = "@llvm_toolchain_llvm//:bin/llvm-cxxfilt", +) ``` ## Upgrading from separate MODULES @@ -100,7 +110,8 @@ The available import targets are: - dash_license_checker - cli_helper - setup_starpls -- rust_coverage_report +- score_coverage_scope +- score_coverage_reporter Formatting, linting, and cr_checker are no longer re-exported from `defs.bzl`; use `@score_tooling//third_party/format:macros.bzl`, `@score_tooling//third_party/lint:macros.bzl`, diff --git a/REUSE.toml b/REUSE.toml index 04fb790c..07bb1216 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -37,7 +37,7 @@ SPDX-License-Identifier = "Apache-2.0" [[annotations]] path = ["cr_checker/resources/config.json", "**/.bazelversion", - "coverage/tests/fixtures/symbol_report.json", + "coverage/requirements.in", "python_basics/.bazelversion", "python_basics/integration_tests/venv-with-extra-requirements/requirements.in", "python_basics/requirements.in", diff --git a/coverage/BUILD b/coverage/BUILD index 3553984b..614491dd 100644 --- a/coverage/BUILD +++ b/coverage/BUILD @@ -11,26 +11,120 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +load("@pip_coverage//:requirements.bzl", "requirement") +load("@rules_cc//cc/toolchains:args.bzl", "cc_args") +load("@rules_cc//cc/toolchains:feature.bzl", "cc_feature") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") +load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") package(default_visibility = ["//visibility:public"]) -exports_files([ - "ferrocene_report_wrapper.sh.tpl", - "scripts/normalize_symbol_report.py", - "scripts/parse_line_coverage.py", -]) +# ============================================================================= +# LLVM source-based coverage pipeline (C++ + Rust). +# Consumer-facing API lives in defs.bzl; see README.md for the adoption guide +# and COVERAGE_GUIDE.md for how the pipeline works. +# ============================================================================= + +# Per-test coverage output generator. Referenced by consumers directly: +# coverage:llvm_cov --coverage_output_generator=@score_tooling//coverage:merger +# It locates llvm-profdata via the LLVM_PROFDATA / RUST_LLVM_PROFDATA +# environment variables exported by the toolchains, so it needs no +# consumer-supplied labels. +py_binary( + name = "merger", + srcs = ["merger.py"], +) + +# Final report generator. NOT referenced directly by consumers — the +# score_coverage_reporter macro wraps it with the consumer's coverage scope, +# workspace root and LLVM tool labels. +py_binary( + name = "reporter", + srcs = ["reporter.py"], + deps = ["@rules_python//python/runfiles"], +) + +py_binary( + name = "justify", + srcs = ["justify.py"], + deps = [requirement("pyyaml")], +) + +py_binary( + name = "effective_coverage", + srcs = ["effective_coverage.py"], +) sh_binary( - name = "ferrocene_report", - srcs = ["ferrocene_report.sh"], - data = [ - "scripts/normalize_symbol_report.py", - "scripts/parse_line_coverage.py", + name = "generate_coverage_html", + srcs = ["generate_coverage_html.sh"], +) + +# Import-only libraries so unit tests can `from coverage.merger import ...`. +py_library( + name = "merger_lib", + srcs = ["merger.py"], + imports = [".."], +) + +py_library( + name = "reporter_lib", + srcs = ["reporter.py"], + imports = [".."], + deps = ["@rules_python//python/runfiles"], +) + +# These compile time options are required to cover abnormal termination cases +# (death tests). LLVM provides them in combination with a specific profile +# setting which is enabled in Bazel via LLVM_PROFILE_CONTINUOUS_MODE. +# Consumers reference this feature from their toolchains_llvm extension config: +# llvm.toolchain(extra_known_features = +# ["@score_tooling//coverage:enable_llvm_coverage_for_death_tests"], ...) +cc_args( + name = "runtime_relocation_args", + actions = [ + "@rules_cc//cc/toolchains/actions:compile_actions", + "@rules_cc//cc/toolchains/actions:link_actions", ], - visibility = ["//visibility:public"], + args = [ + "-mllvm", + "-runtime-counter-relocation", + ], +) + +cc_feature( + name = "enable_llvm_coverage_for_death_tests", + args = [":runtime_relocation_args"], + feature_name = "enable_llvm_coverage_for_death_tests", ) +# In order to update the requirements, change the `requirements.in` file and run: +# `bazel run //coverage:requirements_3_XX.update --@@rules_python+//python/config_settings:python_version=3.XX`. +[ + compile_pip_requirements( + name = "requirements_3_{}".format(version), + src = "requirements.in", + python_version = "3.{}".format(version), + requirements_txt = "requirements_3_{}.txt".format(version), + tags = [ + "manual", + ], + ) + for version in [ + "11", + "12", + ] +] + +# ============================================================================= +# Legacy helpers (kept): +# - llvm_profile_wrapper: still referenced by downstream ferrocene-coverage +# configs (test --run_under=...) during migration to the LLVM pipeline. +# - combined_report: Rust + Python lcov/genhtml report for THIS repository's +# own tools, wired into deploy_docs.yml. +# ============================================================================= + sh_binary( name = "llvm_profile_wrapper", srcs = ["llvm_profile_wrapper.sh"], diff --git a/coverage/COVERAGE_GUIDE.md b/coverage/COVERAGE_GUIDE.md new file mode 100644 index 00000000..75deba79 --- /dev/null +++ b/coverage/COVERAGE_GUIDE.md @@ -0,0 +1,333 @@ + + +# Unified Code Coverage (LLVM) — how the pipeline works + +This document explains, from first principles, how the unified Rust + C++ +coverage pipeline in `@score_tooling//coverage` works: the mechanism, the +module/consumer split, the non-obvious pitfalls, and where the pipeline comes +from. For the step-by-step consumer setup see [README.md](README.md); for a +working consumer workspace see [integration_tests/](integration_tests/). + +--- + +## 1. Background concepts + +### 1.1 What "code coverage" means here + +When we run the test suite, we want to know **which lines of production code +were actually executed**. The compiler helps: it can *instrument* the code — +insert tiny counters at every branch and statement. When an instrumented test +binary runs, it writes the counter values to a file. Tooling then maps those +counters back to source lines and produces a report: green = executed, +red = never executed. + +### 1.2 Bazel in three paragraphs + +Bazel is the build system used by S-CORE. Code is organized into **targets** +(a library, a binary, a test), declared in files named `BUILD`. Targets +reference each other by **labels** like `//src/rust/mycrate:tests` +(`//path/to/package:target_name`). + +External dependencies (compilers, libraries) are declared in `MODULE.bazel`. +A **toolchain** is Bazel's packaging of a compiler + flags; you can register +several and select one per build. Command-line defaults live in `.bazelrc`, +grouped into named **configs**: `bazel test --config=foo` applies all lines +starting with `test:foo`. + +Bazel has a built-in coverage mode: `bazel coverage ` builds the +targets with instrumentation, runs the tests, and post-processes the results. +Two hooks matter for us: `--coverage_output_generator` (a tool that processes +each test's raw coverage output) and `--coverage_report_generator` (a tool +that combines everything into the final report). **This pipeline replaces +both with its own tools** — that is its core. + +### 1.3 Rust in two paragraphs + +Rust code is organized into **crates** (≈ libraries/binaries). The Rust +compiler is `rustc`; S-CORE uses **Ferrocene**, a safety-certified Rust +toolchain distribution. Bazel builds Rust via the `rules_rust` plugin. + +Crucially, `rustc` is built on the same compiler backend as Clang (**LLVM**). +That means Rust and C++ can use the *same* coverage instrumentation format — +which is what makes a single unified report possible. + +### 1.4 The LLVM coverage toolchain + +The pipeline is built on LLVM's "source-based coverage": + +| Artifact | Produced by | Contains | +|---|---|---| +| instrumented binary | Clang (C++) / rustc (Rust) with coverage flags | counters + a "covmap" mapping counters → source lines | +| `.profraw` | running the instrumented test | raw counter values | +| `.profdata` | `llvm-profdata merge` | merged, indexed counters | +| HTML / LCOV / text report | `llvm-cov show/export/report` | human- and machine-readable coverage | + +--- + +## 2. How the pipeline works + +There are two phases. + +### 2.1 Phase 1 — collection (`bazel coverage`) + +``` +bazel coverage --config=llvm_cov //... --build_tests_only +``` + +The `llvm_cov` config (the consumer copies it from +[integration_tests/.bazelrc](integration_tests/.bazelrc)) does four things: + +1. **Swaps the compilers.** C++ is compiled with a hermetic Clang/LLVM + toolchain (`@llvm_toolchain`) instead of GCC; Rust with a Ferrocene + toolchain that has LLVM coverage tools attached — wired in automatically + by score_toolchains_rust >= 0.9.2 from the coverage-tools tarball, see §5. + Both emit the same covmap format. +2. **Turns on instrumentation.** `--experimental_use_llvm_covmap` plus the + `coverage` feature for C++; `rules_rust` adds `-Cinstrument-coverage` to + rustc automatically once the toolchain declares coverage tools. An extra + flag (`-Cllvm-args=-runtime-counter-relocation`) enables "continuous + mode" so coverage survives even if a test terminates abnormally (same + purpose as the `enable_llvm_coverage_for_death_tests` cc_feature on the + C++ side). **Branch coverage** needs one more flag per language: Clang + emits branch regions by default, but rustc only does so with + `-Zcoverage-options=branch` — an unstable option that works on + *rolling* (nightly-based) Ferrocene builds. On a stable-channel + Ferrocene the flag must be dropped (Rust branch columns revert to `-`) + until rustc stabilizes it. +3. **Installs the per-test tool** + (`--coverage_output_generator=@score_tooling//coverage:merger`). After + each test runs, `merger.py` finds the test's `.profraw` files, merges + them into one `.profdata` with `llvm-profdata`, records which + instrumented binary was involved, and zips both up as the test's + `coverage.dat`. +4. **Installs the final tool** (`--coverage_report_generator=` the + consumer's `score_coverage_reporter` target). After all tests finish, + `reporter.py` merges every per-test `.profdata` into one, then runs + `llvm-cov` three times: `show` → HTML report, `export` → LCOV data (for + dashboards), `report` → text summary. All three are zipped into + `bazel-out/_coverage/_coverage_report.dat`. + +**Scope — which files appear in the report.** LLVM covmap instruments +*everything*, including test code and third-party libraries. Filtering +happens at report time using an **allowlist** generated by the consumer's +`score_coverage_scope` target (`coverage_scope.bzl`): an aspect walks the +dependency graph starting from the listed production targets and collects +every in-workspace source file they own. The reporter excludes everything +else (test sources, googletest, external deps, ...). + +**Baseline — files with no tests at all.** A file that no test executes +produces no coverage data, so naive tooling silently omits it — it looks +like there is no problem when in fact coverage is 0%. The scope aspect +therefore also collects the compiled libraries/binaries (`.a` archives for +cc_library/rust_library, the coverage-built executable for rust_binary), and +the reporter runs `llvm-cov --empty-profile` over them so untested files +show up with **exact** 0% line and branch entries — the denominators come +from the compiler's own coverage map, not from any source-text heuristic. +Rust rlib archives need special handling here: their leading `lib.rmeta` +member makes llvm-cov reject the whole archive, so the reporter expands them +into their `.o` members first. + +### 2.2 Phase 2 — report generation & gating + +``` +bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml tools/coverage/coverage_justifications.yaml +``` + +`generate_coverage_html.sh` unpacks the HTML from the zip, then applies the +**justification system**: + +- `justify.py` reads the consumer's `coverage_justifications.yaml` plus + in-code markers and produces a manifest of "argued" lines. A justification + says: *this line cannot reasonably be covered by a test, and here is why* + (e.g. defensive code for conditions that cannot occur). Markers work in + both languages: + + ```rust + unreachable!(); // COV_JUSTIFIED my-justification-id + ``` + ```cpp + default: return Error; // COV_JUSTIFIED my-justification-id + ``` + + Every marker id must exist in the YAML with a category and a written + reason — a justification is a reviewed engineering argument, not an + opt-out. + +- `effective_coverage.py` recolors justified lines **orange** in the HTML + (with the reason as tooltip) and computes: + + ``` + raw coverage = covered / total + effective coverage = (covered + justified) / total + ``` + + It also flags **stale** justifications — lines that are justified but + meanwhile covered by a test — so the database stays clean. + +- Finally the script compares effective line coverage against the + `COVERAGE_THRESHOLD` environment variable (default **100**) and **fails + (exit 1)** when below. The model: every uncovered line must eventually be + either tested or justified; the threshold is ratcheted up as gaps close. + +### 2.3 Day-to-day commands + +```bash +# collect coverage (Rust + C++, one run) +bazel coverage --config=llvm_cov //... --build_tests_only + +# report without gating +COVERAGE_THRESHOLD=0 bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml tools/coverage/coverage_justifications.yaml + +# open it +xdg-open coverage_linux/index.html + +# CI-style archive (HTML + LCOV + justification report + JUnit XMLs) +bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml tools/coverage/coverage_justifications.yaml --archive my-report +``` + +> **Do not** combine `--config=llvm_cov` with configs that register other +> C++ toolchains (e.g. a GCC host config): the last `--extra_toolchains` +> wins resolution, GCC cannot produce covmap data, and the report script +> fails loudly on the resulting non-zip report. + +--- + +## 3. The module/consumer split + +Almost everything in the pipeline is generic. What is repo-specific is +exactly three things: **(a)** the list of production targets in the scope, +**(b)** the justification YAML, **(c)** the toolchain pins in MODULE.bazel. +The split follows directly: + +**Lives in `@score_tooling//coverage` (shared):** + +| File | Role | +|---|---| +| `merger.py` | per-test profraw → profdata (C++ `objects_list.txt` and Rust ELF-manifest discovery) | +| `reporter.py` | final merge + llvm-cov show/export/report + allowlist filtering + `--empty-profile` baselines + rlib expansion | +| `coverage_scope.bzl` | the scope aspect/rule (CcInfo + CrateInfo) | +| `reporter_wrapper.bzl` + `defs.bzl` | the consumer-facing `score_coverage_scope` / `score_coverage_reporter` API | +| `justify.py`, `effective_coverage.py`, `generate_coverage_html.sh` | justification + gating layer | +| `enable_llvm_coverage_for_death_tests` | cc_feature for continuous-mode profiling | + +**Lives in the consumer repository:** + +| Piece | Why it cannot move | +|---|---| +| `score_coverage_scope(deps = [...])` | names the repo's production targets | +| `score_coverage_reporter(...)` | carries the repo's LLVM tool labels and workspace root | +| `coverage_justifications.yaml` | reviewed, repo-specific engineering arguments | +| MODULE.bazel toolchain blocks | LLVM + Ferrocene pins are per-repo decisions | +| the `coverage:llvm_cov` bazelrc block | bazelrc cannot be imported across modules; copied from the canonical snippet | + +Two wiring details make the external hosting work, both easy to get wrong: + +- **Runfiles paths span repositories.** The reporter wrapper mixes files + from `_main` (the consumer), `score_tooling` and toolchain repos, so every + path in the generated launcher uses rlocation form (`../repo/...` → + `repo/...`), and the launcher derives its own `RUNFILES_DIR` from `$0` — + the inherited value points at the *test's* runfiles tree, not ours. +- **The baseline manifest lists consumer files.** The reporter resolves + manifest entries against `_main` explicitly; using the runfiles library's + "current repository" would resolve against `score_tooling` and find + nothing. + +--- + +## 4. Non-obvious pitfalls (why these lines exist) + +These are the "landmines" discovered while bringing the pipeline up in +communication and persistency: + +1. **Warnings-as-errors under Clang.** Repos whose deps request the + `treat_warnings_as_errors` feature may need + `--features=-treat_warnings_as_errors` **and** + `--host_features=-treat_warnings_as_errors` in the coverage config: + Clang emits warnings GCC doesn't, and Bazel builds the coverage reporter + (and hence the scope's libraries) a second time "as a tool" in a separate + configuration — that's what the `--host_features` variant covers. +2. **Never disable the `coverage` feature** under the LLVM toolchain — it + *is* the instrumentation (`-fprofile-instr-generate + -fcoverage-mapping`). +3. **`-Cllvm-args=-runtime-counter-relocation`** for Rust — without it, + continuous-mode profiling errors out and Rust tests write no `.profraw`. +4. **`llvm-cov report` prints raw covmap paths** (`/proc/self/cwd/...`) — + `--path-equivalence` does not rewrite *displayed* paths; the reporter + normalizes them, otherwise the allowlist silently excludes all C++ + files. +5. **The Rust toolchain lists its own `llvm-cov`/`llvm-profdata` binaries as + coverage metadata** — the merger must skip `external/` entries or the + final merge emits "mismatched data" warnings. +6. **Rust branch coverage is opt-in and channel-dependent** — llvm-cov only + renders branch data that the compiler wrote into the covmap; stable rustc + writes none. `-Zcoverage-options=branch` enables it on nightly-based + toolchains (like the Ferrocene rolling build). Verify with + `llvm-cov export`: the `branches` arrays must be non-empty for Rust + files. + +--- + +## 5. Toolchain provisioning (score_toolchains_rust + ferrocene_toolchain_builder) + +Solved at the source, no consumer configuration needed: + +- `ferrocene_toolchain_builder` >= **1.3.1** ships `llvm-cov`, + `llvm-profdata` and `llvm-cxxfilt` in the coverage-tools tarball, built + from the same LLVM tree as rustc (so the tools can always read the + profraw/covmap the compiler emits). 1.3.1 also rebuilds ALL artifacts from + a single tree — toolchain tarballs, miri-sysroots (now including + `libprofiler_builtins`) and coverage tools are ABI-consistent. +- `score_toolchains_rust` >= **0.9.2** (current: 0.10.0) auto-wires the + tools into the generated `rust_toolchain` whenever the coverage-tools + tarball contains them. rules_rust then instruments crates under + `bazel coverage` and exports `RUST_LLVM_COV`/`RUST_LLVM_PROFDATA` to the + coverage runner. + +Consequently consumers need no coverage-specific Rust toolchain at all: the +**standard** toolchains declared in score_toolchains_rust's own MODULE.bazel +(e.g. `@score_toolchains_rust//toolchains/ferrocene:ferrocene_x86_64_unknown_linux_gnu`) +already pin the coverage-tools tarball, so the toolchain registered for +regular builds is the one that produces coverage — same rustc, same LLVM. + +--- + +## 6. Origins and validation evidence + +The pipeline core was built in the `communication` repository +(eclipse-score/communication, Rust support merged with PR #772 and visible +in its nightly coverage reports), then ported to `persistency` and finally +centralized here. During the persistency port, the previous Rust mechanism +(Ferrocene `symbol-report` + `blanket`, per-target 90% gate — the flow this +module's removed `rust_coverage_report` rule drove) was run against the new +pipeline **on the same commit**. Percentages agreed within a few points on +most files (the two tools attribute lines differently: blanket is +symbol-oriented, llvm-cov counts every executable region), with one headline +difference: + +> A 311-line, 0%-covered Rust binary (`kvs_tool.rs`, no tests) was +> **invisible** to the old pipeline while its 90% gate passed. The new +> baseline mechanism reports it at exact 0% and the effective-coverage gate +> accounts for it. + +That gap — untested files silently missing from reports — is the main +correctness argument for this pipeline, alongside unified C++ + Rust +reporting and branch coverage for Rust. + +Planned next step for the ecosystem: a reusable GitHub Actions workflow in +`eclipse-score/cicd-workflows` wrapping the collection + report + artifact +steps, so consumer repos add one `uses:` block instead of a hand-written +job. diff --git a/coverage/README.md b/coverage/README.md index 2b7fcff9..0de94968 100644 --- a/coverage/README.md +++ b/coverage/README.md @@ -11,9 +11,233 @@ SPDX-License-Identifier: Apache-2.0 ----------------------------------------------------------------------------- --> -# Coverage +# Coverage — adoption guide + +Reusable **LLVM source-based coverage pipeline** for S-CORE repositories: + +- **One report for C++ and Rust** (line + branch coverage), produced by + `llvm-cov` directly from covmap instrumentation — no gcov/genhtml. +- **Untested in-scope files appear at exact 0%** — since all targets are + instrumented at build time, the reporter runs `llvm-cov --empty-profile` + over the archives of libraries no test links against. No heuristics; the + line/branch denominators come from the compiler's own coverage map. +- **Justification system**: `COV_JUSTIFIED` in-code markers + a YAML database + turn intentionally-uncovered lines into *justified* lines, tracked in an + **effective coverage** metric with stale-justification detection. +- **Gating**: the report generator exits non-zero when effective coverage is + below `COVERAGE_THRESHOLD` (default 100). + +How the pipeline works internally is documented in +[COVERAGE_GUIDE.md](COVERAGE_GUIDE.md). A complete, working consumer setup is +the [integration_tests/](integration_tests/) workspace — every snippet below +is copied from it. + +## Components + +| Target / file | Purpose | +|---|---| +| `@score_tooling//coverage:merger` | Per-test coverage output generator (profraw → profdata + object metadata). Referenced directly from your bazelrc. | +| `@score_tooling//coverage:reporter` | Final report generator (merged profdata → HTML + LCOV + text summary). Not referenced directly — wrapped by `score_coverage_reporter`. | +| `defs.bzl :: score_coverage_scope` | Declares WHICH targets are in scope; emits the source allowlist + baseline-archive manifest via an aspect. | +| `defs.bzl :: score_coverage_reporter` | Consumer-side wrapper wiring your scope, workspace root and LLVM tools into the reporter. | +| `@score_tooling//coverage:generate_coverage_html` | Orchestration: unpacks the report, runs justifications, enforces the threshold, optionally archives. | +| `@score_tooling//coverage:justify` | Parses the justification YAML + in-code markers into a manifest. | +| `@score_tooling//coverage:effective_coverage` | Post-processes the HTML: restyles justified lines, computes effective coverage, detects stale justifications. | +| `@score_tooling//coverage:enable_llvm_coverage_for_death_tests` | `cc_feature` adding `-mllvm -runtime-counter-relocation` (continuous-mode profiling for death tests). | + +## Prerequisites + +1. A Bzlmod workspace (`MODULE.bazel`). +2. Linux x86_64 host (the pipeline runs on the host platform; do not combine + with QNX/cross platform configs). +3. For Rust: a Ferrocene toolchain built by `ferrocene_toolchain_builder` + **>= 1.3.1** (its coverage-tools tarball ships `llvm-cov`/`llvm-profdata` + built from the same LLVM as rustc) wired through `score_toolchains_rust` + **>= 0.10.0**. + +## 1. Depend on score_tooling -## Combined Rust + Python Coverage +```starlark +bazel_dep(name = "score_tooling", version = "") +``` + +Add one line to your **root** `BUILD` file so the reporter can locate your +workspace root at runtime: + +```starlark +exports_files(["MODULE.bazel"]) +``` + +## 2. Declare the coverage toolchains (MODULE.bazel) + +```starlark +bazel_dep(name = "score_toolchains_rust", version = "0.10.0", dev_dependency = True) +bazel_dep(name = "toolchains_llvm", version = "1.8.0", dev_dependency = True) + +llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm", dev_dependency = True) +llvm.toolchain( + cxx_standard = {"": "c++17"}, + extra_known_features = [ + "@score_tooling//coverage:enable_llvm_coverage_for_death_tests", + ], + llvm_version = "22.1.7", + stdlib = {"": "stdc++"}, +) +use_repo(llvm, "llvm_toolchain", "llvm_toolchain_llvm") +``` + +For Rust, **no coverage-specific toolchain is needed**: the standard +toolchains shipped by score_toolchains_rust >= 0.10.0 already attach +`llvm-cov`/`llvm-profdata` (from the ferrocene_toolchain_builder >= 1.3.1 +coverage-tools tarball, built from the same LLVM as rustc). Just register the +standard toolchain as usual: + +``` +common --extra_toolchains=@score_toolchains_rust//toolchains/ferrocene:ferrocene_x86_64_unknown_linux_gnu +``` + +rules_rust only instruments crates when the `rust_toolchain` declares +`llvm_cov` — a Ferrocene toolchain from an older score_toolchains_rust (or a +custom instance without `coverage_tools_url`) silently produces no Rust +coverage. + +## 3. Declare scope and reporter (BUILD) + +In e.g. `tools/coverage/BUILD`: + +```starlark +load("@score_tooling//coverage:defs.bzl", "score_coverage_reporter", "score_coverage_scope") + +score_coverage_scope( + name = "coverage_scope", + testonly = True, + deps = [ + "//src/mylib", # cc_library + "//src/rust/mycrate", # rust_library + "//src/rust/tool:tool", # rust_binary + ], +) + +score_coverage_reporter( + name = "reporter_wrapper", + testonly = True, + coverage_scope = ":coverage_scope", + llvm_cov = "@llvm_toolchain//:llvm-cov", + llvm_profdata = "@llvm_toolchain//:llvm-profdata", + llvm_cxxfilt = "@llvm_toolchain_llvm//:bin/llvm-cxxfilt", +) +``` + +The scope aspect walks the listed targets and their transitive in-workspace +deps, collecting source files (allowlist) and compiled archives (baselines). +Everything in scope but untested shows up at 0%; everything outside the scope +(tests, mocks, external deps) is filtered out of the report. + +## 4. Import the bazelrc config + +Copy the `coverage:llvm_cov` block from +[integration_tests/.bazelrc](integration_tests/.bazelrc) into your +repository's bazelrc (directly or via `import`). The two labels to adapt: + +``` +coverage:llvm_cov --coverage_output_generator=@score_tooling//coverage:merger +coverage:llvm_cov --coverage_report_generator=//tools/coverage:reporter_wrapper +``` + +The merger reference points into score_tooling as-is; the reporter_wrapper +label is the target you declared in step 3. + +> **Do NOT combine `--config=llvm_cov` with configs that append other +> `--extra_toolchains`** (e.g. a GCC host config): the last toolchain wins +> resolution and a GCC toolchain produces no covmap data. + +## 5. (Optional) Set up justifications + +`tools/coverage/coverage_justifications.yaml`: + +```yaml +version: 1 +justifications: + - id: hw-unreachable-on-x86 + category: platform_specific + platforms: [linux] + reason: | + ARM-only error path; cannot be exercised by x86 CI. +``` + +Mark the code in place: + +```cpp +return false; // COV_JUSTIFIED hw-unreachable-on-x86 + +// or a region: +// COV_JUSTIFIED_START hw-unreachable-on-x86 +if (running_on_arm()) { ... } +// COV_JUSTIFIED_STOP +``` + +Valid categories: `defensive_programming`, `tool_false_positive`, +`platform_specific`, `other`. IDs are kebab-case. Justified lines render +orange in the HTML and count as covered in the *effective* metric; a +justification on a line that is meanwhile covered is flagged as **stale**. + +## 6. Run it + +```bash +bazel coverage --config=llvm_cov //... --build_tests_only + +bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml tools/coverage/coverage_justifications.yaml + +# CI variant: archive HTML + LCOV + JUnit XMLs, gate at 95%: +COVERAGE_THRESHOLD=95 bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml tools/coverage/coverage_justifications.yaml \ + --archive coverage_artifacts +``` + +`--build_tests_only` matters: without it, coverage builds (not runs) every +target matched by the pattern, including e.g. `manual`-tagged or +platform-incompatible test binaries. + +## Customization knobs + +| Need | Knob | +|---|---| +| Different gate | `COVERAGE_THRESHOLD=` env var (default 100; exit 1 below) | +| Output directory | positional `output-dir` argument (default `coverage_`) | +| Platform-specific justifications | `--platform linux\|qnx` (default linux) | +| JUnit XMLs subtree in the archive | `--testlogs-subdir ` (default: whole `bazel-testlogs`) | +| Different LLVM version | your own `llvm.toolchain(...)`; pass its labels in step 3 | +| Rust branch coverage | `-Zcoverage-options=branch` (needs a nightly-based/rolling Ferrocene; drop the flag on stable) | + +## Troubleshooting + +| Symptom | Cause | +|---|---| +| `... is not the LLVM pipeline zip report` | The coverage run used the default lcov path — the `--config=llvm_cov` flags (or your bazelrc import) were not active. | +| No `.rs` files in the report | The Ferrocene toolchain in use has no `llvm_cov` attached (missing `coverage_tools_url`, or a non-coverage toolchain instance won resolution). | +| No C++ files / empty covmap | A GCC toolchain won toolchain resolution — check for conflicting `--extra_toolchains` from another config. | +| `Neither __llvm_profile_counter_bias nor ...` in test logs, no profraw | Continuous mode without runtime counter relocation: the `enable_llvm_coverage_for_death_tests` feature (C++) or the `-Cllvm-args=-runtime-counter-relocation` rustc flag is missing. | +| `no coverage data found` on a Rust archive | Handled automatically (rlib expansion); if you see it, the reporter predates the rlib fix. | +| `error[E0463]: can't find crate for profiler_builtins` | The Ferrocene sysroot lacks profiler_builtins (builder < 1.3.1, or a miri sysroot leaked into coverage builds). | +| `the following arguments are required: --workspace_root` | You pointed `--coverage_report_generator` at `:reporter` directly instead of your `score_coverage_reporter` target. | +| Coverage numbers differ between runs on identical code | Dynamic-linking instrumentation clash — ensure `--dynamic_mode=off` from the bazelrc block is active. | + +## Migration from the removed Ferrocene symbol-report/blanket flow + +`rust_coverage_report`, `//coverage:ferrocene_report` and its helper scripts +were removed. Replace: + +- `bazel run //:rust_coverage` → steps 1–6 above (one report for both + languages, exact untested-file entries, justifications, effective gate). +- `test:ferrocene-coverage --run_under=@score_tooling//coverage:llvm_profile_wrapper` + is no longer needed — Bazel's own coverage collection sets + `LLVM_PROFILE_FILE`. The wrapper target still exists for repositories that + have not migrated yet. + +--- + +## Repository-internal: Combined Rust + Python Coverage The `//coverage:combined_report` target generates a single HTML coverage report for all Rust and Python tools in the repository using Bazel's built-in @@ -78,188 +302,3 @@ genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" \ --output-directory coverage-html/ ``` ---- - -## Ferrocene Rust Coverage - -This directory provides the Ferrocene Rust coverage workflow for Bazel-based -projects. It uses Ferrocene's `symbol-report` and `blanket` tools to generate -HTML coverage reports from `.profraw` files produced by Rust tests. - -The workflow is intentionally split: -- Tests produce `.profraw` files (can run on host or target hardware). -- Reports are generated later on a host machine. - -This makes it easy to collect coverage from cross-compiled tests or from -hardware-in-the-loop runs. - -## Quick Start (Developers) - -1) Run tests with coverage enabled: - -```bash -bazel test --config=ferrocene-x86_64-linux --config=ferrocene-coverage \ - --nocache_test_results \ - //path/to:rust_tests -``` - -2) Generate coverage reports: - -```bash -bazel run //:rust_coverage -- --min-line-coverage 80 -``` - -The default report directory is: - -``` -$(bazel info bazel-bin)/coverage/rust-tests//blanket/index.html -``` - -The script prints per-target line coverage plus an overall summary line. - -## Integrator Setup - -### 1) MODULE.bazel - -Add `score_tooling` and `score_toolchains_rust` as dependencies: - -```starlark -bazel_dep(name = "score_tooling", version = "1.0.0") -bazel_dep(name = "score_toolchains_rust", version = "0.4.0") -``` - -### 2) .bazelrc - -Add a Ferrocene coverage config. Names are examples; choose names that fit -your repo: - -``` -# Ferrocene toolchain for host execution -build:ferrocene-x86_64-linux --host_platform=@score_bazel_platforms//:x86_64-linux -build:ferrocene-x86_64-linux --platforms=@score_bazel_platforms//:x86_64-linux -build:ferrocene-x86_64-linux --extra_toolchains=@score_toolchains_rust//toolchains/ferrocene:ferrocene_x86_64_unknown_linux_gnu - -# Coverage flags for rustc -build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Cinstrument-coverage -build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Clink-dead-code -build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Ccodegen-units=1 -build:ferrocene-coverage --@rules_rust//rust/settings:extra_rustc_flag=-Cdebuginfo=2 -build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cinstrument-coverage -build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Clink-dead-code -build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Ccodegen-units=1 -build:ferrocene-coverage --@rules_rust//rust/settings:extra_exec_rustc_flag=-Cdebuginfo=2 -test:ferrocene-coverage --run_under=@score_tooling//coverage:llvm_profile_wrapper -``` - -### 3) Add a repo-local wrapper target - -In a root `BUILD` file: - -```starlark -load("@score_tooling//coverage:coverage.bzl", "rust_coverage_report") - -rust_coverage_report( - name = "rust_coverage", - bazel_configs = [ - "ferrocene-x86_64-linux", - "ferrocene-coverage", - ], - query = 'kind("rust_test", //...)', - min_line_coverage = "80", -) -``` - -Run it with: - -```bash -bazel run //:rust_coverage -``` - -### 4) Optional: exclude known-problematic targets - -```starlark -query = 'kind("rust_test", //...) except //path/to:tests', -``` - -## Cross/Target Execution - -If tests run on target hardware, copy the `.profraw` files back to the host -and point the report generator to the directory: - -```bash -bazel run //:rust_coverage -- --profraw-dir /path/to/profraw -``` - -## Running from an integration workspace (external labels) - -You can invoke the report generator from a top-level integration repo (for -example, reference_integration) while targeting tests that live in external -modules. Use a query that references external labels and run the wrapper -target from the integration repo: - -```bash -bazel run //images/linux_x86_64:per_rust_coverage --config=ferrocene-coverage -- \ - --query 'kind("rust_test", @score_persistency//src/rust/...)' -``` - -If the `.profraw` files were produced in that same workspace, the reporter -auto-discovers them under `bazel-testlogs/` (including -`bazel-testlogs/external/+` for external labels), so you do not need -to pass `--profraw-dir`. If they were copied from elsewhere, pass -`--profraw-dir` to point to the directory containing the `.profraw` files. -External source paths are resolved via Bazel's output_base so -`external//...` paths are handled. - -## Coverage Gate Behavior - -`--min-line-coverage` applies per target. If any target is below the minimum, -the script exits non-zero so CI can fail the job. An overall summary is printed -for visibility but does not change gating behavior. - -## Common Pitfalls - -- **"running 0 tests"**: The Rust test harness found no `#[test]` functions, - so coverage is 0%. Add tests or exclude the target from the query. -- **"couldn't find source file"** warnings: Usually path remapping or crate - mapping issues. Check that `crate` attributes in `rust_test` targets point to - the library crate (or exclude the target). -- **Cached test results**: Use `--nocache_test_results` if you need to re-run - tests and regenerate `.profraw` files. - -## Troubleshooting - -### Coverage is 0% but tests ran -- Verify the target contains real `#[test]` functions. A rust_test target with - no tests will run but report 0% coverage. -- Ensure you ran tests with `--config=ferrocene-coverage` so `.profraw` files - exist. -- If the test binary is cached, use `--nocache_test_results`. - -### "couldn't find source file" warnings -- Check `crate` mapping on `rust_test` targets. If `crate = "name"` is used, - ensure it refers to the library crate in the same package. -- Confirm the reported paths exist in the workspace. Path remapping is required - so `blanket` can resolve files under `--ferrocene-src`. - -### No `.profraw` files found -- Ensure `test:ferrocene-coverage` sets `--run_under=@score_tooling//coverage:llvm_profile_wrapper`. -- Re-run tests with `--nocache_test_results`. -- If tests ran on target hardware, copy the `.profraw` files back and pass - `--profraw-dir`. - -### Coverage gate fails in CI -- The gate is per-target. A single target below the threshold fails the job. -- Use a stricter query (exclude known-zero targets) or add tests. - -## CI Integration (Suggested Pattern) - -Keep coverage generation separate from docs: - -1) Coverage workflow: - - run `bazel run //:rust_coverage` - - upload `bazel-bin/coverage/rust-tests` as an artifact - -2) Docs workflow: - - download the artifact - - copy into the docs output (e.g. `docs/_static/coverage/`) - - publish Sphinx docs to GitHub Pages diff --git a/coverage/coverage.bzl b/coverage/coverage.bzl deleted file mode 100644 index 4b7b3924..00000000 --- a/coverage/coverage.bzl +++ /dev/null @@ -1,84 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -"""Bazel helpers for Ferrocene Rust coverage workflows.""" - -def _shell_quote(value): - if value == "": - return "''" - return "'" + value.replace("'", "'\"'\"'") + "'" - -def _rust_coverage_report_impl(ctx): - script = ctx.actions.declare_file(ctx.label.name + ".sh") - - args = [] - for cfg in ctx.attr.bazel_configs: - if cfg: - args.extend(["--bazel-config", cfg]) - if ctx.attr.query: - args.extend(["--query", ctx.attr.query]) - if ctx.attr.min_line_coverage: - args.extend(["--min-line-coverage", ctx.attr.min_line_coverage]) - - args_parts = [_shell_quote(a) for a in args] - - # The wrapper script forwards preconfigured args and any extra CLI args. - exec_line = "exec \"${ferrocene_report}\"" - if args_parts: - exec_line += " \\\n " + " \\\n ".join(args_parts) - exec_line += " \\\n \"$@\"" - - # Resolve the report script via runfiles for remote/CI compatibility. - runfile_path = ctx.executable._ferrocene_report.short_path - ctx.actions.expand_template( - template = ctx.file._wrapper_template, - output = script, - substitutions = { - "@@RUNFILE@@": _shell_quote(runfile_path), - "@@EXEC_LINE@@": exec_line, - }, - is_executable = True, - ) - - report_runfiles = ctx.attr._ferrocene_report[DefaultInfo].default_runfiles - runfiles = ctx.runfiles(files = [ctx.executable._ferrocene_report]).merge(report_runfiles) - return [DefaultInfo(executable = script, runfiles = runfiles)] - -rust_coverage_report = rule( - implementation = _rust_coverage_report_impl, - executable = True, - attrs = { - "bazel_configs": attr.string_list( - default = ["ferrocene-coverage"], - doc = "Bazel configs passed to ferrocene_report.", - ), - "query": attr.string( - default = 'kind("rust_test", //...)', - doc = "Bazel query used to discover rust_test targets.", - ), - "min_line_coverage": attr.string( - default = "", - doc = "Optional minimum line coverage percentage.", - ), - "_ferrocene_report": attr.label( - default = Label("//coverage:ferrocene_report"), - executable = True, - cfg = "exec", - ), - "_wrapper_template": attr.label( - default = Label("//coverage:ferrocene_report_wrapper.sh.tpl"), - allow_single_file = True, - ), - }, - doc = "Creates a repo-local wrapper for Ferrocene Rust coverage reports.", -) diff --git a/coverage/coverage_scope.bzl b/coverage/coverage_scope.bzl new file mode 100644 index 00000000..acb722b2 --- /dev/null +++ b/coverage/coverage_scope.bzl @@ -0,0 +1,202 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +""" +Coverage scope rule for deriving file-level allowlists from implementation targets. + +This rule uses an aspect to traverse the build graph starting from the listed +implementation targets (cc_library, rust_library, rust_binary) through their +transitive deps. At each node it collects the actual source files. + +- cc_library (and rust_library, which also provides CcInfo in rules_rust + 0.68.x): source files come from the srcs/hdrs attributes; static archives + (including the rlib exposed as a .a symlink) are collected for baseline + (zero-coverage) reporting. +- rust_binary (no CcInfo): source files come from CrateInfo.srcs; the + coverage-built executable itself serves as the baseline object. + +The resulting allowlist contains one source file path per line (relative to the +workspace root). The coverage reporter uses this to restrict reports to exactly +the files that are part of the covered implementation. +""" + +load("@rules_rust//rust:rust_common.bzl", "CrateInfo") + +# ============================================================================= +# Provider to carry collected source file paths through the aspect +# ============================================================================= + +_CoverageScopeInfo = provider( + doc = "Carries source file paths and object files collected by the coverage scope aspect.", + fields = { + "source_files": "Depset of source file path strings (workspace-relative).", + "object_files": "Depset of compiled archive/executable File objects for baseline coverage.", + }, +) + +# ============================================================================= +# Aspect: traverses library/binary deps to collect files +# ============================================================================= + +def _coverage_scope_aspect_impl(target, ctx): + """Collects source file paths and archive files from the build graph.""" + direct_files = [] + direct_archives = [] + transitive = [] + transitive_archives = [] + + # At cc_library / rust_library targets (rust_library provides CcInfo with + # its rlib exposed as a .a symlink): collect srcs, hdrs, and static archive + if CcInfo in target: + for attr_name in ["srcs", "hdrs"]: + if hasattr(ctx.rule.attr, attr_name): + for src in getattr(ctx.rule.attr, attr_name): + for f in src.files.to_list(): + if not f.path.startswith("external/") and f.is_source: + direct_files.append(f.short_path) + + # Only collect workspace-internal labels and archives + if not str(target.label).startswith("@@") or str(target.label).startswith("@@//"): + # Collect .a archive files for baseline coverage. + for linker_input in target[CcInfo].linking_context.linker_inputs.to_list(): + for lib in linker_input.libraries: + for archive in [lib.static_library, lib.pic_static_library]: + if archive and "/external/" not in archive.path and not archive.path.startswith("external/"): + direct_archives.append(archive) + break + elif CrateInfo in target: + # rust_binary: no CcInfo, collect .rs sources from CrateInfo and use the + # coverage-built executable as the baseline object. + for f in target[CrateInfo].srcs.to_list(): + if not f.path.startswith("external/") and f.is_source: + direct_files.append(f.short_path) + out = target[CrateInfo].output + if out and "/external/" not in out.path and not out.path.startswith("external/"): + direct_archives.append(out) + + # Propagate from children traversed by the aspect + for attr_name in ["components", "implementation", "deps", "implementation_deps", "exported_deps"]: + if hasattr(ctx.rule.attr, attr_name): + for dep in getattr(ctx.rule.attr, attr_name): + if _CoverageScopeInfo in dep: + transitive.append(dep[_CoverageScopeInfo].source_files) + transitive_archives.append(dep[_CoverageScopeInfo].object_files) + + return [_CoverageScopeInfo( + source_files = depset(direct_files, transitive = transitive), + object_files = depset(direct_archives, transitive = transitive_archives), + )] + +_coverage_scope_aspect = aspect( + implementation = _coverage_scope_aspect_impl, + attr_aspects = ["components", "implementation", "deps", "implementation_deps", "exported_deps"], + doc = "Traverses cc_library/rust_library/rust_binary hierarchy to collect implementation source files.", +) + +# ============================================================================= +# Rule: aggregates aspect results into an allowlist file +# ============================================================================= + +def _coverage_scope_impl(ctx): + """Aggregates source file paths from all deps and writes allowlist + baseline objects.""" + all_files = {} + all_objects = [] + + for dep in ctx.attr.deps: + if _CoverageScopeInfo in dep: + for path in dep[_CoverageScopeInfo].source_files.to_list(): + if path: + all_files[path] = True + all_objects.append(dep[_CoverageScopeInfo].object_files) + + sorted_files = sorted(all_files.keys()) + object_depset = depset(transitive = all_objects) + + # Write the allowlist file + output = ctx.actions.declare_file(ctx.attr.name + "_allowlist.txt") + ctx.actions.write( + output = output, + content = "\n".join(sorted_files) + "\n" if sorted_files else "", + ) + + # Write archive file paths for baseline coverage (reporter uses these as --object args) + archive_paths = sorted(set([f.short_path for f in object_depset.to_list()])) + objects_output = ctx.actions.declare_file(ctx.attr.name + "_objects.txt") + ctx.actions.write( + output = objects_output, + content = "\n".join(archive_paths) + "\n" if archive_paths else "", + ) + + return [ + DefaultInfo(files = depset([output, objects_output], transitive = [object_depset])), + OutputGroupInfo( + allowlist = depset([output]), + objects = depset([objects_output]), + object_files = object_depset, + ), + ] + +def _coverage_transition_impl(settings, attr): + # This dictionary modifies the build configuration + return { + "//command_line_option:collect_code_coverage": True, + } + +# Define the transition +coverage_transition = transition( + implementation = _coverage_transition_impl, + inputs = [], + outputs = ["//command_line_option:collect_code_coverage"], +) + +def _coverage_wrapper_impl(ctx): + # Forward the executable or providers from the underlying target + actual_target = ctx.attr.actual[0] + return [actual_target[DefaultInfo]] + +# Define a rule that applies the transition to its 'actual' dependency +coverage_wrapper = rule( + implementation = _coverage_wrapper_impl, + attrs = { + "actual": attr.label( + mandatory = True, + cfg = coverage_transition, # Applying the transition here + ), + # Mandatory attribute needed when a rule uses a transition + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +coverage_scope = rule( + implementation = _coverage_scope_impl, + doc = """Generates a file-level coverage allowlist from implementation targets. + + Uses an aspect to traverse the listed targets (cc_library, rust_library, + rust_binary) and their transitive deps, collecting all source files + (srcs + hdrs / CrateInfo.srcs). Outputs a text file with one + workspace-relative file path per line. + + This allowlist is consumed by the coverage reporter to restrict coverage + reporting to exactly the source files that are part of the implementation. + """, + attrs = { + "deps": attr.label_list( + mandatory = True, + aspects = [_coverage_scope_aspect], + cfg = coverage_transition, + doc = "Implementation targets whose transitive deps define the coverage scope.", + ), + }, +) diff --git a/coverage/defs.bzl b/coverage/defs.bzl new file mode 100644 index 00000000..cef30119 --- /dev/null +++ b/coverage/defs.bzl @@ -0,0 +1,90 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Public API of the score_tooling LLVM source-based coverage pipeline. + +Consumers instantiate two targets in their own repository (typically in a +tools/coverage/BUILD or quality/coverage/BUILD file): + + load("@score_tooling//coverage:defs.bzl", + "score_coverage_reporter", "score_coverage_scope") + + score_coverage_scope( + name = "coverage_scope", + testonly = True, + deps = ["//src/mylib", "//src/rust/mycrate"], + ) + + score_coverage_reporter( + name = "reporter_wrapper", + testonly = True, + coverage_scope = ":coverage_scope", + llvm_cov = "@llvm_toolchain//:llvm-cov", + llvm_profdata = "@llvm_toolchain//:llvm-profdata", + llvm_cxxfilt = "@llvm_toolchain_llvm//:bin/llvm-cxxfilt", + ) + +and point Bazel at them from their coverage bazelrc config: + + coverage:llvm_cov --coverage_output_generator=@score_tooling//coverage:merger + coverage:llvm_cov --coverage_report_generator=//tools/coverage:reporter_wrapper + +See coverage/README.md for the complete adoption guide (toolchains, bazelrc, +justifications, CI) and coverage/COVERAGE_GUIDE.md for how the pipeline works. +""" + +load("//coverage:coverage_scope.bzl", _coverage_scope = "coverage_scope") +load("//coverage:reporter_wrapper.bzl", _reporter_wrapper = "reporter_wrapper") + +score_coverage_scope = _coverage_scope + +def score_coverage_reporter( + name, + coverage_scope, + llvm_cov, + llvm_profdata, + llvm_cxxfilt = None, + module_bazel = "//:MODULE.bazel", + **kwargs): + """Declare the consumer-side coverage report generator. + + The generated executable is passed to Bazel as + --coverage_report_generator=//:. It wires the consumer's + coverage scope, workspace root and LLVM tools into score_tooling's + reporter. + + Args: + name: Target name, referenced by --coverage_report_generator. + coverage_scope: A score_coverage_scope target listing the production + targets that define the coverage scope. + llvm_cov: Label of the llvm-cov binary (the consumer's LLVM toolchain, + e.g. "@llvm_toolchain//:llvm-cov"). Must come from the same LLVM + major version that produced the coverage instrumentation. + llvm_profdata: Label of the llvm-profdata binary. + llvm_cxxfilt: Optional label of llvm-cxxfilt for symbol demangling + (C++ Itanium and Rust v0/legacy). toolchains_llvm exposes it as + "@llvm_toolchain_llvm//:bin/llvm-cxxfilt". + module_bazel: The consumer's root MODULE.bazel, used at runtime to + locate the real workspace root. Requires + exports_files(["MODULE.bazel"]) in the consumer's root BUILD file. + **kwargs: Common rule attributes (testonly, visibility, tags, ...). + """ + _reporter_wrapper( + name = name, + coverage_scope = coverage_scope, + module_bazel = module_bazel, + llvm_cov = llvm_cov, + llvm_profdata = llvm_profdata, + llvm_cxxfilt = llvm_cxxfilt, + **kwargs + ) diff --git a/coverage/effective_coverage.py b/coverage/effective_coverage.py new file mode 100644 index 00000000..12679393 --- /dev/null +++ b/coverage/effective_coverage.py @@ -0,0 +1,1180 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Effective coverage calculator and HTML post-processor. + +Takes an HTML coverage report (llvm-cov or gcovr) and the resolved justification +manifest. Modifies the HTML to show justified lines in a distinct color (yellow/orange) +and calculates effective coverage metrics. + +Supports two HTML formats: + - llvm-cov: produced by our custom reporter (Linux) + - gcovr: produced by lcov_to_html.py via gcovr (QNX) + +Usage: + python effective_coverage.py --html-dir --manifest --output [--lcov ] +""" + +import argparse +import json +import math +import os +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Tuple + + +# Pattern to match a table row in llvm-cov HTML source pages +# Format: ......... +LINE_NUMBER_RE = re.compile(r"") +COVERED_LINE_TD_RE = re.compile(r"") + + +def floor_two_decimals(value: float) -> float: + """Floor a percentage value to two decimal places.""" + return math.floor(value * 100.0) / 100.0 + + +def main() -> None: + """Main entry point.""" + args = parse_args() + + # Load the justification manifest + manifest = load_manifest(args.manifest) + justified_files = manifest.get("justified_files", {}) + + # Find all source HTML files in the report + html_dir = args.html_dir + if not html_dir.exists(): + print(f"ERROR: HTML report directory not found: {html_dir}", file=sys.stderr) + sys.exit(1) + + # Detect HTML format: llvm-cov uses style.css, gcovr uses index...html naming. + html_format = detect_html_format(html_dir) + + if html_format == "gcovr": + _main_gcovr(args, html_dir, justified_files) + else: + _main_llvm_cov(args, html_dir, justified_files) + + +def _main_llvm_cov(args: argparse.Namespace, html_dir: Path, justified_files: Dict) -> None: + """Main logic for llvm-cov HTML format.""" + + # Parse raw coverage totals from the index page (matches llvm-cov exactly). + totals = parse_index_page_totals(html_dir) + raw_covered, raw_total = totals["lines"] + raw_branch_covered, raw_branch_total = totals["branches"] + + # Process each source HTML file (restyle justified lines + count them) + total_justified = 0 + total_stale = 0 + total_justified_branches = 0 + applied_justifications: List[Dict[str, Any]] = [] + stale_justifications: List[Dict[str, Any]] = [] + # Track per-file justification counts for index page updates + per_file_stats: Dict[str, Dict[str, int]] = {} + + source_html_files = find_source_html_files(html_dir) + for html_file in source_html_files: + rel_source_path = extract_source_path_from_html(html_file, html_dir) + if not rel_source_path: + continue + + file_justifications = find_matching_justifications(rel_source_path, justified_files) + + file_stats = process_html_file(html_file, file_justifications, applied_justifications, stale_justifications) + + total_justified += file_stats["justified"] + total_stale += file_stats["stale"] + total_justified_branches += file_stats["justified_branches"] + + if file_stats["justified"] > 0 or file_stats["justified_branches"] > 0: + per_file_stats[rel_source_path] = file_stats + + # Calculate stats using llvm-cov's exact numbers + raw_uncovered = raw_total - raw_covered + unjustified_uncovered = raw_uncovered - total_justified + + effective_branch_covered = raw_branch_covered + total_justified_branches + + stats = { + "total_instrumented_lines": raw_total, + "covered_lines": raw_covered, + "justified_lines": total_justified, + "unjustified_uncovered_lines": max(0, unjustified_uncovered), + "stale_justifications": total_stale, + "raw_line_coverage_pct": floor_two_decimals(100.0 * raw_covered / raw_total) if raw_total > 0 else 0.0, + "effective_line_coverage_pct": floor_two_decimals(100.0 * (raw_covered + total_justified) / raw_total) + if raw_total > 0 + else 0.0, + "total_branches": raw_branch_total, + "covered_branches": raw_branch_covered, + "justified_branches": total_justified_branches, + "raw_branch_coverage_pct": floor_two_decimals(100.0 * raw_branch_covered / raw_branch_total) + if raw_branch_total > 0 + else 0.0, + "effective_branch_coverage_pct": floor_two_decimals(100.0 * effective_branch_covered / raw_branch_total) + if raw_branch_total > 0 + else 0.0, + } + + # Inject CSS for justified lines into style.css + inject_justified_css(html_dir) + + # Update the index page with effective coverage info and per-file stats + update_index_page(html_dir, stats, per_file_stats) + + # Write output report + report = { + "version": 1, + "summary": stats, + "applied_justifications": applied_justifications, + "stale_justifications": stale_justifications, + } + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + + # Write human-readable summary + summary_path = output_path.parent / "summary.txt" + write_summary(summary_path, stats, stale_justifications) + + # Print summary + print( + f"INFO: Effective line coverage: {stats['effective_line_coverage_pct']}% " + f"(raw: {stats['raw_line_coverage_pct']}%, " + f"justified: {stats['justified_lines']} lines, " + f"unjustified uncovered: {stats['unjustified_uncovered_lines']} lines)", + file=sys.stderr, + ) + if stats["justified_branches"] > 0: + print( + f"INFO: Effective branch coverage: {stats['effective_branch_coverage_pct']}% " + f"(raw: {stats['raw_branch_coverage_pct']}%, " + f"justified: {stats['justified_branches']} branches)", + file=sys.stderr, + ) + if stale_justifications: + print( + f"WARNING: {len(stale_justifications)} stale justifications " + f"(lines are actually covered, justification can be removed)", + file=sys.stderr, + ) + + +def process_html_file( + html_file: Path, + justifications: Dict[int, Dict[str, str]], + applied_justifications: List[Dict[str, Any]], + stale_justifications: List[Dict[str, Any]], +) -> Dict[str, int]: + """Process a single source HTML file. Modifies it in-place. + + Restyles justified lines: changes the count cell to show "J" with justified-line + class, and changes red code regions to justified (orange) background. + Also restyles uncovered branches on justified lines. + Only counts justified/stale lines for the justification report — raw coverage + numbers are taken from the index page to match llvm-cov exactly. + """ + file_stats = { + "justified": 0, + "stale": 0, + "justified_branches": 0, + } + + with open(html_file, "r", encoding="utf-8") as f: + content = f.read() + + if not justifications: + return file_stats + + # Determine effective line status (covered if ANY instantiation covers it) + row_pattern = re.compile( + r"
\d+
" + r"" + ) + line_effective_status: Dict[int, str] = {} + for m in row_pattern.finditer(content): + line_num = int(m.group(1)) + line_class = m.group(2) + if line_class == "covered-line": + line_effective_status[line_num] = "covered" + elif line_class == "uncovered-line": + if line_num not in line_effective_status: + line_effective_status[line_num] = "uncovered" + + # Determine which lines have truly uncovered branches (never covered in any instantiation). + # A branch direction is "truly uncovered" if no instantiation covers it. + branch_check_pattern = re.compile( + r"Branch \(" + r"(\d+:\d+)\):\s*\[(.*?)\]" + ) + covered_branch_dirs_check: Dict[str, set] = {} # branch_id → set of covered directions + uncovered_branch_dirs_check: Dict[str, set] = {} # branch_id → set of uncovered directions + branch_line_map: Dict[str, int] = {} # branch_id → line_num + + for m in branch_check_pattern.finditer(content): + line_num = int(m.group(1)) + branch_id = m.group(2) + branch_content = m.group(3) + branch_line_map[branch_id] = line_num + if branch_id not in covered_branch_dirs_check: + covered_branch_dirs_check[branch_id] = set() + uncovered_branch_dirs_check[branch_id] = set() + for direction in ("True", "False"): + if f"class='None'>{direction}" in branch_content: + covered_branch_dirs_check[branch_id].add(direction) + if f"class='red branch'>{direction}" in branch_content: + uncovered_branch_dirs_check[branch_id].add(direction) + + # Lines with truly uncovered branches (uncovered in ALL instantiations) + lines_with_uncovered_branches: set = set() + for branch_id, uncov_dirs in uncovered_branch_dirs_check.items(): + cov_dirs = covered_branch_dirs_check.get(branch_id, set()) + truly_uncovered = uncov_dirs - cov_dirs + if truly_uncovered: + lines_with_uncovered_branches.add(branch_line_map[branch_id]) + + # Determine which justified lines are stale vs applicable. + # A justification is stale only if the line is covered AND has no uncovered branches. + for line_num, justification in justifications.items(): + status = line_effective_status.get(line_num) + has_uncovered_branches = line_num in lines_with_uncovered_branches + if status == "covered" and not has_uncovered_branches: + file_stats["stale"] += 1 + stale_justifications.append( + { + "file": html_file.stem, + "line": line_num, + "id": justification.get("id", ""), + "reason": "Line is already covered and has no uncovered branches — justification is stale", + } + ) + elif status == "uncovered": + file_stats["justified"] += 1 + applied_justifications.append( + { + "file": html_file.stem, + "line": line_num, + "id": justification.get("id", ""), + "category": justification.get("category", ""), + } + ) + elif status == "covered" and has_uncovered_branches: + # Line is covered but has uncovered branches — justification applies to branches only + applied_justifications.append( + { + "file": html_file.stem, + "line": line_num, + "id": justification.get("id", ""), + "category": justification.get("category", ""), + } + ) + + # Restyle justified lines in the HTML (all occurrences including instantiations). + # Full row pattern to capture and replace the entire row: + # ...
0
...
... + full_row_pattern = re.compile( + r"(
\d+
)" + r"(
)\d+(
)" + r"(
)(.*?)(
)" + ) + + modified = False + + def replace_full_row(match: re.Match) -> str: + nonlocal modified + line_num = int(match.group(2)) + if line_num not in justifications: + return match.group(0) + + justification = justifications[line_num] + reason = justification.get("reason", "").replace("'", "'").replace('"', """) + jid = justification.get("id", "") + tooltip = f"Justified [{jid}]: {reason}" + modified = True + + # Rebuild the row with justified styling: + # 1. Line number td (unchanged) + line_td = match.group(1) + # 2. Count td: change class and show "J" instead of "0" + count_td = f"
J{match.group(4)}"
+        # 3. Code td: replace 'region red' spans with 'region justified'
+        code_start = match.group(5)
+        code_content = match.group(6).replace("class='region red'", "class='region justified'")
+        code_end = match.group(7)
+
+        return line_td + count_td + code_start + code_content + code_end
+
+    new_content = full_row_pattern.sub(replace_full_row, content)
+
+    # Restyle branches on justified lines.
+    # Branch format in expansion-view:
+    # Branch (195:17):
+    #   [True: 0, ...]
+    # We find branches at justified line numbers and restyle red branch → justified branch
+    # Counting: A branch direction is "uncovered" only if ALL instantiations show it as red.
+    # (Same as llvm-cov's logic: covered if ANY instantiation covers it.)
+    branch_pattern = re.compile(
+        r"(Branch \("
+        r"(\d+:\d+)\):\s*\[)(.*?\])"
+    )
+
+    # First pass: determine which branch directions are covered in any instantiation
+    covered_branch_dirs: set = set()  # (line:col, direction) that are covered somewhere
+    for m in branch_pattern.finditer(new_content):
+        line_num = int(m.group(2))
+        if line_num not in justifications:
+            continue
+        branch_id = m.group(3)
+        branch_content = m.group(4)
+        # A direction is covered if it does NOT have 'red branch' class
+        for direction in ("True", "False"):
+            # Check if this direction appears as covered (class='None' means covered)
+            covered_marker = f"class='None'>{direction}"
+            if covered_marker in branch_content:
+                covered_branch_dirs.add((branch_id, direction))
+
+    # Second pass: restyle and count only truly uncovered branch directions
+    justified_branch_ids: set = set()  # Track unique uncovered (line:col, direction) pairs
+
+    def replace_branch(match: re.Match) -> str:
+        nonlocal modified
+        line_num = int(match.group(2))
+        if line_num not in justifications:
+            return match.group(0)
+
+        branch_content = match.group(4)
+        if "class='red branch'" not in branch_content:
+            return match.group(0)
+
+        modified = True
+        branch_id = match.group(3)  # e.g. "68:13"
+
+        # Count unique uncovered branch directions that are NEVER covered in any instantiation
+        for direction in ("True", "False"):
+            if f"class='red branch'>{direction}" in branch_content:
+                uid = (branch_id, direction)
+                if uid not in covered_branch_dirs and uid not in justified_branch_ids:
+                    justified_branch_ids.add(uid)
+                    file_stats["justified_branches"] += 1
+
+        # Restyle: red branch → justified-branch, uncovered-line → justified-line
+        branch_content = branch_content.replace("class='red branch'", "class='justified-branch'")
+        branch_content = branch_content.replace("class='uncovered-line'", "class='justified-line'")
+        return match.group(1) + branch_content
+
+    new_content = branch_pattern.sub(replace_branch, new_content)
+
+    if modified:
+        with open(html_file, "w", encoding="utf-8") as f:
+            f.write(new_content)
+
+    return file_stats
+
+
+def parse_index_page_totals(html_dir: Path) -> Dict[str, Tuple[int, int]]:
+    """Parse the TOTALS row from the llvm-cov index.html to get exact coverage numbers.
+
+    Returns dict with 'lines' and 'branches' keys, each (covered, total).
+    The index page TOTALS row format: "93.55% (17565/18777)" — func, line, branch.
+    """
+    index_file = html_dir / "index.html"
+    if not index_file.exists():
+        return {"lines": (0, 0), "branches": (0, 0)}
+
+    with open(index_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    pct_pattern = re.compile(r"(\d+\.\d+)%\s*\((\d+)/(\d+)\)")
+    matches = pct_pattern.findall(content)
+
+    result = {"lines": (0, 0), "branches": (0, 0)}
+
+    if len(matches) >= 3:
+        # Last 3 matches are from TOTALS row: func, line, branch
+        totals_matches = matches[-3:]
+        _, line_covered, line_total = totals_matches[1]
+        result["lines"] = (int(line_covered), int(line_total))
+        _, branch_covered, branch_total = totals_matches[2]
+        result["branches"] = (int(branch_covered), int(branch_total))
+
+    if result["lines"] == (0, 0):
+        print("WARNING: Could not parse coverage totals from index.html", file=sys.stderr)
+
+    return result
+
+
+def inject_justified_css(html_dir: Path) -> None:
+    """Add CSS for justified lines to style.css."""
+    style_file = html_dir / "style.css"
+    if not style_file.exists():
+        return
+
+    justified_css = """
+/* Coverage justification styling */
+.justified-line {
+  text-align: right;
+  color: #a60;
+}
+.region.justified {
+  background-color: #fa04;
+}
+.justified-branch {
+  color: #a60;
+  font-weight: bold;
+}
+tr:has(> td.justified-line) > td.code {
+  background-color: #fff3e0;
+}
+@media (prefers-color-scheme: dark) {
+  .justified-line {
+    color: #fa0;
+  }
+  .justified-branch {
+    color: #fa0;
+  }
+  tr:has(> td.justified-line) > td.code {
+    background-color: #3d2800;
+  }
+  .region.justified {
+    background-color: #fa03;
+  }
+}
+"""
+
+    with open(style_file, "a", encoding="utf-8") as f:
+        f.write(justified_css)
+
+
+def update_index_page(html_dir: Path, stats: Dict[str, Any], per_file_stats: Dict[str, Dict[str, int]]) -> None:
+    """Update the index page with effective coverage info and per-file adjusted percentages."""
+    index_file = html_dir / "index.html"
+    if not index_file.exists():
+        return
+
+    with open(index_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Banner with overall effective coverage (lines + branches)
+    branch_info = ""
+    if stats.get("justified_branches", 0) > 0:
+        branch_info = (
+            f" | Effective Branch Coverage: {stats['effective_branch_coverage_pct']}%"
+            f" (Raw: {stats['raw_branch_coverage_pct']}%, Justified: {stats['justified_branches']} branches)"
+        )
+
+    banner = (
+        f"
" + f"Effective Line Coverage: {stats['effective_line_coverage_pct']}% " + f"(Raw: {stats['raw_line_coverage_pct']}% | " + f"Justified: {stats['justified_lines']} lines | " + f"Unjustified Uncovered: {stats['unjustified_uncovered_lines']} lines)" + f"{branch_info}" + f"
" + ) + + # Insert after the tag or after the first

+ if "

" in content: + content = content.replace("

", banner + "

", 1) + else: + content = content.replace("", f"{banner}", 1) + + # Update per-file rows in the index table. + # For each file with justifications, find its row and update line% and branch% cells. + # Row format:
displayname
+ #
  XX.XX% (covered/total)
← function + #
  XX.XX% (covered/total)
← line + #
  XX.XX% (covered/total)
← branch + # + pct_cell_pattern = re.compile(r"
\s*(\d+\.\d+)%\s*\((\d+)/(\d+)\)
") + + for file_path, fstats in per_file_stats.items(): + justified_lines = fstats.get("justified", 0) + justified_branches = fstats.get("justified_branches", 0) + if justified_lines == 0 and justified_branches == 0: + continue + + # Find the row for this file in the index page + # The href contains the full path to the HTML file + if file_path not in content: + continue + + # Find the containing this file path + file_idx = content.find(file_path) + if file_idx < 0: + continue + row_start = content.rfind("", file_idx) + if row_start < 0 or row_end < 0: + continue + + row = content[row_start : row_end + 5] + + # Find all percentage cells in this row (func, line, branch) + cells = list(pct_cell_pattern.finditer(row)) + if len(cells) < 2: + continue + + new_row = row + # Update line coverage cell (second cell, index 1) + if justified_lines > 0 and len(cells) >= 2: + line_cell = cells[1] + covered = int(line_cell.group(3)) + total = int(line_cell.group(4)) + eff_covered = covered + justified_lines + eff_pct = floor_two_decimals(100.0 * eff_covered / total) if total > 0 else 0.0 + color = _get_coverage_color(eff_pct) + old_cell = line_cell.group(0) + new_cell = f"
{eff_pct:>7.2f}% ({eff_covered}/{total})
" + new_row = new_row.replace(old_cell, new_cell) + + # Update branch coverage cell (third cell, index 2) + if justified_branches > 0 and len(cells) >= 3: + branch_cell = cells[2] + covered = int(branch_cell.group(3)) + total = int(branch_cell.group(4)) + eff_covered = covered + justified_branches + eff_pct = floor_two_decimals(100.0 * eff_covered / total) if total > 0 else 0.0 + color = _get_coverage_color(eff_pct) + old_cell = branch_cell.group(0) + new_cell = f"
{eff_pct:>7.2f}% ({eff_covered}/{total})
" + new_row = new_row.replace(old_cell, new_cell) + + if new_row != row: + content = content.replace(row, new_row) + + # Update the TOTALS row + content = _update_totals_row(content, stats) + + with open(index_file, "w", encoding="utf-8") as f: + f.write(content) + + +def _get_coverage_color(pct: float) -> str: + """Return the llvm-cov color class for a coverage percentage.""" + if pct >= 100.0: + return "green" + elif pct >= 80.0: + return "yellow" + else: + return "red" + + +def _update_totals_row(content: str, stats: Dict[str, Any]) -> str: + """Update the TOTALS row in the index page with effective coverage numbers.""" + # Find the TOTALS row — it's the last row before + totals_idx = content.rfind("Totals") + if totals_idx < 0: + return content + + row_start = content.rfind("", totals_idx) + if row_start < 0 or row_end < 0: + return content + + row = content[row_start : row_end + 5] + + pct_cell_pattern = re.compile(r"
\s*(\d+\.\d+)%\s*\((\d+)/(\d+)\)
") + cells = list(pct_cell_pattern.finditer(row)) + + new_row = row + + # Update line coverage in totals (index 1) + if len(cells) >= 2 and stats.get("justified_lines", 0) > 0: + line_cell = cells[1] + eff_covered = stats["covered_lines"] + stats["justified_lines"] + total = stats["total_instrumented_lines"] + eff_pct = stats["effective_line_coverage_pct"] + color = _get_coverage_color(eff_pct) + old_cell = line_cell.group(0) + new_cell = f"
{eff_pct:>7.2f}% ({eff_covered}/{total})
" + new_row = new_row.replace(old_cell, new_cell) + + # Update branch coverage in totals (index 2) + if len(cells) >= 3 and stats.get("justified_branches", 0) > 0: + branch_cell = cells[2] + eff_covered = stats["covered_branches"] + stats["justified_branches"] + total = stats["total_branches"] + eff_pct = stats["effective_branch_coverage_pct"] + color = _get_coverage_color(eff_pct) + old_cell = branch_cell.group(0) + new_cell = f"
{eff_pct:>7.2f}% ({eff_covered}/{total})
" + new_row = new_row.replace(old_cell, new_cell) + + if new_row != row: + content = content.replace(row, new_row) + + return content + + +def find_source_html_files(html_dir: Path) -> List[Path]: + """Find all per-source HTML files (not index.html, style.css, etc.).""" + coverage_dir = html_dir / "coverage" + if not coverage_dir.exists(): + # Some llvm-cov versions put source files directly in html_dir + coverage_dir = html_dir + + files = [] + for html_file in coverage_dir.rglob("*.html"): + if html_file.name in ("index.html",): + continue + files.append(html_file) + return sorted(files) + + +def extract_source_path_from_html(html_file: Path, html_dir: Path) -> str: + """Extract the relative source file path from the HTML file path. + + llvm-cov creates paths like: html_report/coverage/.html + We need to extract the relative path within the project. + """ + rel = str(html_file.relative_to(html_dir)) + # Remove "coverage/" prefix if present + if rel.startswith("coverage/"): + rel = rel[len("coverage/") :] + # Remove .html suffix + if rel.endswith(".html"): + rel = rel[:-5] + return rel + + +def find_matching_justifications( + source_path: str, justified_files: Dict[str, Dict[str, Dict[str, str]]] +) -> Dict[int, Dict[str, str]]: + """Find justifications that match the given source path. + + The source_path from HTML may be an absolute path or relative. + The justified_files keys are relative to source root. + We match by suffix. + """ + result: Dict[int, Dict[str, str]] = {} + + for justified_path, line_justifications in justified_files.items(): + # Match if the source_path ends with the justified_path + if source_path.endswith(justified_path) or justified_path.endswith(source_path): + for line_str, justification in line_justifications.items(): + result[int(line_str)] = justification + + return result + + +def write_summary(path: Path, stats: Dict[str, Any], stale: List[Dict[str, Any]]) -> None: + """Write human-readable summary.""" + with open(path, "w", encoding="utf-8") as f: + f.write("Coverage Justification Summary\n") + f.write("=" * 40 + "\n\n") + f.write(f"Total instrumented lines: {stats['total_instrumented_lines']}\n") + f.write(f"Covered lines: {stats['covered_lines']}\n") + f.write(f"Justified lines: {stats['justified_lines']}\n") + f.write(f"Unjustified uncovered: {stats['unjustified_uncovered_lines']}\n") + f.write(f"\n") + f.write(f"Raw line coverage: {stats['raw_line_coverage_pct']}%\n") + f.write(f"Effective line coverage: {stats['effective_line_coverage_pct']}%\n") + f.write(f"\n") + if stats.get("total_branches", 0) > 0: + f.write(f"Total branches: {stats['total_branches']}\n") + f.write(f"Covered branches: {stats['covered_branches']}\n") + f.write(f"Justified branches: {stats['justified_branches']}\n") + f.write(f"Raw branch coverage: {stats['raw_branch_coverage_pct']}%\n") + f.write(f"Effective branch coverage: {stats['effective_branch_coverage_pct']}%\n") + f.write(f"\n") + if stale: + f.write(f"Stale justifications ({len(stale)}):\n") + for s in stale: + f.write(f" - {s['file']}:{s['line']} [{s['id']}]\n") + f.write("\n") + + +def load_manifest(path: Path) -> Dict[str, Any]: + """Load the justification manifest JSON.""" + if not path.exists(): + print(f"ERROR: Manifest not found: {path}", file=sys.stderr) + sys.exit(1) + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Effective coverage calculator and HTML post-processor") + parser.add_argument( + "--html-dir", + type=Path, + required=True, + help="Path to llvm-cov HTML report directory", + ) + parser.add_argument( + "--manifest", + type=Path, + required=True, + help="Path to resolved justification manifest (from justify.py)", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Output path for justification report (JSON)", + ) + parser.add_argument( + "--lcov", + type=Path, + default=None, + help="Optional path to LCOV data file (used for gcovr format totals)", + ) + return parser.parse_args() + + +# ============================================================================= +# Format detection +# ============================================================================= + + +def detect_html_format(html_dir: Path) -> str: + """Detect whether the HTML report was generated by llvm-cov or gcovr. + + Returns 'llvm_cov' or 'gcovr'. + """ + # gcovr produces files named based on --html-details argument. + # We use index.html, so per-source files are index...html + if any(html_dir.glob("index.*.*.html")): + return "gcovr" + if (html_dir / "coverage_details.html").exists(): + return "gcovr" + return "llvm_cov" + + +def _parse_lcov_totals(lcov_path: Path) -> Dict[str, Tuple[int, int]]: + """Parse coverage totals from an LCOV data file. + + Sums LH/LF (line hit/found) and BRH/BRF (branch hit/found) across all records. + """ + total_lines = 0 + hit_lines = 0 + total_branches = 0 + hit_branches = 0 + + with open(lcov_path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if line.startswith("LF:"): + total_lines += int(line[3:]) + elif line.startswith("LH:"): + hit_lines += int(line[3:]) + elif line.startswith("BRF:"): + total_branches += int(line[4:]) + elif line.startswith("BRH:"): + hit_branches += int(line[4:]) + + return { + "lines": (hit_lines, total_lines), + "branches": (hit_branches, total_branches), + } + + +# ============================================================================= +# gcovr support +# ============================================================================= + + +def _main_gcovr(args: argparse.Namespace, html_dir: Path, justified_files: Dict) -> None: + """Main logic for gcovr HTML format (produced by lcov_to_html.py via gcovr).""" + + # Parse coverage totals from LCOV file or gcovr index page. + if args.lcov and args.lcov.exists(): + totals = _parse_lcov_totals(args.lcov) + else: + totals = _parse_gcovr_index_totals(html_dir) + + raw_covered, raw_total = totals["lines"] + raw_branch_covered, raw_branch_total = totals["branches"] + + # Process each source HTML file + total_justified = 0 + total_stale = 0 + total_justified_branches = 0 + applied_justifications: List[Dict[str, Any]] = [] + stale_justifications: List[Dict[str, Any]] = [] + per_file_stats: Dict[str, Dict[str, int]] = {} + + source_html_files = _find_gcovr_source_files(html_dir) + for html_file in source_html_files: + rel_source_path = _extract_gcovr_source_path(html_file) + if not rel_source_path: + continue + + file_justifications = find_matching_justifications(rel_source_path, justified_files) + + file_stats = _process_gcovr_file(html_file, file_justifications, applied_justifications, stale_justifications) + + total_justified += file_stats["justified"] + total_stale += file_stats["stale"] + total_justified_branches += file_stats["justified_branches"] + + if file_stats["justified"] > 0 or file_stats["justified_branches"] > 0: + per_file_stats[rel_source_path] = file_stats + + # Calculate stats + raw_uncovered = raw_total - raw_covered + unjustified_uncovered = raw_uncovered - total_justified + effective_branch_covered = raw_branch_covered + total_justified_branches + + stats = { + "total_instrumented_lines": raw_total, + "covered_lines": raw_covered, + "justified_lines": total_justified, + "unjustified_uncovered_lines": max(0, unjustified_uncovered), + "stale_justifications": total_stale, + "raw_line_coverage_pct": floor_two_decimals(100.0 * raw_covered / raw_total) if raw_total > 0 else 0.0, + "effective_line_coverage_pct": floor_two_decimals(100.0 * (raw_covered + total_justified) / raw_total) + if raw_total > 0 + else 0.0, + "total_branches": raw_branch_total, + "covered_branches": raw_branch_covered, + "justified_branches": total_justified_branches, + "raw_branch_coverage_pct": floor_two_decimals(100.0 * raw_branch_covered / raw_branch_total) + if raw_branch_total > 0 + else 0.0, + "effective_branch_coverage_pct": floor_two_decimals(100.0 * effective_branch_covered / raw_branch_total) + if raw_branch_total > 0 + else 0.0, + } + + # Inject CSS for justified lines + _inject_gcovr_justified_css(html_dir) + + # Update the gcovr index page with effective coverage banner + _update_gcovr_index_page(html_dir, stats) + + # Write output report + report = { + "version": 1, + "summary": stats, + "applied_justifications": applied_justifications, + "stale_justifications": stale_justifications, + } + + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + print(f"Effective coverage report written to: {args.output}") + else: + print(json.dumps(report, indent=2)) + + +def _parse_gcovr_index_totals(html_dir: Path) -> Dict[str, Tuple[int, int]]: + """Parse coverage totals from gcovr's index page. + + gcovr format shows coverage in the summary header with patterns like: + 75.0% + 30 / 40 + """ + index_file = html_dir / "index.html" + if not index_file.exists(): + # Fallback for older naming + index_file = html_dir / "coverage_details.html" + if not index_file.exists(): + return {"lines": (0, 0), "branches": (0, 0)} + + content = index_file.read_text(encoding="utf-8", errors="replace") + + # gcovr summary table has rows with line/branch stats. + # Look for the summary div with class "summary" containing coverage percentages. + # Pattern: "N / M" in table cells for covered/total + lines_covered, lines_total = 0, 0 + branches_covered, branches_total = 0, 0 + + # gcovr index uses meter elements with data attributes or text like "N / M" + # The summary section shows: Line Coverage: XX.X% (covered / total) + ratio_pattern = re.compile(r"(\d+)\s*/\s*(\d+)") + + # Find the summary section — gcovr puts line/branch/function stats in order. + # Look for the summary-coverage section + summary_match = re.search(r'class="[^"]*summary[^"]*".*?', content, re.DOTALL) + if summary_match: + summary_text = summary_match.group(0) + ratios = ratio_pattern.findall(summary_text) + # gcovr order: lines, functions, branches (if branches enabled) + if len(ratios) >= 1: + lines_covered, lines_total = int(ratios[0][0]), int(ratios[0][1]) + if len(ratios) >= 3: + branches_covered, branches_total = int(ratios[2][0]), int(ratios[2][1]) + elif len(ratios) >= 2: + # Might be lines then branches (no functions section) + branches_covered, branches_total = int(ratios[1][0]), int(ratios[1][1]) + + if lines_total == 0: + # Fallback: scan the whole page for ratio patterns + all_ratios = ratio_pattern.findall(content) + if len(all_ratios) >= 1: + lines_covered, lines_total = int(all_ratios[0][0]), int(all_ratios[0][1]) + if len(all_ratios) >= 3: + branches_covered, branches_total = int(all_ratios[2][0]), int(all_ratios[2][1]) + + return { + "lines": (lines_covered, lines_total), + "branches": (branches_covered, branches_total), + } + + +def _find_gcovr_source_files(html_dir: Path) -> List[Path]: + """Find all per-source HTML files in a gcovr report. + + gcovr --html-details creates files named: + index...html + The index is index.html (no dot-separated parts after "index"). + """ + files = [] + for html_file in sorted(html_dir.glob("index.*.html")): + # Skip function pages (contain ".functions." in name) + if ".functions." in html_file.name: + continue + files.append(html_file) + return files + + +def _extract_gcovr_source_path(html_file: Path) -> str: + """Extract the source file path from a gcovr per-source HTML file. + + gcovr embeds the directory in a "Directory:" table row and the filename + in the Box-header. Combining these gives the full relative source path. + """ + try: + content = html_file.read_text(encoding="utf-8", errors="replace")[:8000] + except OSError: + return "" + + # Extract directory from: Directory:\npath/ + directory = "" + dir_match = re.search(r"Directory:\s*([^<]*)", content) + if dir_match: + directory = dir_match.group(1).strip() + + # Extract filename from the Box-header + filename = "" + name_match = re.search( + r'class="Box-header[^"]*d-flex flex-space-between[^"]*"[^>]*>\s*\n?\s*([^\n<]+)', + content, + ) + if name_match: + filename = name_match.group(1).strip() + + if not filename: + # Fallback: look in + title_match = re.search(r"<title>([^<]+?)(?:\s*-\s*GCC[^<]*)?", content) + if title_match: + filename = title_match.group(1).strip() + + if directory and filename: + return directory + filename + return filename + + +def _process_gcovr_file( + html_file: Path, + justifications: Dict[int, Dict[str, str]], + applied_justifications: List[Dict[str, Any]], + stale_justifications: List[Dict[str, Any]], +) -> Dict[str, int]: + """Process a single gcovr source HTML file. Modifies it in-place. + + gcovr line format: + + {num} + ... + 5 + source code + + + Coverage classes: + - coveredLine: line is covered (count > 0) + - uncoveredLine: line is not covered (count == 0) + - partialCoveredLine: some branches not taken + - excludedLine: excluded from coverage + """ + file_stats = {"justified": 0, "stale": 0, "justified_branches": 0} + + if not justifications: + return file_stats + + with open(html_file, "r", encoding="utf-8") as f: + content = f.read() + + # Parse line coverage status from gcovr HTML. + # Each source line: + # followed by + line_pattern = re.compile( + r']*>.*?' + r' on lines with branch issues + branch_pattern = re.compile( + r']*>.*?(.*?)', + re.DOTALL, + ) + + line_effective_status: Dict[int, str] = {} + lines_with_uncovered_branches: set = set() + + for m in line_pattern.finditer(content): + line_num = int(m.group(1)) + cov_class = m.group(2) + if cov_class == "coveredLine": + line_effective_status[line_num] = "covered" + elif cov_class == "uncoveredLine": + line_effective_status[line_num] = "uncovered" + elif cov_class == "partialCoveredLine": + line_effective_status[line_num] = "covered" + lines_with_uncovered_branches.add(line_num) + + # Also check branch details for not-taken branches + for m in branch_pattern.finditer(content): + line_num = int(m.group(1)) + branch_content = m.group(2) + if "notTakenBranch" in branch_content: + lines_with_uncovered_branches.add(line_num) + + # Determine stale vs applicable justifications + for line_num, justification in justifications.items(): + status = line_effective_status.get(line_num) + has_uncovered_branches = line_num in lines_with_uncovered_branches + if status == "covered" and not has_uncovered_branches: + file_stats["stale"] += 1 + stale_justifications.append( + { + "file": _extract_gcovr_source_path(html_file), + "line": line_num, + "id": justification.get("id", ""), + "reason": "Line is already covered and has no uncovered branches — justification is stale", + } + ) + elif status == "uncovered": + file_stats["justified"] += 1 + applied_justifications.append( + { + "file": _extract_gcovr_source_path(html_file), + "line": line_num, + "id": justification.get("id", ""), + "category": justification.get("category", ""), + } + ) + # Mark the line as justified in the HTML + content = _mark_gcovr_line_justified(content, line_num) + elif status == "covered" and has_uncovered_branches: + file_stats["justified_branches"] += 1 + applied_justifications.append( + { + "file": _extract_gcovr_source_path(html_file), + "line": line_num, + "id": justification.get("id", ""), + "category": justification.get("category", ""), + } + ) + + # Write modified content back + if file_stats["justified"] > 0: + with open(html_file, "w", encoding="utf-8") as f: + f.write(content) + + return file_stats + + +def _mark_gcovr_line_justified(content: str, line_num: int) -> str: + """Replace 'uncoveredLine' with 'justifiedLine' for a specific line in gcovr HTML.""" + # Find the line anchor and replace the coverage class in that row + # Pattern: ...uncoveredLine... (within the same ) + pattern = re.compile( + rf'(]*>.*?)', + re.DOTALL, + ) + match = pattern.search(content) + if match: + original = match.group(0) + modified = original.replace("uncoveredLine", "justifiedLine") + modified = modified.replace("show_uncoveredLine", "show_justifiedLine") + content = content[: match.start()] + modified + content[match.end() :] + return content + + +def _inject_gcovr_justified_css(html_dir: Path) -> None: + """Inject CSS for justified lines into gcovr's stylesheet.""" + # gcovr names its CSS file based on the output filename (coverage_details.css) + css_files = list(html_dir.glob("*.css")) + if not css_files: + return + + justified_css = """ +/* Justified line styling (injected by effective_coverage.py) */ +.justifiedLine { background-color: #ffe4b5 !important; } +.show_justifiedLine { display: table-cell; } +.button_toggle_justifiedLine { background-color: #ffe4b5; border: 1px solid #daa520; } +""" + # Inject into the first CSS file found + with open(css_files[0], "a", encoding="utf-8") as f: + f.write(justified_css) + + +def _update_gcovr_index_page(html_dir: Path, stats: Dict[str, Any]) -> None: + """Update the gcovr index page with an effective coverage banner.""" + index_file = html_dir / "index.html" + if not index_file.exists(): + index_file = html_dir / "coverage_details.html" + if not index_file.exists(): + return + + with open(index_file, "r", encoding="utf-8") as f: + content = f.read() + + branch_info = "" + if stats.get("justified_branches", 0) > 0: + branch_info = ( + f" | Effective Branch Coverage: {stats['effective_branch_coverage_pct']}%" + f" (Raw: {stats['raw_branch_coverage_pct']}%, Justified: {stats['justified_branches']} branches)" + ) + + banner = ( + f'
' + f"Effective Line Coverage: {stats['effective_line_coverage_pct']}% " + f"(Raw: {stats['raw_line_coverage_pct']}% | " + f"Justified: {stats['justified_lines']} lines | " + f"Unjustified Uncovered: {stats['unjustified_uncovered_lines']} lines)" + f"{branch_info}" + f"
" + ) + + # Insert before the file listing + if '
" in content: + content = content.replace("", f"{banner}", 1) + + with open(index_file, "w", encoding="utf-8") as f: + f.write(content) + + +if __name__ == "__main__": + main() diff --git a/coverage/ferrocene_report.sh b/coverage/ferrocene_report.sh deleted file mode 100755 index 365b2006..00000000 --- a/coverage/ferrocene_report.sh +++ /dev/null @@ -1,969 +0,0 @@ -#!/usr/bin/env bash -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -set -euo pipefail - -usage() { - cat <<'USAGE' -Generate Ferrocene Rust coverage reports from Bazel rust_test targets. - -Usage: - bazel run @score_tooling//coverage:ferrocene_report -- [options] - -Options: - --query Bazel query for rust_test targets. - Default: kind("rust_test", //...) - --targets Comma-separated list of test labels (overrides --query). - --out-dir Output directory for reports. - Default: /coverage/rust-tests - --profraw-dir Directory containing .profraw files for all targets. - If set, used for every target. - --profraw-root Root for bazel-testlogs. Default: bazel info bazel-testlogs - --bazel-config Bazel config to use for build/cquery. - Repeat to pass multiple configs. - Default: ferrocene-coverage - --min-line-coverage

Minimum line coverage percentage (0-100). - If any target is below, exit non-zero. - --target-triple Rust target triple. Default: x86_64-unknown-linux-gnu - --ferrocene-repo Ferrocene repo name. Default: ferrocene_x86_64_unknown_linux_gnu - --help Show this help. - -Notes: -- Run tests first with: bazel test --config=ferrocene-coverage -- The config should set LLVM_PROFILE_FILE to TEST_UNDECLARED_OUTPUTS_DIR. -- For cross/target execution, copy .profraw files back and pass --profraw-dir. -USAGE -} - -QUERY='kind("rust_test", //...)' -TARGETS_CSV="" -OUT_DIR="" -PROFRAW_DIR="" -PROFRAW_ROOT="" -BAZEL_CONFIGS=() -TARGET_TRIPLE="x86_64-unknown-linux-gnu" -FERROCENE_REPO="ferrocene_x86_64_unknown_linux_gnu" -SYMBOL_REPORT_LABEL="" -BLANKET_LABEL="" -MIN_LINE_COVERAGE="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --query) - QUERY="$2"; shift 2 ;; - --targets) - TARGETS_CSV="$2"; shift 2 ;; - --out-dir) - OUT_DIR="$2"; shift 2 ;; - --profraw-dir) - PROFRAW_DIR="$2"; shift 2 ;; - --profraw-root) - PROFRAW_ROOT="$2"; shift 2 ;; - --bazel-config) - BAZEL_CONFIGS+=("$2"); shift 2 ;; - --min-line-coverage) - MIN_LINE_COVERAGE="$2"; shift 2 ;; - --target-triple) - TARGET_TRIPLE="$2"; shift 2 ;; - --ferrocene-repo) - FERROCENE_REPO="$2"; shift 2 ;; - -h|--help) - usage; exit 0 ;; - *) - echo "Unknown arg: $1" >&2 - usage - exit 2 - ;; - esac - done - -workspace="${BUILD_WORKSPACE_DIRECTORY:-$(pwd)}" -cd "${workspace}" - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -parse_line_coverage_py="${script_dir}/scripts/parse_line_coverage.py" -normalize_symbol_report_py="${script_dir}/scripts/normalize_symbol_report.py" -if [[ ! -f "${parse_line_coverage_py}" || ! -f "${normalize_symbol_report_py}" ]]; then - echo "Coverage helper scripts not found under ${script_dir}/scripts" >&2 - exit 1 -fi - -if [[ -z "${OUT_DIR}" ]]; then - # Keep reports under bazel-bin by default so they track the build output tree. - OUT_DIR="$(bazel info bazel-bin)/coverage/rust-tests" -fi - -if [[ ${#BAZEL_CONFIGS[@]} -eq 0 ]]; then - BAZEL_CONFIGS=("ferrocene-coverage") -fi - -BAZEL_FLAGS=() -for cfg in "${BAZEL_CONFIGS[@]}"; do - if [[ -n "${cfg}" ]]; then - BAZEL_FLAGS+=("--config=${cfg}") - fi -done - -if [[ -z "${SYMBOL_REPORT_LABEL}" ]]; then - SYMBOL_REPORT_LABEL="@score_toolchains_rust//toolchains/ferrocene:${FERROCENE_REPO}_symbol-report" -fi -if [[ -z "${BLANKET_LABEL}" ]]; then - BLANKET_LABEL="@score_toolchains_rust//toolchains/ferrocene:${FERROCENE_REPO}_blanket" -fi - -if [[ -z "${PROFRAW_ROOT}" ]]; then - PROFRAW_ROOT="$(bazel info bazel-testlogs)" -fi - -output_base="$(bazel info output_base)" -exec_root="$(bazel info execution_root)" - -cquery_expr() { - local label="$1" - local expr="$2" - bazel cquery "${BAZEL_FLAGS[@]}" --output=starlark --starlark:expr="${expr}" "${label}" -} - -query_attr_build() { - local label="$1" - local attr="$2" - local tmp - tmp="$(mktemp)" - bazel query --output=build "${label}" >"${tmp}" - python3 - "$attr" "${tmp}" <<'PY' -import re -import sys - -attr = sys.argv[1] -with open(sys.argv[2], "r", encoding="utf-8") as fh: - data = fh.read() - -# Match attr = on a single line. -m = re.search(r'\\b%s\\s*=\\s*([^\\n]+)' % re.escape(attr), data) -if not m: - sys.exit(0) -val = m.group(1).strip() - -# Trim trailing commas. -if val.endswith(","): - val = val[:-1].strip() - -# If list spans multiple lines, capture the first [...] block. -if val.startswith("[") and "]" not in val: - m = re.search(r'\\b%s\\s*=\\s*(\\[[^\\]]*\\])' % re.escape(attr), data, re.S) - if m: - val = m.group(1).strip() - -print(val) -PY - rm -f "${tmp}" -} - -query_labels_attr() { - local label="$1" - local attr="$2" - bazel query "labels(${attr}, ${label})" 2>/dev/null | head -n 1 -} - -aq_rustc_info() { - local label="$1" - local crate_root_rel="$2" - local exec_root="$3" - local tmp - tmp="$(mktemp)" - if ! bazel aquery "${BAZEL_FLAGS[@]}" --include_commandline --output=jsonproto "mnemonic(Rustc, ${label})" >"${tmp}" 2>/dev/null; then - rm -f "${tmp}" - return 0 - fi - python3 - "${crate_root_rel}" "${exec_root}" "${tmp}" <<'PY' -import json -import sys -import shlex -from pathlib import Path - -crate_root = sys.argv[1] -exec_root = sys.argv[2] -path = sys.argv[3] -try: - data = json.load(open(path, "r", encoding="utf-8")) -except Exception: - sys.exit(0) - -actions = data.get("actions", []) -if not actions: - sys.exit(0) - -def match_action(act): - args = act.get("arguments", []) - if crate_root and crate_root in args: - return True - if crate_root: - for arg in args: - if arg.endswith(crate_root): - return True - return False - -action = None -for act in actions: - if match_action(act): - action = act - break -if action is None: - action = actions[0] - -args = list(action.get("arguments", []) or []) - -def resolve_path(p: Path) -> Path: - if p.is_absolute(): - return p - if exec_root: - return Path(exec_root) / p - return p - -def expand_params(argv): - expanded = [] - for arg in argv: - if arg.startswith("@") and len(arg) > 1: - p = resolve_path(Path(arg[1:])) - if p.is_file(): - content = p.read_text(encoding="utf-8", errors="ignore") - try: - expanded.extend(shlex.split(content)) - except ValueError: - expanded.extend([line for line in content.splitlines() if line.strip()]) - continue - expanded.append(arg) - return expanded - -args = expand_params(args) - -if "--" in args: - args = args[args.index("--") + 1:] - -link_output = "" -for a in args: - if a.startswith("--emit=link="): - link_output = a.split("=", 2)[2] - break - -out = [] -i = 0 -while i < len(args): - a = args[i] - if a == "--extern" and i + 1 < len(args): - out.extend(["--extern", args[i + 1]]) - i += 2 - continue - if a.startswith("--extern="): - out.append(a) - i += 1 - continue - if a == "-L" and i + 1 < len(args): - out.extend(["-L", args[i + 1]]) - i += 2 - continue - if a.startswith("-L"): - out.append(a) - i += 1 - continue - if a == "--cfg" and i + 1 < len(args): - out.extend(["--cfg", args[i + 1]]) - i += 2 - continue - if a.startswith("--cfg="): - out.append(a) - i += 1 - continue - if a == "--test": - out.append(a) - i += 1 - continue - i += 1 - -lines = [link_output] + out -sys.stdout.write("\n".join(lines)) -PY - rm -f "${tmp}" -} - -tool_cquery_expr() { - local label="$1" - local expr="$2" - bazel cquery --output=starlark --starlark:expr="${expr}" "${label}" -} - -# Build and locate the coverage tool wrappers (host tools; avoid target platform flags). -bazel build "${SYMBOL_REPORT_LABEL}" "${BLANKET_LABEL}" >/dev/null - -symbol_report_rel="$(tool_cquery_expr "${SYMBOL_REPORT_LABEL}" 'target.files.to_list()[0].path')" -blanket_rel="$(tool_cquery_expr "${BLANKET_LABEL}" 'target.files.to_list()[0].path')" -if [[ -z "${symbol_report_rel}" || -z "${blanket_rel}" ]]; then - echo "Failed to resolve coverage tool wrapper paths." >&2 - exit 1 -fi - -if [[ "${symbol_report_rel}" == /* ]]; then - symbol_report_bin="${symbol_report_rel}" -else - symbol_report_bin="${exec_root}/${symbol_report_rel}" -fi -if [[ "${blanket_rel}" == /* ]]; then - blanket_bin="${blanket_rel}" -else - blanket_bin="${exec_root}/${blanket_rel}" -fi - -resolve_realpath() { - local path="$1" - if command -v realpath >/dev/null 2>&1; then - realpath "${path}" - return 0 - fi - if command -v readlink >/dev/null 2>&1; then - readlink -f "${path}" - return 0 - fi - echo "${path}" -} - -strip_quotes() { - local v="$1" - v="${v%\"}" - v="${v#\"}" - echo "${v}" -} - -label_to_path() { - local label - local pkg="${2:-}" - label="$(strip_quotes "$1")" - # External labels look like "@repo//pkg:target". Strip the repo prefix so - # path conversion works for both workspace and external repos. - if [[ "${label}" == @*//?* ]]; then - label="//${label#*//}" - fi - # If the label still starts with "@", we do not know how to map it to a path. - if [[ "${label}" == @* ]]; then - echo "" - return 0 - fi - if [[ "${label}" == :* ]]; then - if [[ -n "${pkg}" ]]; then - echo "${pkg}/${label#:}" - else - echo "${label#:}" - fi - return 0 - fi - if [[ "${label}" == //* ]]; then - local rest="${label#//}" - local pkg="${rest%%:*}" - if [[ "${rest}" == *:* ]]; then - local name="${rest#*:}" - if [[ -n "${pkg}" ]]; then - echo "${pkg}/${name}" - else - echo "${name}" - fi - else - echo "${pkg}" - fi - return 0 - fi - echo "${label}" -} - -normalize_scalar() { - local v - v="$(strip_quotes "$1")" - if [[ "${v}" =~ ^Label\\(\"(.*)\"\\)$ ]]; then - v="${BASH_REMATCH[1]}" - fi - if [[ "${v}" == \[*\] ]]; then - v="${v#[}" - v="${v%]}" - v="$(strip_quotes "${v%%,*}")" - fi - echo "${v}" -} - -normalize_label() { - local label - local pkg="${2:-}" - label="$(strip_quotes "$1")" - if [[ "${label}" =~ ^Label\\(\"(.*)\"\\)$ ]]; then - label="${BASH_REMATCH[1]}" - fi - if [[ "${label}" == :* ]]; then - if [[ -n "${pkg}" ]]; then - echo "//${pkg}:${label#:}" - else - echo "${label#:}" - fi - return 0 - fi - echo "${label}" -} - -label_pkg() { - local label - label="$(strip_quotes "$1")" - if [[ "${label}" =~ ^Label\\(\"(.*)\"\\)$ ]]; then - label="${BASH_REMATCH[1]}" - fi - # External labels include "@repo//". Strip the repo prefix to get the package. - if [[ "${label}" == @*//?* ]]; then - label="//${label#*//}" - fi - if [[ "${label}" == //* ]]; then - local rest="${label#//}" - echo "${rest%%:*}" - return 0 - fi - echo "" -} - -# Resolve the "external/" prefix for an external label. -# We use bazel query --output=location to find a real file path, then extract -# the repo name from either "external//..." or ".../external//...". -workspace_root_for_label() { - local label - label="$(strip_quotes "$1")" - if [[ "${label}" =~ ^Label\\(\"(.*)\"\\)$ ]]; then - label="${BASH_REMATCH[1]}" - fi - # Non-external labels live in the workspace, so no external prefix is needed. - if [[ "${label}" != @* ]]; then - echo "" - return 0 - fi - # The location output may be absolute; handle both direct external paths - # and absolute paths that contain "/external//". - local location - location="$(bazel query --output=location "${label}" 2>/dev/null | head -n 1)" - location="${location%%:*}" - local rest="" - if [[ "${location}" == external/* ]]; then - rest="${location#external/}" - elif [[ "${location}" == */external/* ]]; then - rest="${location#*/external/}" - fi - local repo="${rest%%/*}" - if [[ -n "${repo}" ]]; then - echo "external/${repo}" - fi -} - -resolve_runfile() { - local bin="$1" - local name="$2" - local runfiles_dir="" - - if [[ -d "${bin}.runfiles" ]]; then - runfiles_dir="${bin}.runfiles" - elif [[ -f "${bin}.runfiles_manifest" ]]; then - local entry - entry="$(grep -m1 "/${name}$" "${bin}.runfiles_manifest" || true)" - if [[ -n "${entry}" ]]; then - echo "${entry#* }" - return 0 - fi - fi - - if [[ -z "${runfiles_dir}" ]]; then - return 1 - fi - - find "${runfiles_dir}" -type f -name "${name}" -print -quit 2>/dev/null -} - -# Derive the Ferrocene sysroot from the wrapper location. -if [[ -z "${SYSROOT:-}" ]]; then - symbol_report_real="$(resolve_realpath "${symbol_report_bin}" || true)" - if [[ -n "${symbol_report_real}" && "${symbol_report_real}" == */symbol-report.sh ]]; then - SYSROOT="$(cd "$(dirname "${symbol_report_real}")" && pwd)" - else - symbol_report_runfile="$(resolve_runfile "${symbol_report_bin}" "symbol-report.sh" || true)" - if [[ -z "${symbol_report_runfile}" ]]; then - echo "Failed to locate symbol-report.sh in runfiles for ${symbol_report_bin}" >&2 - exit 1 - fi - SYSROOT="$(cd "$(dirname "${symbol_report_runfile}")" && pwd)" - fi -fi - -prefer_wrapper_script() { - local bin="$1" - local name="$2" - if [[ "${bin}" == */${name} ]]; then - local candidate - candidate="$(dirname "${bin}")/${name}.sh" - if [[ -f "${candidate}" ]]; then - echo "${candidate}" - return 0 - fi - fi - echo "${bin}" -} - -symbol_report_bin="$(prefer_wrapper_script "${symbol_report_bin}" "symbol-report")" -blanket_bin="$(prefer_wrapper_script "${blanket_bin}" "blanket")" - -raw_binary_path() { - local bin="$1" - if [[ "${bin}" == *.sh ]]; then - local raw="${bin%.sh}" - if [[ -x "${raw}" ]]; then - echo "${raw}" - return 0 - fi - fi - echo "${bin}" -} - -extra_ld_for_missing_driver() { - local bin="$1" - local sysroot="$2" - if [[ ! -x "${bin}" ]]; then - echo "" - return 0 - fi - local missing - missing="$(ldd "${bin}" 2>/dev/null | awk '/librustc_driver-.*not found/ {print $1; exit}')" - if [[ -z "${missing}" ]]; then - echo "" - return 0 - fi - local existing - existing="$(ls "${sysroot}"/lib/librustc_driver-*.so 2>/dev/null | head -n1 || true)" - if [[ -z "${existing}" ]]; then - echo "" - return 0 - fi - local tmpdir - tmpdir="$(mktemp -d)" - ln -s "${existing}" "${tmpdir}/${missing}" - echo "${tmpdir}" -} - -symbol_report_cmd=("${symbol_report_bin}") -if [[ ! -x "${symbol_report_bin}" ]]; then - if [[ "${symbol_report_bin}" == *.sh && -f "${symbol_report_bin}" ]]; then - symbol_report_cmd=(bash "${symbol_report_bin}") - else - echo "symbol-report wrapper not executable at ${symbol_report_bin}" >&2 - exit 1 - fi -fi - -blanket_cmd=("${blanket_bin}") -if [[ ! -x "${blanket_bin}" ]]; then - if [[ "${blanket_bin}" == *.sh && -f "${blanket_bin}" ]]; then - blanket_cmd=(bash "${blanket_bin}") - else - echo "blanket wrapper not executable at ${blanket_bin}" >&2 - exit 1 - fi -fi - -symbol_report_raw="$(raw_binary_path "${symbol_report_bin}")" -blanket_raw="$(raw_binary_path "${blanket_bin}")" -symbol_report_extra_ld="$(extra_ld_for_missing_driver "${symbol_report_raw}" "${SYSROOT}")" -blanket_extra_ld="$(extra_ld_for_missing_driver "${blanket_raw}" "${SYSROOT}")" - -mapfile -t targets < <( - if [[ -n "${TARGETS_CSV}" ]]; then - echo "${TARGETS_CSV}" | tr ',' '\n' - else - bazel query "${QUERY}" - fi -) - -if [[ ${#targets[@]} -eq 0 ]]; then - echo "No targets found for query: ${QUERY}" >&2 - exit 1 -fi - -mkdir -p "${OUT_DIR}" - -parse_line_coverage() { - local html_path="$1" - # Blanket reports line coverage in the HTML summary; parse it for gating. - python3 "${parse_line_coverage_py}" "${html_path}" -} - -failures=() -total_covered=0 -total_lines=0 -parsed_targets=0 - -for label in "${targets[@]}"; do - if [[ -z "${label}" ]]; then - continue - fi - - pkg="${label#//}" - pkg="${pkg%%:*}" - name="${label##*:}" - - # Resolve the package path and repo root so test.outputs works for - # workspace labels (//pkg:target) and external labels (@repo//pkg:target). - label_pkg_path="$(label_pkg "${label}")" - if [[ -z "${label_pkg_path}" ]]; then - label_pkg_path="${pkg}" - fi - label_repo_root="$(workspace_root_for_label "${label}")" - - if [[ -n "${PROFRAW_DIR}" ]]; then - test_out_dir="${PROFRAW_DIR}" - elif [[ -n "${label_repo_root}" ]]; then - test_out_dir="${PROFRAW_ROOT}/${label_repo_root}/${label_pkg_path}/${name}/test.outputs" - else - test_out_dir="${PROFRAW_ROOT}/${label_pkg_path}/${name}/test.outputs" - fi - - shopt -s nullglob - profraw_files=("${test_out_dir}"/*.profraw) - shopt -u nullglob - - if [[ ${#profraw_files[@]} -eq 0 ]]; then - echo "Skipping ${label}: no .profraw files in ${test_out_dir}" >&2 - continue - fi - - # Build the test target with the chosen config to ensure the instrumented binary exists. - bazel build "${BAZEL_FLAGS[@]}" "${label}" >/dev/null - - bin_rel="$(cquery_expr "${label}" 'target.files.to_list()[0].path')" - if [[ -z "${bin_rel}" ]]; then - echo "Skipping ${label}: could not resolve test binary path" >&2 - continue - fi - - if [[ "${bin_rel}" == /* ]]; then - bin_path="${bin_rel}" - else - bin_path="${exec_root}/${bin_rel}" - fi - - if [[ ! -x "${bin_path}" ]]; then - echo "Skipping ${label}: test binary not found at ${bin_path}" >&2 - continue - fi - - # rust_test can reference a crate by label (preferred) or by name. - # We try label first, then fall back to the raw attribute. - crate_label_raw="$(query_labels_attr "${label}" "crate")" - if [[ -z "${crate_label_raw}" ]]; then - crate_label_raw="$(query_attr_build "${label}" "crate")" - fi - crate_label="$(normalize_scalar "${crate_label_raw}")" - crate_type="" - crate_target="" - if [[ -n "${crate_label}" ]]; then - crate_target="$(normalize_label "${crate_label}" "${pkg}")" - crate_type="lib" - else - crate_target="${label}" - crate_type="$(normalize_scalar "$(query_attr_build "${label}" "crate_type")")" - fi - if [[ -z "${crate_type}" ]]; then - crate_type="bin" - fi - - crate_pkg="$(label_pkg "${crate_target}")" - if [[ -z "${crate_pkg}" ]]; then - crate_pkg="${pkg}" - fi - repo_root="$(workspace_root_for_label "${crate_target}")" - - crate_root_raw="$(query_labels_attr "${crate_target}" "crate_root")" - if [[ -z "${crate_root_raw}" ]]; then - crate_root_raw="$(query_attr_build "${crate_target}" "crate_root")" - fi - crate_root="$(label_to_path "${crate_root_raw}" "${crate_pkg}")" - # First, try conventional crate roots to avoid choosing a random source file. - if [[ -z "${crate_root}" ]]; then - for candidate in \ - "${crate_pkg}/src/lib.rs" \ - "${crate_pkg}/src/main.rs" \ - "${crate_pkg}/lib.rs" \ - "${crate_pkg}/main.rs"; do - if [[ -n "${repo_root}" ]]; then - if [[ -f "${exec_root}/${repo_root}/${candidate}" ]]; then - crate_root="${candidate}" - break - fi - elif [[ -f "${workspace}/${candidate}" ]]; then - crate_root="${candidate}" - break - fi - done - fi - # If there is no conventional root, fall back to the crate's declared srcs. - if [[ -z "${crate_root}" ]]; then - srcs_label="$(query_labels_attr "${crate_target}" "srcs")" - if [[ -n "${srcs_label}" ]]; then - srcs_path="$(label_to_path "${srcs_label}" "${crate_pkg}")" - if [[ -n "${srcs_path}" && "${srcs_path}" == *.rs ]]; then - crate_root="${srcs_path}" - fi - fi - fi - if [[ -z "${crate_root}" ]]; then - # As a last resort, try rust_test srcs when the test target defines them. - # This handles rust_test targets that directly list their sources. - srcs_label="$(query_labels_attr "${label}" "srcs")" - if [[ -n "${srcs_label}" ]]; then - srcs_path="$(label_to_path "${srcs_label}" "${pkg}")" - if [[ -n "${srcs_path}" && "${srcs_path}" == *.rs ]]; then - crate_root="${srcs_path}" - fi - fi - fi - # Without a crate root, symbol-report cannot build the crate. - if [[ -z "${crate_root}" ]]; then - echo "Skipping ${label}: could not determine crate root for ${crate_target}" >&2 - continue - fi - # Convert the crate root into an absolute path. External repos live under - # exec_root/external/ (symlinked from output_base), while workspace - # sources live under $workspace. - if [[ "${crate_root}" != /* ]]; then - if [[ -n "${repo_root}" && "${crate_root}" != "${repo_root}/"* ]]; then - crate_root="${repo_root}/${crate_root}" - fi - if [[ "${crate_root}" == external/* ]]; then - crate_root="${exec_root}/${crate_root}" - else - crate_root="${workspace}/${crate_root}" - fi - fi - - # Keep a workspace- or exec_root-relative path for reporting and mapping. - crate_root_rel="${crate_root}" - if [[ "${crate_root_rel}" == "${workspace}/"* ]]; then - crate_root_rel="${crate_root_rel#${workspace}/}" - elif [[ "${crate_root_rel}" == "${exec_root}/"* ]]; then - crate_root_rel="${crate_root_rel#${exec_root}/}" - fi - - crate_name="$(normalize_scalar "$(query_attr_build "${crate_target}" "crate_name")")" - if [[ -z "${crate_name}" ]]; then - crate_name="${crate_target##*:}" - crate_name="${crate_name#//}" - fi - edition="$(normalize_scalar "$(query_attr_build "${crate_target}" "edition")")" - if [[ -z "${edition}" ]]; then - edition="2021" - fi - - mapfile -t aquery_lines < <(aq_rustc_info "${crate_target}" "${crate_root_rel}" "${exec_root}") - link_out="${aquery_lines[0]:-}" - extra_rustc_flags=("${aquery_lines[@]:1}") - test_link_out="" - if [[ "${crate_target}" != "${label}" ]]; then - mapfile -t test_aquery_lines < <(aq_rustc_info "${label}" "${crate_root_rel}" "${exec_root}") - test_link_out="${test_aquery_lines[0]:-}" - fi - extra_rustc_args=() - for flag in "${extra_rustc_flags[@]}"; do - if [[ "${flag}" == --extern=* ]]; then - extra_rustc_args+=("--extern" "${flag#--extern=}") - elif [[ "${flag}" == --cfg=* ]]; then - extra_rustc_args+=("--cfg" "${flag#--cfg=}") - elif [[ "${flag}" == --test ]]; then - extra_rustc_args+=("--test") - else - extra_rustc_args+=("${flag}") - fi - done - if [[ "${FERROCENE_REPORT_DEBUG:-0}" == "1" ]]; then - echo "Debug: ${label} crate_label_raw=${crate_label_raw}" >&2 - echo "Debug: ${label} crate_label=${crate_label}" >&2 - echo "Debug: ${label} crate_target=${crate_target} crate_type=${crate_type} crate_name=${crate_name} edition=${edition}" >&2 - echo "Debug: ${label} link_out=${link_out}" >&2 - if [[ -n "${test_link_out}" ]]; then - echo "Debug: ${label} test_link_out=${test_link_out}" >&2 - fi - echo "Debug: ${label} crate_root=${crate_root_rel} flags from aquery:" >&2 - printf ' %s\n' "${extra_rustc_flags[@]}" >&2 - echo "Debug: ${label} normalized rustc args:" >&2 - printf ' %q\n' "${extra_rustc_args[@]}" >&2 - fi - - if [[ -n "${link_out}" ]]; then - if [[ "${link_out}" == /* ]]; then - candidate_path="${link_out}" - else - candidate_path="${exec_root}/${link_out}" - fi - if [[ -x "${candidate_path}" ]]; then - bin_rel="${link_out}" - bin_path="${candidate_path}" - fi - fi - if [[ -n "${test_link_out}" ]]; then - if [[ "${test_link_out}" == /* ]]; then - candidate_path="${test_link_out}" - else - candidate_path="${exec_root}/${test_link_out}" - fi - if [[ -x "${candidate_path}" ]]; then - bin_rel="${test_link_out}" - bin_path="${candidate_path}" - fi - fi - - safe_label="${label//\//_}" - safe_label="${safe_label//:/_}" - safe_label="${safe_label//@/_}" - - report_dir="${OUT_DIR}/${safe_label}" - mkdir -p "${report_dir}" - - symbol_report_json="${report_dir}/symbol-report.json" - - sysroot_arg="${SYSROOT}" - if [[ "${SYSROOT}" == "${exec_root}/"* ]]; then - sysroot_arg="${SYSROOT#${exec_root}/}" - fi - - # Remap execroot/workspace paths to keep symbol-report filenames stable. - remap_args=() - if [[ -n "${exec_root}" ]]; then - remap_args+=("--remap-path-prefix=${exec_root}/=.") - fi - if [[ "${workspace}" != "${exec_root}" ]]; then - remap_args+=("--remap-path-prefix=${workspace}/=.") - fi - - # Pass the absolute crate root; relative external paths fail to canonicalize. - ( - cd "${exec_root}" - SYMBOL_REPORT_OUT="${symbol_report_json}" \ - LD_LIBRARY_PATH="${symbol_report_extra_ld}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" \ - "${symbol_report_cmd[@]}" \ - --crate-name "${crate_name}" \ - --edition "${edition}" \ - --crate-type "${crate_type}" \ - "${extra_rustc_args[@]}" \ - --target "${TARGET_TRIPLE}" \ - --sysroot "${sysroot_arg}" \ - -o /dev/null \ - "${remap_args[@]}" \ - "${crate_root}" - ) - - # Normalize symbol-report paths to be workspace-relative (like the demo), - # so blanket can reliably locate sources. - python3 "${normalize_symbol_report_py}" "${symbol_report_json}" "${workspace}" "${exec_root}" - - bin_arg="${bin_path}" - if [[ "${bin_rel}" != /* ]]; then - bin_arg="${bin_rel}" - fi - - # Blanket resolves report filenames by joining them with --ferrocene-src. - # Use a path-equivalence so source files map cleanly to report entries. - # For external crates, profiler paths are absolute under output_base/external, - # so we point --ferrocene-src there instead of the workspace. - ferrocene_src="${workspace}" - if [[ "${crate_root_rel}" == external/* ]]; then - ferrocene_src="${output_base}" - fi - crate_root_dir_rel="$(dirname "${crate_root_rel}")" - path_prefix="${crate_root_rel%%/*}" - if [[ -n "${path_prefix}" && "${path_prefix}" != "${crate_root_rel}" && "${path_prefix}" != "." ]]; then - # Broader remap to cover any file under the top-level directory (e.g. src/...). - path_equiv_args=("--path-equivalence" "${path_prefix},${ferrocene_src}/${path_prefix}") - elif [[ "${crate_root_dir_rel}" == "." ]]; then - path_equiv_args=("--path-equivalence" ".,${ferrocene_src}") - else - path_equiv_args=("--path-equivalence" "${crate_root_dir_rel},${ferrocene_src}/${crate_root_dir_rel}") - fi - - ( - cd "${workspace}" - LD_LIBRARY_PATH="${blanket_extra_ld}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" \ - "${blanket_cmd[@]}" show \ - $(printf -- '--instr-profile=%s ' "${profraw_files[@]}") \ - --object "${bin_arg}" \ - --report "${symbol_report_json}" \ - --ferrocene-src "${ferrocene_src}" \ - "${path_equiv_args[@]}" \ - --html-out "${report_dir}/blanket/index.html" - ) - - line_cov="" - line_cov_info="" - if line_cov="$(parse_line_coverage "${report_dir}/blanket/index.html" 2>/dev/null)"; then - read -r line_pct line_cov_lines line_total_lines <<<"${line_cov}" - line_cov_info="${line_pct}% (${line_cov_lines}/${line_total_lines} lines)" - echo "Line coverage for ${label}: ${line_cov_info}" - if [[ -n "${line_cov_lines}" && -n "${line_total_lines}" ]]; then - total_covered=$((total_covered + line_cov_lines)) - total_lines=$((total_lines + line_total_lines)) - parsed_targets=$((parsed_targets + 1)) - fi - if [[ -n "${MIN_LINE_COVERAGE}" ]]; then - if ! python3 - "${line_pct}" "${MIN_LINE_COVERAGE}" <<'PY' -import sys -try: - val = float(sys.argv[1]) - minv = float(sys.argv[2]) -except ValueError: - sys.exit(0) -sys.exit(0 if val >= minv else 1) -PY - then - failures+=("${label} (${line_cov_info})") - fi - fi - else - echo "Warning: could not parse line coverage from ${report_dir}/blanket/index.html" >&2 - if [[ -n "${MIN_LINE_COVERAGE}" ]]; then - failures+=("${label} (no line coverage parsed)") - fi - fi - - echo "Report for ${label}: ${report_dir}/blanket/index.html" - done - -if [[ ${total_lines} -gt 0 ]]; then - # Aggregate line coverage across all targets that parsed successfully. - overall_pct="$(python3 - "${total_covered}" "${total_lines}" <<'PY' -import sys -cov = int(sys.argv[1]) -total = int(sys.argv[2]) -if total == 0: - print("0.00") -else: - print(f"{(cov / total) * 100:.2f}") -PY -)" - echo "---" - echo "Overall line coverage: ${overall_pct}% (${total_covered}/${total_lines} lines across ${parsed_targets} targets)" - echo "---" -else - echo "---" - echo "Overall line coverage: n/a (no per-target line coverage parsed)" - echo "---" -fi - -if [[ ${#failures[@]} -gt 0 ]]; then - # Fail CI when any target is below the minimum line-coverage threshold. - echo "Line coverage gate failed (min ${MIN_LINE_COVERAGE}%):" >&2 - printf ' %s\n' "${failures[@]}" >&2 - exit 3 -fi diff --git a/coverage/ferrocene_report_wrapper.sh.tpl b/coverage/ferrocene_report_wrapper.sh.tpl deleted file mode 100755 index 19559b4f..00000000 --- a/coverage/ferrocene_report_wrapper.sh.tpl +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* - -set -euo pipefail - -report_runfile=@@RUNFILE@@ -if [[ -n "${RUNFILES_DIR:-}" ]]; then - ferrocene_report="${RUNFILES_DIR}/${report_runfile}" -elif [[ -n "${RUNFILES_MANIFEST_FILE:-}" ]]; then - ferrocene_report="$(grep -m1 "^${report_runfile} " "${RUNFILES_MANIFEST_FILE}" | cut -d' ' -f2-)" -else - ferrocene_report="${report_runfile}" -fi - -if [[ ! -x "${ferrocene_report}" ]]; then - echo "ferrocene_report not found at ${ferrocene_report}" >&2 - exit 1 -fi - -@@EXEC_LINE@@ diff --git a/coverage/generate_coverage_html.sh b/coverage/generate_coverage_html.sh new file mode 100755 index 00000000..7597637a --- /dev/null +++ b/coverage/generate_coverage_html.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# Generates an HTML coverage report from `bazel coverage` output produced by +# the LLVM pipeline: the custom reporter produces a zip with html_report/, +# lcov_report/ and text_report/ inside. +# +# Usage (from the CONSUMER workspace): +# bazel coverage --config=llvm_cov //... --build_tests_only +# bazel run @score_tooling//coverage:generate_coverage_html -- \ +# --yaml \ +# [--archive ] [--platform ] \ +# [--testlogs-subdir ] [output-dir] +# +# Arguments: +# --yaml Justification YAML, relative to the workspace +# root (required). +# --archive Also create a zip archive named .zip +# containing the HTML report, raw LCOV data and JUnit XMLs. +# --platform Target platform for justification filtering +# (default: linux). Also affects the default output +# directory (coverage_). +# --testlogs-subdir Subdirectory of bazel-testlogs to collect JUnit +# XMLs from when archiving (default: entire +# bazel-testlogs tree). +# output-dir Directory to write the HTML report to +# (default: coverage_) +# +# Environment: +# COVERAGE_THRESHOLD Minimum effective line coverage percentage +# (default: 100). The script exits non-zero when +# effective coverage is below this threshold. + +set -euo pipefail + +ARCHIVE_NAME="" +PLATFORM="linux" +OUTPUT_DIR="" +JUSTIFICATION_YAML_REL="" +TESTLOGS_SUBDIR="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --yaml) + JUSTIFICATION_YAML_REL="${2:?--yaml requires a path argument}" + shift 2 + ;; + --archive) + ARCHIVE_NAME="${2:?--archive requires a name argument}" + shift 2 + ;; + --platform) + PLATFORM="${2:?--platform requires a platform argument (e.g. linux or qnx)}" + shift 2 + ;; + --testlogs-subdir) + TESTLOGS_SUBDIR="${2:?--testlogs-subdir requires a path argument}" + shift 2 + ;; + *) + OUTPUT_DIR="$1" + shift + ;; + esac +done + +if [[ -z "${JUSTIFICATION_YAML_REL}" ]]; then + echo "ERROR: --yaml is required." >&2 + exit 1 +fi + +# Set default output directory based on platform if not explicitly provided. +if [[ -z "${OUTPUT_DIR}" ]]; then + OUTPUT_DIR="coverage_${PLATFORM}" +fi + +# Change to the workspace root so that all subsequent bazel calls and +# relative paths work correctly. +cd "${BUILD_WORKSPACE_DIRECTORY}" + +# Resolve OUTPUT_DIR to absolute path (relative to workspace root). +OUTPUT_DIR="${BUILD_WORKSPACE_DIRECTORY}/${OUTPUT_DIR}" + +# The coverage report is at _coverage_report.dat: a zip containing +# html_report/, lcov_report/ and text_report/ (LLVM pipeline). +COVERAGE_REPORT="${BUILD_WORKSPACE_DIRECTORY}/bazel-out/_coverage/_coverage_report.dat" + +if [[ ! -f "${COVERAGE_REPORT}" ]]; then + echo "ERROR: Coverage report not found at ${COVERAGE_REPORT}" >&2 + echo " Run 'bazel coverage --config=llvm_cov //... --build_tests_only' first." >&2 + exit 1 +fi + +TMPDIR_EXTRACT="${TMPDIR:-/tmp}/coverage_extract_$$" +mkdir -p "${TMPDIR_EXTRACT}" +trap 'rm -rf "${TMPDIR_EXTRACT}"' EXIT + +rm -rf "${OUTPUT_DIR}" + +if ! file -b "${COVERAGE_REPORT}" | grep -q "Zip archive"; then + echo "ERROR: ${COVERAGE_REPORT} is not the LLVM pipeline zip report." >&2 + echo " Run 'bazel coverage --config=llvm_cov //... --build_tests_only' first." >&2 + exit 1 +fi + +# Extract HTML from the zip produced by our custom reporter. +unzip -q -o "${COVERAGE_REPORT}" -d "${TMPDIR_EXTRACT}" + +if [[ -d "${TMPDIR_EXTRACT}/html_report" ]]; then + cp -r "${TMPDIR_EXTRACT}/html_report" "${OUTPUT_DIR}" +else + echo "ERROR: html_report/ not found in ${COVERAGE_REPORT}" >&2 + exit 1 +fi + +echo "Coverage report written to: ${OUTPUT_DIR}" + +# --------------------------------------------------------------------------- +# Run coverage justification processing. +# --------------------------------------------------------------------------- +JUSTIFICATION_YAML="${BUILD_WORKSPACE_DIRECTORY}/${JUSTIFICATION_YAML_REL}" + +if [[ ! -f "${JUSTIFICATION_YAML}" ]]; then + echo "ERROR: ${JUSTIFICATION_YAML} not found." >&2 + exit 1 +fi + +echo "" +echo "Running coverage justification processing..." + +JUSTIFICATION_DIR="${TMPDIR_EXTRACT}/justification_report" +mkdir -p "${JUSTIFICATION_DIR}" + +# Run justify.py / effective_coverage.py via nested bazel invocations from the +# consumer workspace. This deliberately avoids runfiles resolution across +# module boundaries (canonical repo names vary between Bazel versions). +bazel run @score_tooling//coverage:justify -- \ + --yaml "${JUSTIFICATION_YAML}" \ + --source-root "${BUILD_WORKSPACE_DIRECTORY}" \ + --platform "${PLATFORM}" \ + --output "${JUSTIFICATION_DIR}/manifest.json" + +bazel run @score_tooling//coverage:effective_coverage -- \ + --html-dir "${OUTPUT_DIR}" \ + --manifest "${JUSTIFICATION_DIR}/manifest.json" \ + --output "${JUSTIFICATION_DIR}/report.json" + +# Display effective coverage summary and enforce the threshold. +if [[ ! -f "${JUSTIFICATION_DIR}/summary.txt" ]]; then + echo "ERROR: Effective coverage summary was not produced." >&2 + exit 1 +fi + +echo "" +cat "${JUSTIFICATION_DIR}/summary.txt" + +# Extract effective coverage percentage for threshold check. +EFFECTIVE_PCT=$(grep -oP 'Effective line coverage:\s+\K[0-9.]+' \ + "${JUSTIFICATION_DIR}/summary.txt" 2>/dev/null || echo "0") + +# Threshold check (default: 100%). Fails the run when below. +THRESHOLD="${COVERAGE_THRESHOLD:-100}" +if ! awk "BEGIN {exit (${EFFECTIVE_PCT} >= ${THRESHOLD}) ? 0 : 1}"; then + echo "ERROR: Effective coverage ${EFFECTIVE_PCT}% is below threshold ${THRESHOLD}%" >&2 + RC=1 +else + RC=0 +fi + +# --------------------------------------------------------------------------- +# Optional: create a zip archive with the HTML report, raw LCOV data and +# JUnit XML test results. +# --------------------------------------------------------------------------- +if [[ -n "${ARCHIVE_NAME}" ]]; then + mkdir -p artifacts + + # Copy JUnit XML test results preserving directory structure. + find "bazel-testlogs/${TESTLOGS_SUBDIR}" -name 'test.xml' -exec cp --parents {} artifacts/ \; + + # Copy the HTML coverage report + cp -r "${OUTPUT_DIR}" artifacts/ + + # Include the LCOV .dat file from the reporter zip. + if [[ -f "${TMPDIR_EXTRACT}/lcov_report/lcov.dat" ]]; then + cp "${TMPDIR_EXTRACT}/lcov_report/lcov.dat" artifacts/coverage_report.dat + fi + + # Include the justification report (manifest + effective coverage json). + if [[ -d "${JUSTIFICATION_DIR}" ]]; then + cp -r "${JUSTIFICATION_DIR}" artifacts/ + fi + + zip -r "${ARCHIVE_NAME}.zip" artifacts/ + rm -rf artifacts/ + echo "Coverage archive written to: ${ARCHIVE_NAME}.zip" +fi + +exit "${RC}" diff --git a/coverage/integration_tests/.bazelrc b/coverage/integration_tests/.bazelrc new file mode 100644 index 00000000..8c38b51e --- /dev/null +++ b/coverage/integration_tests/.bazelrc @@ -0,0 +1,78 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +common --registry=https://raw.githubusercontent.com/eclipse-score/bazel_registry/main/ +common --registry=https://bcr.bazel.build + +# Rust toolchain for regular builds and coverage alike: score_toolchains_rust's +# standard Ferrocene toolchain ships with llvm-cov/llvm-profdata attached +# (>= 0.10.0), which is what activates Rust coverage instrumentation. +common --extra_toolchains=@score_toolchains_rust//toolchains/ferrocene:ferrocene_x86_64_unknown_linux_gnu + +test --test_output=errors + +# ============================================================================ +# LLVM source-based coverage (Linux, Clang + Ferrocene) — unified Rust + C++ +# This is the CANONICAL consumer snippet: copy this block (adjusting the +# --coverage_report_generator label) into your repository's bazelrc. +# Use with: bazel coverage --config=llvm_cov //... --build_tests_only +# ============================================================================ + +coverage:llvm_cov --nocache_test_results +coverage:llvm_cov --cxxopt=-O0 +coverage:llvm_cov --combined_report=lcov +coverage:llvm_cov --experimental_fetch_all_coverage_outputs + +# Per-test .so files can carry clashing instrumentation between production +# and test code; the first object loaded wins, causing flaky coverage gaps. +# Static linking avoids this entirely. +coverage:llvm_cov --dynamic_mode=off + +coverage:llvm_cov --experimental_generate_llvm_lcov +# NOTE: --experimental_use_llvm_covmap causes Bazel to instrument ALL targets. +# Source filtering is handled at report time using the allowlist generated by +# the score_coverage_scope target. +coverage:llvm_cov --experimental_use_llvm_covmap +coverage:llvm_cov --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux + +# The merger lives in score_tooling (no consumer-specific wiring needed); the +# reporter_wrapper is instantiated in THIS repository (tools/coverage/BUILD) +# because it carries the coverage scope and LLVM tool labels. +coverage:llvm_cov --coverage_output_generator=@score_tooling//coverage:merger +coverage:llvm_cov --coverage_report_generator=//tools/coverage:reporter_wrapper + +# Keep raw profraw files instead of converting to LCOV; the merger handles +# profraw -> profdata directly. +coverage:llvm_cov --test_env=GENERATE_LLVM_LCOV=0 +coverage:llvm_cov --test_env=COVERAGE_GCOV_PATH=/usr/bin/true + +# Continuous-mode profiling: required to cover abnormal termination (death +# tests). Needs runtime counter relocation in both languages. +coverage:llvm_cov --test_env=LLVM_PROFILE_CONTINUOUS_MODE=1 +coverage:llvm_cov --features=enable_llvm_coverage_for_death_tests +coverage:llvm_cov --@rules_rust//rust/settings:extra_rustc_flag=-Cllvm-args=-runtime-counter-relocation + +# Under the LLVM toolchain the `coverage` feature expands to +# -fprofile-instr-generate -fcoverage-mapping — exactly the covmap +# instrumentation this pipeline needs. +coverage:llvm_cov --features=coverage + +# Keep unlinked functions in the coverage map so unreferenced code shows as +# uncovered instead of vanishing. +coverage:llvm_cov --@rules_rust//rust/settings:extra_rustc_flag=-Clink-dead-code + +# Enable branch coverage instrumentation for Rust. NOTE: -Z flags are +# unstable rustc options; they work here because the Ferrocene toolchain in +# use is a rolling (nightly-based) build. If you pin a stable-channel +# Ferrocene release, drop this flag (losing the Rust branch column). +coverage:llvm_cov --@rules_rust//rust/settings:extra_rustc_flag=-Zcoverage-options=branch diff --git a/coverage/integration_tests/.bazelversion b/coverage/integration_tests/.bazelversion new file mode 100644 index 00000000..f9c71a52 --- /dev/null +++ b/coverage/integration_tests/.bazelversion @@ -0,0 +1 @@ +8.5.1 diff --git a/coverage/tests/parse_line_coverage_test.sh b/coverage/integration_tests/BUILD old mode 100755 new mode 100644 similarity index 60% rename from coverage/tests/parse_line_coverage_test.sh rename to coverage/integration_tests/BUILD index eafcb053..047af1d5 --- a/coverage/tests/parse_line_coverage_test.sh +++ b/coverage/integration_tests/BUILD @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # ******************************************************************************* # Copyright (c) 2026 Contributors to the Eclipse Foundation # @@ -11,16 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -set -euo pipefail -fixture="${TEST_SRCDIR}/${TEST_WORKSPACE}/coverage/tests/fixtures/blanket_index.html" -parser="${TEST_SRCDIR}/${TEST_WORKSPACE}/coverage/scripts/parse_line_coverage.py" - -output="$(python3 "${parser}" "${fixture}")" - -if [[ "${output}" != "100.00 8 8" ]]; then - echo "unexpected coverage summary: ${output}" >&2 - exit 1 -fi - -echo "ok" +# Required by score_coverage_reporter: the reporter locates the real +# workspace root at runtime through the MODULE.bazel runfile. +exports_files(["MODULE.bazel"]) diff --git a/coverage/integration_tests/MODULE.bazel b/coverage/integration_tests/MODULE.bazel new file mode 100644 index 00000000..c6cd3731 --- /dev/null +++ b/coverage/integration_tests/MODULE.bazel @@ -0,0 +1,68 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +# Integration test workspace for the score_tooling LLVM coverage pipeline. +# This module mirrors EXACTLY what a consumer repository has to declare — +# its MODULE.bazel, .bazelrc and tools/coverage/BUILD are the reference +# implementation for the adoption guide in //coverage:README.md. +module(name = "coverage_integration_tests") + +bazel_dep(name = "score_tooling", version = "1.3.1") +local_path_override( + module_name = "score_tooling", + path = "../..", +) + +bazel_dep(name = "platforms", version = "1.0.0") +bazel_dep(name = "rules_cc", version = "0.2.17") +bazel_dep(name = "rules_python", version = "1.8.5") +bazel_dep(name = "rules_rust", version = "0.68.2-score") + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.12", +) + +# ******************************************************************************* +# Coverage toolchains (only active during `bazel coverage --config=llvm_cov`) +# ******************************************************************************* + +bazel_dep(name = "score_toolchains_rust", version = "0.10.0", dev_dependency = True) +bazel_dep(name = "toolchains_llvm", version = "1.8.0", dev_dependency = True) + +# C++ LLVM/Clang toolchain used for coverage builds (source-based covmap). +llvm = use_extension( + "@toolchains_llvm//toolchain/extensions:llvm.bzl", + "llvm", + dev_dependency = True, +) +llvm.toolchain( + cxx_standard = {"": "c++17"}, + extra_known_features = [ + # Provided by score_tooling; enables runtime counter relocation for + # continuous-mode profiling (death test coverage). + "@score_tooling//coverage:enable_llvm_coverage_for_death_tests", + ], + llvm_version = "22.1.7", + stdlib = {"": "stdc++"}, +) +use_repo(llvm, "llvm_toolchain", "llvm_toolchain_llvm") + +# NOTE: no custom Ferrocene instance is needed for Rust coverage. The +# standard toolchains shipped by score_toolchains_rust >= 0.10.0 already +# attach llvm-cov/llvm-profdata from the ferrocene_toolchain_builder +# coverage-tools tarball (built from the same LLVM as rustc), and +# rules_rust instruments crates under `bazel coverage` whenever the +# rust_toolchain declares llvm_cov. Registration happens in .bazelrc: +# common --extra_toolchains=@score_toolchains_rust//toolchains/ferrocene:ferrocene_x86_64_unknown_linux_gnu diff --git a/coverage/integration_tests/run_integration_test.sh b/coverage/integration_tests/run_integration_test.sh new file mode 100755 index 00000000..946e1642 --- /dev/null +++ b/coverage/integration_tests/run_integration_test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# End-to-end test of the score_tooling LLVM coverage pipeline, run against +# this consumer-style workspace. Asserts the properties the pipeline +# guarantees: +# 1. Untested in-scope files (C++ AND Rust) appear at exact 0% in the LCOV. +# 2. The effective-coverage gate fails at threshold 100 and passes at a low +# threshold. +# 3. The justified line raises effective coverage above raw coverage. + +set -euo pipefail +cd "$(dirname "$0")" + +echo "=== Running coverage build ===" +bazel coverage --config=llvm_cov //... --build_tests_only + +YAML="tools/coverage/coverage_justifications.yaml" + +echo "=== Gate must FAIL at threshold 100 (uncovered fixtures exist) ===" +if COVERAGE_THRESHOLD=100 bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml "${YAML}" --archive coverage_artifacts; then + echo "ERROR: coverage gate passed at threshold 100 despite uncovered files" >&2 + exit 1 +fi +echo "OK: gate failed as expected" + +echo "=== Gate must PASS at a low threshold ===" +COVERAGE_THRESHOLD=10 bazel run @score_tooling//coverage:generate_coverage_html -- \ + --yaml "${YAML}" +echo "OK: gate passed as expected" + +echo "=== Untested files must appear at exact 0% in the LCOV ===" +unzip -p coverage_artifacts.zip artifacts/coverage_report.dat > lcov.dat + +check_zero_coverage() { + local file="$1" + if ! grep -q "SF:.*${file}" lcov.dat; then + echo "ERROR: ${file} missing from LCOV (baseline mechanism broken)" >&2 + exit 1 + fi + # The record for the file must report zero lines hit. + if ! awk -v f="${file}" ' + $0 ~ "^SF:" && $0 ~ f {rec=1} + rec && /^LH:/ {print $0; exit ($0 == "LH:0") ? 0 : 1} + rec && /^end_of_record/ {exit 1}' lcov.dat; then + echo "ERROR: ${file} is present but not at 0% coverage" >&2 + exit 1 + fi + echo "OK: ${file} present at 0%" +} + +check_zero_coverage "src/uncovered.cpp" +check_zero_coverage "rust/main.rs" + +echo "=== Covered files must be present with hits ===" +grep -q "SF:.*src/coverable.cpp" lcov.dat || { echo "ERROR: coverable.cpp missing" >&2; exit 1; } +grep -q "SF:.*rust/lib.rs" lcov.dat || { echo "ERROR: lib.rs missing" >&2; exit 1; } +echo "OK" + +echo "=== Justified line must raise effective coverage above raw ===" +SUMMARY="$(unzip -p coverage_artifacts.zip artifacts/justification_report/summary.txt)" +echo "${SUMMARY}" +JUSTIFIED="$(echo "${SUMMARY}" | grep -oP 'Justified lines:\s+\K[0-9]+')" +if [[ "${JUSTIFIED}" -lt 1 ]]; then + echo "ERROR: expected at least one justified line, got ${JUSTIFIED}" >&2 + exit 1 +fi +RAW="$(echo "${SUMMARY}" | grep -oP 'Raw line coverage:\s+\K[0-9.]+')" +EFFECTIVE="$(echo "${SUMMARY}" | grep -oP 'Effective line coverage:\s+\K[0-9.]+')" +if ! awk "BEGIN {exit (${EFFECTIVE} > ${RAW}) ? 0 : 1}"; then + echo "ERROR: effective coverage ${EFFECTIVE}% not above raw ${RAW}%" >&2 + exit 1 +fi +echo "OK: effective ${EFFECTIVE}% > raw ${RAW}%" + +echo "" +echo "=== All integration checks passed ===" diff --git a/coverage/integration_tests/rust/BUILD b/coverage/integration_tests/rust/BUILD new file mode 100644 index 00000000..6d05d684 --- /dev/null +++ b/coverage/integration_tests/rust/BUILD @@ -0,0 +1,37 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test") + +package(default_visibility = ["//visibility:public"]) + +rust_library( + name = "integration_lib", + srcs = ["lib.rs"], + edition = "2021", +) + +rust_test( + name = "integration_lib_test", + crate = ":integration_lib", + edition = "2021", +) + +# No test runs this binary — its main.rs must show up at exact 0% via the +# --empty-profile baseline over the coverage-built executable. +rust_binary( + name = "untested_tool", + srcs = ["main.rs"], + edition = "2021", + deps = [":integration_lib"], +) diff --git a/coverage/integration_tests/rust/lib.rs b/coverage/integration_tests/rust/lib.rs new file mode 100644 index 00000000..337ba3fa --- /dev/null +++ b/coverage/integration_tests/rust/lib.rs @@ -0,0 +1,45 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +/// Classifies an integer. Three branches; the unit test exercises only two so +/// the report is guaranteed to contain covered and uncovered Rust branches. +pub fn classify(value: i32) -> &'static str { + if value < 0 { + "negative" + } else if value == 0 { + "zero" + } else { + "positive" + } +} + +/// Never called by any test: must appear as uncovered (0 hits) in the report +/// thanks to -Clink-dead-code keeping it in the coverage map. +pub fn never_called(value: i32) -> i32 { + value * 2 +} + +#[cfg(test)] +mod tests { + use super::classify; + + #[test] + fn classifies_negative() { + assert_eq!(classify(-5), "negative"); + } + + #[test] + fn classifies_zero() { + assert_eq!(classify(0), "zero"); + } +} diff --git a/coverage/integration_tests/rust/main.rs b/coverage/integration_tests/rust/main.rs new file mode 100644 index 00000000..a36ecb8e --- /dev/null +++ b/coverage/integration_tests/rust/main.rs @@ -0,0 +1,19 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +// This binary has no test at all: its source must appear at exactly 0% in the +// coverage report via the --empty-profile baseline over the coverage-built +// executable (rust_binary targets provide no CcInfo archive). +fn main() { + println!("{}", integration_lib::classify(1)); +} diff --git a/coverage/tests/ferrocene_report_help_test.sh b/coverage/integration_tests/src/BUILD old mode 100755 new mode 100644 similarity index 50% rename from coverage/tests/ferrocene_report_help_test.sh rename to coverage/integration_tests/src/BUILD index 9bdfeac4..983b6b15 --- a/coverage/tests/ferrocene_report_help_test.sh +++ b/coverage/integration_tests/src/BUILD @@ -1,4 +1,3 @@ -#!/usr/bin/env bash # ******************************************************************************* # Copyright (c) 2026 Contributors to the Eclipse Foundation # @@ -11,20 +10,28 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -set -euo pipefail -script="${TEST_SRCDIR}/${TEST_WORKSPACE}/coverage/ferrocene_report.sh" +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_cc//cc:cc_test.bzl", "cc_test") -output="$(bash "${script}" --help)" +package(default_visibility = ["//visibility:public"]) -if ! echo "${output}" | grep -q "Generate Ferrocene Rust coverage reports"; then - echo "help output did not contain expected header" >&2 - exit 1 -fi +cc_library( + name = "coverable", + srcs = ["coverable.cpp"], + hdrs = ["coverable.h"], +) -if ! echo "${output}" | grep -q -- "--min-line-coverage"; then - echo "help output missing --min-line-coverage" >&2 - exit 1 -fi +# No test links against this library — it must show up at exact 0% in the +# report via the --empty-profile baseline. +cc_library( + name = "uncovered", + srcs = ["uncovered.cpp"], + hdrs = ["uncovered.h"], +) -echo "ok" +cc_test( + name = "coverable_test", + srcs = ["coverable_test.cpp"], + deps = [":coverable"], +) diff --git a/coverage/integration_tests/src/coverable.cpp b/coverage/integration_tests/src/coverable.cpp new file mode 100644 index 00000000..f31b9978 --- /dev/null +++ b/coverage/integration_tests/src/coverable.cpp @@ -0,0 +1,27 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include "src/coverable.h" + +namespace coverage_integration { + +const char* classify(int value) { + if (value < 0) { + return "negative"; + } + if (value == 0) { + return "zero"; + } + return "positive"; // COV_JUSTIFIED itest-positive-branch +} + +} // namespace coverage_integration diff --git a/coverage/integration_tests/src/coverable.h b/coverage/integration_tests/src/coverable.h new file mode 100644 index 00000000..bb43b27b --- /dev/null +++ b/coverage/integration_tests/src/coverable.h @@ -0,0 +1,24 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef COVERAGE_INTEGRATION_TESTS_SRC_COVERABLE_H +#define COVERAGE_INTEGRATION_TESTS_SRC_COVERABLE_H + +namespace coverage_integration { + +// Classifies an integer. Three branches; the test exercises only two so the +// report is guaranteed to contain both covered and uncovered branches. +const char* classify(int value); + +} // namespace coverage_integration + +#endif // COVERAGE_INTEGRATION_TESTS_SRC_COVERABLE_H diff --git a/coverage/integration_tests/src/coverable_test.cpp b/coverage/integration_tests/src/coverable_test.cpp new file mode 100644 index 00000000..eb22a5e1 --- /dev/null +++ b/coverage/integration_tests/src/coverable_test.cpp @@ -0,0 +1,28 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "src/coverable.h" + +// Deliberately exercises only the negative and zero branches; the positive +// branch stays uncovered (and justified via the COV_JUSTIFIED marker). +int main() { + using coverage_integration::classify; + if (std::strcmp(classify(-5), "negative") != 0) { + return 1; + } + if (std::strcmp(classify(0), "zero") != 0) { + return 1; + } + return 0; +} diff --git a/coverage/integration_tests/src/uncovered.cpp b/coverage/integration_tests/src/uncovered.cpp new file mode 100644 index 00000000..6eaf59d9 --- /dev/null +++ b/coverage/integration_tests/src/uncovered.cpp @@ -0,0 +1,24 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include "src/uncovered.h" + +namespace coverage_integration { + +int never_called(int value) { + if (value > 10) { + return value * 2; + } + return value - 1; +} + +} // namespace coverage_integration diff --git a/coverage/integration_tests/src/uncovered.h b/coverage/integration_tests/src/uncovered.h new file mode 100644 index 00000000..0c5a4a27 --- /dev/null +++ b/coverage/integration_tests/src/uncovered.h @@ -0,0 +1,24 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef COVERAGE_INTEGRATION_TESTS_SRC_UNCOVERED_H +#define COVERAGE_INTEGRATION_TESTS_SRC_UNCOVERED_H + +namespace coverage_integration { + +// Linked into no test on purpose: this library must still appear in the +// coverage report at exactly 0% via the --empty-profile baseline mechanism. +int never_called(int value); + +} // namespace coverage_integration + +#endif // COVERAGE_INTEGRATION_TESTS_SRC_UNCOVERED_H diff --git a/coverage/integration_tests/tools/coverage/BUILD b/coverage/integration_tests/tools/coverage/BUILD new file mode 100644 index 00000000..d7762b04 --- /dev/null +++ b/coverage/integration_tests/tools/coverage/BUILD @@ -0,0 +1,38 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@score_tooling//coverage:defs.bzl", "score_coverage_reporter", "score_coverage_scope") + +# The production targets whose transitive sources define the coverage scope. +# Untested targets listed here (src:uncovered, rust:untested_tool) appear in +# the report at exact 0% via the --empty-profile baseline. +score_coverage_scope( + name = "coverage_scope", + testonly = True, + visibility = ["//visibility:private"], + deps = [ + "//rust:integration_lib", + "//rust:untested_tool", + "//src:coverable", + "//src:uncovered", + ], +) + +score_coverage_reporter( + name = "reporter_wrapper", + testonly = True, + coverage_scope = ":coverage_scope", + llvm_cov = "@llvm_toolchain//:llvm-cov", + llvm_cxxfilt = "@llvm_toolchain_llvm//:bin/llvm-cxxfilt", + llvm_profdata = "@llvm_toolchain//:llvm-profdata", +) diff --git a/coverage/scripts/parse_line_coverage.py b/coverage/integration_tests/tools/coverage/coverage_justifications.yaml old mode 100755 new mode 100644 similarity index 51% rename from coverage/scripts/parse_line_coverage.py rename to coverage/integration_tests/tools/coverage/coverage_justifications.yaml index 09271a2a..7dadf7c5 --- a/coverage/scripts/parse_line_coverage.py +++ b/coverage/integration_tests/tools/coverage/coverage_justifications.yaml @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # ******************************************************************************* # Copyright (c) 2026 Contributors to the Eclipse Foundation # @@ -11,22 +10,16 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -import re -import sys -from pathlib import Path - -if len(sys.argv) != 2: - print("usage: parse_line_coverage.py ", file=sys.stderr) - sys.exit(2) - -path = Path(sys.argv[1]) -try: - text = path.read_text(encoding="utf-8", errors="ignore") -except FileNotFoundError: - sys.exit(1) - -m = re.search(r"([0-9]+(?:\.[0-9]+)?)%\s*\((\d+)/(\d+)\s+lines\)", text) -if not m: - sys.exit(2) - -print(f"{m.group(1)} {m.group(2)} {m.group(3)}") +# Coverage justifications for the integration test workspace. +# +# Justified lines count as covered in the EFFECTIVE coverage metric. The line +# is marked in-code with a `COV_JUSTIFIED ` comment (src/coverable.cpp); +# this file provides the id, category and reason. +version: 1 +justifications: + - id: itest-positive-branch + category: other + platforms: [linux] + reason: | + Intentionally uncovered branch: the integration test asserts that a + justified line raises effective coverage above raw coverage. diff --git a/coverage/justify.py b/coverage/justify.py new file mode 100644 index 00000000..488b7c55 --- /dev/null +++ b/coverage/justify.py @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Coverage justification processor. + +Parses the YAML justification database and source files for COV_JUSTIFIED markers. +Resolves all justified lines and produces a manifest mapping file:line → justification. + +Usage: + python justify.py --yaml --source-root --output + +Supports two ways to specify justified lines: +1. YAML locations: directly specify file + line ranges in the YAML +2. In-code markers: COV_JUSTIFIED , COV_JUSTIFIED_START / COV_JUSTIFIED_STOP +""" + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Set, Tuple + +import yaml + + +# Marker patterns +COV_JUSTIFIED_LINE_RE = re.compile(r"COV_JUSTIFIED\s+([\w-]+)") +COV_JUSTIFIED_START_RE = re.compile(r"COV_JUSTIFIED_START\s+([\w-]+)") +COV_JUSTIFIED_STOP_RE = re.compile(r"COV_JUSTIFIED_STOP") + +VALID_CATEGORIES = { + "defensive_programming", + "tool_false_positive", + "platform_specific", + "other", +} + +VALID_PLATFORMS = { + "linux", + "qnx", +} + + +def main() -> None: + """Main entry point.""" + args = parse_args() + + justifications_data = load_yaml(args.yaml) + validate_yaml(justifications_data) + + # Build lookup: id -> justification entry + justifications_by_id: Dict[str, Dict[str, Any]] = {} + for entry in justifications_data.get("justifications", []): + justifications_by_id[entry["id"]] = entry + + # Filter justifications by platform if --platform is specified. + if args.platform: + justifications_by_id = { + jid: entry for jid, entry in justifications_by_id.items() if _matches_platform(entry, args.platform) + } + + # Resolve all justified lines + resolved: Dict[str, Dict[int, Dict[str, str]]] = {} + warnings: List[str] = [] + errors: List[str] = [] + + # 1. Process YAML direct locations + for jid, entry in justifications_by_id.items(): + for location in entry.get("locations", []): + file_path = location["file"] + full_path = Path(args.source_root) / file_path + + if not full_path.exists(): + errors.append(f"File not found for justification '{entry['id']}': {file_path}") + continue + + lines = resolve_location_lines(location) + if file_path not in resolved: + resolved[file_path] = {} + for line in lines: + resolved[file_path][line] = { + "id": entry["id"], + "category": entry["category"], + "reason": entry["reason"].strip(), + } + + # 2. Scan source files for in-code COV_JUSTIFIED markers + source_files = collect_source_files(args.source_root, args.file_filter) + for source_file in source_files: + rel_path = str(source_file.relative_to(args.source_root)) + scan_warnings, scan_lines = scan_file_for_markers(source_file, rel_path, justifications_by_id) + warnings.extend(scan_warnings) + + if scan_lines: + if rel_path not in resolved: + resolved[rel_path] = {} + for line_num, justification_info in scan_lines.items(): + resolved[rel_path][line_num] = justification_info + + # Output manifest + manifest = { + "version": 1, + "source_root": str(args.source_root), + "justified_files": { + filepath: {str(k): v for k, v in lines.items()} for filepath, lines in sorted(resolved.items()) + }, + "warnings": warnings, + "errors": errors, + } + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + + # Print diagnostics + total_justified_lines = sum(len(lines) for lines in resolved.values()) + print( + f"INFO: Resolved {total_justified_lines} justified lines across {len(resolved)} files.", + file=sys.stderr, + ) + if warnings: + for w in warnings: + print(f"WARNING: {w}", file=sys.stderr) + if errors: + for e in errors: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + +def resolve_location_lines(location: Dict[str, Any]) -> List[int]: + """Resolve line numbers from a YAML location entry.""" + if "lines" in location: + return location["lines"] + elif "line_start" in location and "line_end" in location: + return list(range(location["line_start"], location["line_end"] + 1)) + elif "line" in location: + return [location["line"]] + return [] + + +def _matches_platform(entry: Dict[str, Any], platform: str) -> bool: + """Check if a justification entry applies to the given platform. + + The ``platforms`` field is mandatory and validated by ``validate_yaml``. + """ + platforms = entry.get("platforms", []) + return platform in platforms + + +def scan_file_for_markers( + file_path: Path, + rel_path: str, + justifications_by_id: Dict[str, Dict[str, Any]], +) -> Tuple[List[str], Dict[int, Dict[str, str]]]: + """Scan a source file for COV_JUSTIFIED markers.""" + warnings = [] + justified_lines: Dict[int, Dict[str, str]] = {} + + try: + with open(file_path, "r", encoding="utf-8", errors="replace") as f: + lines = f.readlines() + except (IOError, OSError): + return warnings, justified_lines + + region_stack: List[Tuple[int, str]] = [] # (start_line, justification_id) + + for line_num, line in enumerate(lines, start=1): + # Check for COV_JUSTIFIED_START + start_match = COV_JUSTIFIED_START_RE.search(line) + if start_match: + jid = start_match.group(1) + if jid not in justifications_by_id: + warnings.append(f"{rel_path}:{line_num}: COV_JUSTIFIED_START references unknown ID '{jid}'") + else: + region_stack.append((line_num, jid)) + continue + + # Check for COV_JUSTIFIED_STOP + stop_match = COV_JUSTIFIED_STOP_RE.search(line) + if stop_match: + if not region_stack: + warnings.append(f"{rel_path}:{line_num}: COV_JUSTIFIED_STOP without matching START") + else: + start_line, jid = region_stack.pop() + if jid in justifications_by_id: + entry = justifications_by_id[jid] + for ln in range(start_line + 1, line_num): + justified_lines[ln] = { + "id": jid, + "category": entry["category"], + "reason": entry["reason"].strip(), + } + continue + + # Check for single-line COV_JUSTIFIED (but not START/STOP) + if "COV_JUSTIFIED_START" not in line and "COV_JUSTIFIED_STOP" not in line: + line_match = COV_JUSTIFIED_LINE_RE.search(line) + if line_match: + jid = line_match.group(1) + if jid not in justifications_by_id: + warnings.append(f"{rel_path}:{line_num}: COV_JUSTIFIED references unknown ID '{jid}'") + else: + entry = justifications_by_id[jid] + justified_lines[line_num] = { + "id": jid, + "category": entry["category"], + "reason": entry["reason"].strip(), + } + + # Check for unclosed regions + for start_line, jid in region_stack: + warnings.append(f"{rel_path}:{start_line}: COV_JUSTIFIED_START '{jid}' without matching STOP") + + return warnings, justified_lines + + +def collect_source_files(source_root: Path, file_filter: str) -> List[Path]: + """Collect source files to scan for markers.""" + extensions = file_filter.split(",") if file_filter else ["cpp", "h", "hpp", "cc", "rs"] + files = [] + for ext in extensions: + for path in source_root.rglob(f"*.{ext.strip()}"): + # Skip Bazel convenience symlinks (bazel-bin, bazel-out, bazel-, ...) + # so the marker scan does not descend into build outputs. + rel_parts = path.relative_to(source_root).parts + if rel_parts and rel_parts[0].startswith("bazel-"): + continue + files.append(path) + return sorted(files) + + +def load_yaml(yaml_path: Path) -> Dict[str, Any]: + """Load YAML justification database.""" + if not yaml_path.exists(): + print(f"ERROR: Justification YAML not found: {yaml_path}", file=sys.stderr) + sys.exit(1) + + with open(yaml_path, "r", encoding="utf-8") as f: + content = f.read() + + return yaml.safe_load(content) + + +def validate_yaml(data: Dict[str, Any]) -> None: + """Validate the justification YAML structure and types.""" + try: + errors = [] + + if not isinstance(data, dict): + print("ERROR: YAML validation: root must be a mapping", file=sys.stderr) + sys.exit(1) + + if "version" not in data: + errors.append("Missing 'version' field") + elif not isinstance(data["version"], int): + errors.append(f"'version' must be an integer, got {type(data['version']).__name__}") + + if "justifications" not in data: + errors.append("Missing 'justifications' field") + for e in errors: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + if not isinstance(data["justifications"], list): + errors.append(f"'justifications' must be a list, got {type(data['justifications']).__name__}") + for e in errors: + print(f"ERROR: YAML validation: {e}", file=sys.stderr) + sys.exit(1) + + seen_ids: Set[str] = set() + for i, entry in enumerate(data["justifications"]): + prefix = f"justifications[{i}]" + + if not isinstance(entry, dict): + errors.append(f"{prefix}: must be a mapping, got {type(entry).__name__}") + continue + + if "id" not in entry: + errors.append(f"{prefix}: missing 'id'") + continue + + jid = entry["id"] + if not isinstance(jid, str): + errors.append(f"{prefix}: 'id' must be a string, got {type(jid).__name__}") + continue + + if jid in seen_ids: + errors.append(f"{prefix}: duplicate ID '{jid}'") + seen_ids.add(jid) + + if not re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", jid): + errors.append(f"{prefix}: ID '{jid}' must be kebab-case") + + if "category" not in entry: + errors.append(f"{prefix}: missing 'category'") + elif not isinstance(entry["category"], str): + errors.append(f"{prefix}: 'category' must be a string, got {type(entry['category']).__name__}") + elif entry["category"] not in VALID_CATEGORIES: + errors.append( + f"{prefix}: invalid category '{entry['category']}'. Must be one of: {sorted(VALID_CATEGORIES)}" + ) + + if "platforms" not in entry: + errors.append(f"{prefix}: missing 'platforms'") + elif not isinstance(entry["platforms"], list): + errors.append(f"{prefix}: 'platforms' must be a list, got {type(entry['platforms']).__name__}") + elif not entry["platforms"]: + errors.append(f"{prefix}: 'platforms' must not be empty") + else: + for p in entry["platforms"]: + if not isinstance(p, str): + errors.append(f"{prefix}: 'platforms' entries must be strings, got {type(p).__name__}") + elif p not in VALID_PLATFORMS: + errors.append(f"{prefix}: invalid platform '{p}'. Must be one of: {sorted(VALID_PLATFORMS)}") + + if "reason" not in entry: + errors.append(f"{prefix}: missing 'reason'") + elif not isinstance(entry["reason"], str): + errors.append(f"{prefix}: 'reason' must be a string, got {type(entry['reason']).__name__}") + elif not entry["reason"].strip(): + errors.append(f"{prefix}: 'reason' must not be empty") + + if "locations" in entry: + if not isinstance(entry["locations"], list): + errors.append(f"{prefix}: 'locations' must be a list, got {type(entry['locations']).__name__}") + else: + for j, loc in enumerate(entry["locations"]): + loc_prefix = f"{prefix}.locations[{j}]" + if not isinstance(loc, dict): + errors.append(f"{loc_prefix}: must be a mapping, got {type(loc).__name__}") + continue + if "file" not in loc: + errors.append(f"{loc_prefix}: missing 'file'") + elif not isinstance(loc["file"], str): + errors.append(f"{loc_prefix}: 'file' must be a string, got {type(loc['file']).__name__}") + for int_field in ("line", "line_start", "line_end"): + if int_field in loc and not isinstance(loc[int_field], int): + errors.append( + f"{loc_prefix}: '{int_field}' must be an integer, " + f"got {type(loc[int_field]).__name__}" + ) + if "lines" in loc: + if not isinstance(loc["lines"], list): + errors.append( + f"{loc_prefix}: 'lines' must be a list, got {type(loc['lines']).__name__}" + ) + elif not all(isinstance(ln, int) for ln in loc["lines"]): + errors.append(f"{loc_prefix}: 'lines' must contain only integers") + + if errors: + for e in errors: + print(f"ERROR: YAML validation: {e}", file=sys.stderr) + sys.exit(1) + except Exception as error: + print(f"ERROR: YAML validation: {error}", file=sys.stderr) + sys.exit(1) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Coverage justification processor") + parser.add_argument( + "--yaml", + type=Path, + required=True, + help="Path to coverage_justifications.yaml", + ) + parser.add_argument( + "--source-root", + type=Path, + required=True, + help="Root directory of source files", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Output path for resolved justification manifest (JSON)", + ) + parser.add_argument( + "--file-filter", + type=str, + default="cpp,h,hpp,cc,rs", + help="Comma-separated file extensions to scan (default: cpp,h,hpp,cc,rs)", + ) + parser.add_argument( + "--platform", + type=str, + default=None, + choices=sorted(VALID_PLATFORMS), + help="Target platform for filtering justifications (default: all platforms apply)", + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/coverage/merger.py b/coverage/merger.py new file mode 100644 index 00000000..fc508f61 --- /dev/null +++ b/coverage/merger.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Per-test coverage output generator using llvm-cov. + +This script is invoked by Bazel as the --coverage_output_generator for each test. +It receives profraw files from test execution, merges them into profdata, generates +an HTML coverage report using llvm-cov show, and packages everything into a zip file +that the reporter can later aggregate. + +Expected Bazel interface (from collect_coverage.sh): + --coverage_dir= Directory containing *.profraw files + --output_file= Where to write the output (zip) + --source_file_manifest= File listing instrumented sources and object files + --filter_sources= Source path regexes to exclude (repeatable) + [--sources_to_replace_file=] Optional source mapping file +""" + +import argparse +import json +import os +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import List, Set + + +def main() -> None: + args = parse_args() + + # Get object files from the manifest. + object_files = get_object_files_from_manifest(args.source_file_manifest) + if not object_files: + print("INFO: No instrumented object files found, skipping coverage.", file=sys.stderr) + cleanup_dangling_symlinks(args.coverage_dir) + sys.exit(0) + + # Find profraw files. + profraw_files = sorted(args.coverage_dir.glob("*.profraw")) + if not profraw_files: + print("INFO: No *.profraw files found, skipping coverage.", file=sys.stderr) + cleanup_dangling_symlinks(args.coverage_dir) + sys.exit(0) + + llvm_profdata = find_llvm_profdata() + if not llvm_profdata: + print( + "ERROR: llvm-profdata not found (neither LLVM_PROFDATA nor " + "RUST_LLVM_PROFDATA resolved to an existing binary).", + file=sys.stderr, + ) + sys.exit(1) + + # Merge profraw → profdata. + profdata_dir = args.coverage_dir / "profdata" + profdata_dir.mkdir(exist_ok=True) + profdata_file = profdata_dir / "target.profdata" + + run_command( + [ + llvm_profdata, + "merge", + "--sparse", + "--output", + str(profdata_file), + ] + + [str(f) for f in profraw_files] + ) + + # Create meta.json with object files for the reporter. + meta_dir = args.coverage_dir / "meta" + meta_dir.mkdir(exist_ok=True) + meta = { + "object_files": [os.path.realpath(f) for f in sorted(object_files)], + } + with open(meta_dir / "meta.json", "w", encoding="utf-8") as f: + json.dump(meta, f) + + # Package into zip at output_file. + create_zip( + root=args.coverage_dir, + directories=[profdata_dir, meta_dir], + output_file=args.output_file, + ) + + # Clean up dangling symlinks in coverage_dir that would cause Bazel tree + # artifact validation to fail (e.g. the 'gcov' symlink created by + # collect_cc_coverage.sh's init_gcov() pointing into the destroyed sandbox). + cleanup_dangling_symlinks(args.coverage_dir) + + target = os.environ.get("TEST_TARGET", "unknown") + print(f"INFO: Coverage merger completed for '{target}'", file=sys.stderr) + + +def find_llvm_profdata() -> str: + """Locate the llvm-profdata binary. + + C++ tests: Bazel exports LLVM_PROFDATA from the cc toolchain. + Rust tests: rules_rust exports RUST_LLVM_PROFDATA (an execroot-relative + path to the rust_toolchain's llvm_profdata) instead; resolve it against + the current directory, ROOT and the runfiles dir. + """ + direct = os.environ.get("LLVM_PROFDATA") + if direct and Path(direct).exists(): + return direct + + rust = os.environ.get("RUST_LLVM_PROFDATA") + if rust: + exec_root = Path(os.environ.get("ROOT", ".")) + runfiles_dir = Path(os.environ.get("RUNFILES_DIR", "")) / os.environ.get("TEST_WORKSPACE", "_main") + for candidate in [Path(rust), exec_root / rust, runfiles_dir / rust]: + if candidate.exists(): + return str(candidate) + + return "" + + +def cleanup_dangling_symlinks(directory: Path) -> None: + """Remove symlinks in the coverage directory that would become dangling. + + Bazel's tree artifact validation rejects directories containing dangling + symlinks. The 'gcov' symlink created by collect_cc_coverage.sh's init_gcov() + points into the sandbox which is torn down before validation runs. Since we + use llvm-cov directly, this symlink is not needed. + """ + gcov_link = directory / "gcov" + if gcov_link.is_symlink(): + gcov_link.unlink() + + # Also remove any other symlinks pointing into sandbox paths. + for entry in directory.iterdir(): + if entry.is_symlink(): + target = os.readlink(entry) + if "sandbox" in target: + entry.unlink() + + +def get_object_files_from_manifest(source_file_manifest: Path) -> Set[str]: + """Parse the coverage manifest to find instrumented object files.""" + runfiles_dir = Path(os.environ.get("RUNFILES_DIR", "")) / os.environ.get("TEST_WORKSPACE", "_main") + root = os.environ.get("ROOT") + if not root: + # Bazel's coverage collection exports ROOT (the exec root) to the + # LCOV merger action; without it manifest paths cannot be resolved. + print( + "ERROR: ROOT environment variable is not set; the merger must be " + "invoked by Bazel's coverage collection (--coverage_output_generator).", + file=sys.stderr, + ) + sys.exit(1) + exec_root = Path(root) + + object_files = set() + with open(source_file_manifest, encoding="utf-8") as f: + manifests = [line.strip() for line in f.readlines()] + + for manifest in manifests: + if "objects_list.txt" in manifest: + with open(manifest, encoding="utf-8") as f: + for line in f: + obj_path = line.strip() + if not obj_path: + continue + # Try runfiles first, then exec_root. + candidate = runfiles_dir / obj_path + if candidate.exists(): + object_files.add(str(candidate)) + else: + object_files.add(str(exec_root / obj_path)) + else: + # Rust tests: rules_rust lists the instrumented test executable + # itself in the manifest (via coverage metadata_files) instead of + # an objects_list.txt. Pick up manifest entries that are ELF + # binaries directly. Skip external/ entries: the rust_toolchain + # also lists its llvm-cov/llvm-profdata binaries as metadata + # files, and those are not instrumented objects. + if manifest.startswith("external/") or "/external/" in manifest: + continue + for candidate in [runfiles_dir / manifest, exec_root / manifest, Path(manifest)]: + if candidate.is_file() and is_elf(candidate): + object_files.add(str(candidate)) + break + + return object_files + + +def is_elf(path: Path) -> bool: + """Return True if the file at path is an ELF binary.""" + try: + with open(path, "rb") as f: + return f.read(4) == b"\x7fELF" + except OSError: + return False + + +def run_command(cmd: List[str]) -> subprocess.CompletedProcess: + """Run a command and exit on failure.""" + try: + return subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + except subprocess.CalledProcessError as e: + print(f"ERROR: Command failed with code {e.returncode}:", file=sys.stderr) + print(f" {' '.join(cmd)}", file=sys.stderr) + if e.stdout: + print(e.stdout, file=sys.stderr) + sys.exit(1) + + +def create_zip(root: Path, directories: List[Path], output_file: Path) -> None: + """Create a zip file from the given directories relative to root.""" + with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf: + for directory in directories: + if not directory.exists(): + continue + for dirpath, _, files in os.walk(directory): + for filename in files: + file_path = Path(dirpath) / filename + arcname = file_path.relative_to(root) + zf.write(file_path, arcname) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments matching the Bazel LCOV_MERGER interface.""" + parser = argparse.ArgumentParser(description="LLVM coverage merger for Bazel") + parser.add_argument("--coverage_dir", type=Path, required=True) + parser.add_argument("--output_file", type=Path, required=True) + parser.add_argument("--source_file_manifest", type=Path, required=True) + parser.add_argument("--filter_sources", action="append", default=[]) + parser.add_argument("--sources_to_replace_file", type=str, default=None) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/coverage/reporter.py b/coverage/reporter.py new file mode 100644 index 00000000..f3967573 --- /dev/null +++ b/coverage/reporter.py @@ -0,0 +1,775 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Final coverage report generator using llvm-cov. + +This script is invoked by Bazel as the --coverage_report_generator after all tests +complete. It reads the per-test zip files produced by the merger, merges all profdata +into one, and generates the final combined HTML report. + +Expected Bazel interface: + --reports_file= Text file listing paths to all per-test coverage outputs + --output_file= Where to write the final report (zip) +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import List, Optional, Set, Tuple +from python.runfiles import Runfiles + + +def main() -> None: + """Main entry point.""" + args = parse_args() + r = Runfiles.Create() + + # Read the list of per-test report files. + reports = read_reports_file(args.reports_file) + if not reports: + print("INFO: No coverage reports listed; writing empty output.", file=sys.stderr) + write_empty_output(args.output_file) + return + + # Extract profdata and object files from each per-test zip. + valid_profdata_files, valid_object_files = extract_reports(reports) + + if not valid_profdata_files or not valid_object_files: + print("INFO: No valid profdata or object files found; writing empty output.", file=sys.stderr) + write_empty_output(args.output_file) + return + + sorted_objects = sorted(valid_object_files) + + # Resolve the llvm tools. The reporter_wrapper passes explicit rlocation + # paths for the consumer-supplied toolchain labels; the bare + # "llvm_toolchain/..." forms remain as a fallback for in-repo setups. + llvm_bin_path = resolve_tool(r, args.llvm_cov, "llvm_toolchain/llvm-cov") + llvm_profdata_path = resolve_tool(r, args.llvm_profdata, "llvm_toolchain/llvm-profdata") + if not llvm_bin_path or not llvm_profdata_path: + print( + "ERROR: llvm-cov/llvm-profdata not found in runfiles. Pass --llvm_cov " + "and --llvm_profdata (the score_coverage_reporter macro does this).", + file=sys.stderr, + ) + sys.exit(1) + + # Merge all per-test profdata files. + merged_profdata = Path.cwd() / "merged_coverage.profdata" + merge_inputs = sorted(set(valid_profdata_files)) + run_command( + [ + str(llvm_profdata_path), + "merge", + "--output", + str(merged_profdata), + ] + + merge_inputs + ) + + # Load baseline objects (production library archives) for zero-coverage baseline. + baseline_objects = load_baseline_objects(r, args.baseline_objects, args.workspace_root) + + # Rust rlib archives (exposed as .a symlinks by rules_rust) start with a + # lib.rmeta member, which makes llvm-cov reject the whole archive with + # "no coverage data found" even though the .o members carry the covmap. + # Expand such archives into their object members. + baseline_objects = expand_rlib_archives(baseline_objects, Path.cwd() / "rlib_baseline_objects") + + # Determine filter regexes: prefer allowlist-based filtering, fall back to manual regexes. + workspace_root = args.workspace_root + allowlist_files = [] + filter_regexes = [] + baseline_only_archives = [] + baseline_only_files = set() + + if args.coverage_allowlist: + allowlist_files = load_coverage_allowlist(r, args.coverage_allowlist) + if allowlist_files: + print(f"INFO: Using coverage allowlist with {len(allowlist_files)} source files.", file=sys.stderr) + allowlist_set = set(allowlist_files) + + # Get files covered by test binaries. + test_covered_files = get_covered_files(llvm_bin_path, sorted_objects, str(merged_profdata), workspace_root) + print(f"INFO: Test binaries cover {len(test_covered_files)} files.", file=sys.stderr) + + # Get files from baseline archives via a SEPARATE llvm-cov run. + # Combining archives with test binaries in a single llvm-cov invocation + # causes some files to vanish (suspected llvm-cov deduplication issue). + # Some archives may have oversized coverage mappings ("malformed coverage + # data"), so we iteratively remove bad ones. + baseline_files = set() + if baseline_objects: + baseline_files = get_covered_files(llvm_bin_path, baseline_objects, None, workspace_root) + print(f"INFO: Baseline archives contain {len(baseline_files)} files.", file=sys.stderr) + + # Files only in baseline archives (not in any test binary). + baseline_only_files = (baseline_files & allowlist_set) - test_covered_files + baseline_only_archives = [] + if baseline_only_files: + print( + f"INFO: {len(baseline_only_files)} allowlisted files only in baseline " + f"(e.g., {sorted(baseline_only_files)[:5]})", + file=sys.stderr, + ) + # Use all valid baseline archives for LCOV generation. + # The _filter_lcov function will filter to only baseline-only files. + baseline_only_archives = list(baseline_objects) + + # Union of test + baseline for exclude-set calculation. + all_covered_files = test_covered_files | baseline_files + files_to_exclude = all_covered_files - allowlist_set + filter_regexes = [re.escape(f) + "$" for f in sorted(files_to_exclude)] + print(f"INFO: Excluding {len(filter_regexes)} files not in allowlist.", file=sys.stderr) + else: + print("ERROR: Coverage allowlist is empty, falling back to filter_regexes.txt.", file=sys.stderr) + sys.exit(-1) + cxxfilt = find_cxxfilt(llvm_bin_path, r, args.llvm_cxxfilt) + common_args = { + "llvm_bin_path": llvm_bin_path, + "objects": sorted_objects, + "instr_profile": str(merged_profdata), + "filter_regexes": sorted(filter_regexes), + "workspace_root": workspace_root, + } + + # Generate HTML report including baseline-only files when valid archives are available. + html_report_dir = Path.cwd() / "html_report" + if baseline_only_archives: + all_html_objects = sorted_objects + baseline_only_archives + html_args = { + **common_args, + "objects": all_html_objects, + } + try: + run_llvm_cov_show( + **html_args, + output_format="html", + html_report_dir=html_report_dir, + cxxfilt=cxxfilt, + ) + except SystemExit: + # Some baseline archives caused llvm-cov show to fail; retry with test binaries only. + print( + "WARNING: HTML generation with baseline archives failed; falling back to test-only HTML.", + file=sys.stderr, + ) + run_llvm_cov_show( + **common_args, + output_format="html", + html_report_dir=html_report_dir, + cxxfilt=cxxfilt, + ) + else: + run_llvm_cov_show( + **common_args, + output_format="html", + html_report_dir=html_report_dir, + cxxfilt=cxxfilt, + ) + + # Rewrite absolute workspace paths in the HTML pages so unpacked report + # archives remain browsable outside the machine that produced them. + _make_html_paths_relative(html_report_dir, workspace_root) + + # Generate LCOV report from test binaries. + lcov_report_dir = Path.cwd() / "lcov_report" + lcov_report_dir.mkdir(exist_ok=True) + lcov_result = run_llvm_cov_export(**common_args) + lcov_content = lcov_result.stdout + + # If there are baseline-only files, generate a separate baseline LCOV and merge. + if baseline_only_archives: + baseline_lcov_args = { + "llvm_bin_path": llvm_bin_path, + "objects": baseline_only_archives, + "instr_profile": None, + "filter_regexes": [], # No filtering — we only have the needed archives. + "workspace_root": workspace_root, + } + baseline_lcov = run_llvm_cov_export(**baseline_lcov_args) + if baseline_lcov.stdout: + # Filter baseline LCOV to only include baseline-only files. + filtered_baseline = _filter_lcov(baseline_lcov.stdout, baseline_only_files) + if filtered_baseline: + lcov_content += filtered_baseline + print(f"INFO: Merged baseline LCOV for {len(baseline_only_files)} files.", file=sys.stderr) + + # Strip the absolute workspace root from SF: records so the LCOV file is + # portable (IDE gutters, SonarQube, reports produced inside containers). + lcov_content = _make_lcov_paths_relative(lcov_content, workspace_root) + + with open(lcov_report_dir / "lcov.dat", "w", encoding="utf-8") as f: + f.write(lcov_content) + + # Generate text summary. + text_report_dir = Path.cwd() / "text_report" + text_report_dir.mkdir(exist_ok=True) + summary = run_llvm_cov_report(**common_args) + with open(text_report_dir / "summary.txt", "w", encoding="utf-8") as f: + f.write(summary.stdout) + print(summary.stdout, file=sys.stderr) + + # Package everything into the output zip. + directories = [html_report_dir, lcov_report_dir, text_report_dir] + create_zip( + root=Path.cwd(), + directories=directories, + output_file=args.output_file, + ) + + print(f"INFO: Coverage reporter completed. Output: {args.output_file}", file=sys.stderr) + + +def _make_lcov_paths_relative(lcov_content: str, workspace_root: str) -> str: + """Rewrite absolute SF: paths under workspace_root to workspace-relative ones. + + Paths outside the workspace (external deps that survived filtering) are + left unchanged. + """ + prefix = workspace_root if workspace_root.endswith("/") else workspace_root + "/" + sf_prefix = "SF:" + prefix + lines = [] + for line in lcov_content.splitlines(keepends=True): + if line.startswith(sf_prefix): + lines.append("SF:" + line[len(sf_prefix) :]) + else: + lines.append(line) + return "".join(lines) + + +_SOURCE_TITLE_RE = re.compile(r"(

)([^<]*)(
)") + + +def _make_html_paths_relative(html_dir: Path, workspace_root: str) -> None: + """Rewrite absolute workspace paths in llvm-cov HTML page titles. + + Only the source-name-title header text is touched — hrefs and the on-disk + page layout embed the same path components without a leading slash, and a + blanket text replacement would corrupt them. + """ + if not html_dir.exists(): + return + prefix = workspace_root if workspace_root.endswith("/") else workspace_root + "/" + + def _repl(match: "re.Match") -> str: + title = match.group(2) + if title.startswith(prefix): + title = title[len(prefix) :] + return match.group(1) + title + match.group(3) + + for page in html_dir.rglob("*.html"): + text = page.read_text(encoding="utf-8", errors="replace") + new_text = _SOURCE_TITLE_RE.sub(_repl, text) + if new_text != text: + page.write_text(new_text, encoding="utf-8") + + +def _filter_lcov(lcov_content: str, target_files: set) -> str: + """Filter LCOV content to only include records for target files. + + LCOV format: SF: starts a record, end_of_record ends it. + """ + result = [] + current_record = [] + include = False + + for line in lcov_content.splitlines(keepends=True): + if line.startswith("SF:"): + current_record = [line] + filepath = line[3:].strip() + # Check if the file path (or its suffix) matches any target file. + include = any(filepath.endswith(f) for f in target_files) + elif line.strip() == "end_of_record": + current_record.append(line) + if include: + result.extend(current_record) + current_record = [] + include = False + else: + current_record.append(line) + + return "".join(result) + + +def get_covered_files( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + workspace_root: str, +) -> set: + """Run a quick llvm-cov report to discover all files with coverage data. + + Returns a set of workspace-relative file paths. + """ + cmd = [ + str(llvm_bin_path), + "report", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + ] + if instr_profile is None: + cmd.append("--empty-profile") + else: + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + result = run_command(cmd) + if result.returncode != 0: + return set() + + files = set() + in_files = False + for line in result.stdout.splitlines(): + if line.startswith("---"): + in_files = True + continue + if line.startswith("TOTAL"): + break + if not in_files: + continue + # Extract filename (everything before first multi-space + digit sequence) + match = re.match(r"^(.+?)\s{2,}\d+", line) + if match: + filename = match.group(1).strip() + # Normalize to workspace-relative form. llvm-cov report prints the + # raw covmap path: for C++ that is the recorded compilation dir + # /proc/self/cwd/, --path-equivalence does not rewrite the + # DISPLAYED path. Rust covmap paths are already exec-root relative. + for prefix in (workspace_root, "/proc/self/cwd/"): + if filename.startswith(prefix): + filename = filename[len(prefix) :] + break + files.add(filename) + + return files + + +def run_llvm_cov_show( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + filter_regexes: List[str], + workspace_root: str, + output_format: str, + html_report_dir: Path = None, + cxxfilt: str = "", +) -> subprocess.CompletedProcess: + """Run llvm-cov show.""" + cmd = [ + str(llvm_bin_path), + "show", + f"--format={output_format}", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + f"--compilation-dir={workspace_root}", + "--show-branches=count", + "--show-region-summary=0", + ] + + if cxxfilt: + cmd.append(f"--Xdemangler={cxxfilt}") + + for regex in filter_regexes: + cmd.append(f"--ignore-filename-regex={regex}") + + if html_report_dir: + cmd.append(f"--output-dir={html_report_dir}") + cmd.append("--coverage-watermark=100,50") + cmd.append("--show-expansions") + + if instr_profile is None: + cmd.append("--empty-profile") + else: + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + return run_command(cmd) + + +def run_llvm_cov_export( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + filter_regexes: List[str], + workspace_root: str, +) -> subprocess.CompletedProcess: + """Run llvm-cov export to produce LCOV format.""" + cmd = [ + str(llvm_bin_path), + "export", + "--format=lcov", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + f"--compilation-dir={workspace_root}", + ] + + for regex in filter_regexes: + cmd.append(f"--ignore-filename-regex={regex}") + + if instr_profile is None: + cmd.append("--empty-profile") + else: + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + # Keep stderr separate: llvm-cov warnings must not end up in the LCOV data. + return run_command(cmd, separate_stderr=True) + + +def run_llvm_cov_report( + llvm_bin_path: Path, + objects: List[str], + instr_profile: Optional[str], + filter_regexes: List[str], + workspace_root: str, +) -> subprocess.CompletedProcess: + """Run llvm-cov report for a text summary.""" + cmd = [ + str(llvm_bin_path), + "report", + f"--path-equivalence=/proc/self/cwd/,{workspace_root}", + "--show-region-summary=0", + "--show-branch-summary=1", + ] + + for regex in filter_regexes: + cmd.append(f"--ignore-filename-regex={regex}") + + if instr_profile is None: + cmd.append("--empty-profile") + else: + cmd.extend(["--instr-profile", instr_profile]) + cmd.append(objects[0]) + for obj in objects[1:]: + cmd.extend(["--object", obj]) + + return run_command(cmd) + + +def extract_reports(reports: List[str]) -> Tuple[Set[str], Set[str]]: + """Extract profdata and object files from per-test zip files.""" + valid_profdata_files = set() + valid_object_files = set() + + for i, report_path in enumerate(reports): + # Skip baseline_coverage files (LCOV format, not our zip). + if "baseline_coverage" in report_path: + continue + + report = Path(report_path) + if not report.exists() or report.stat().st_size == 0: + continue + + # Check if it's a valid zip. + if not zipfile.is_zipfile(report): + continue + + profdata_name = f"coverage_report_{i:08d}.profdata" + + try: + with zipfile.ZipFile(report, "r") as archive: + # Extract meta. + meta_json = archive.read("meta/meta.json") + target_meta = json.loads(meta_json) + + # Extract profdata. + profdata_content = archive.read("profdata/target.profdata") + profdata_path = Path.cwd() / profdata_name + with open(profdata_path, "wb") as f: + f.write(profdata_content) + + valid_profdata_files.add(str(profdata_path)) + + # Collect object files. + for obj in target_meta.get("object_files", []): + if obj and Path(obj).exists(): + valid_object_files.add(os.path.realpath(obj)) + + except (zipfile.BadZipFile, KeyError, json.JSONDecodeError) as e: + print(f"WARNING: Skipping invalid report {report_path}: {e}", file=sys.stderr) + continue + + return valid_profdata_files, valid_object_files + + +def read_reports_file(reports_file: Path) -> List[str]: + """Read the reports file listing all per-test coverage outputs.""" + with open(reports_file, encoding="utf-8") as f: + return [line.strip() for line in f if line.strip()] + + +def _read_ar_members(path: str) -> List[tuple]: + """Parse a Unix ar archive, returning (name, data_offset, size) tuples. + + Handles the GNU long-name table ("//" member with "/" references). + Returns an empty list when the file is not an ar archive. + """ + members = [] + longnames = b"" + with open(path, "rb") as f: + if f.read(8) != b"!\n": + return [] + while True: + header = f.read(60) + if len(header) < 60: + break + name = header[0:16].decode(errors="replace").rstrip() + try: + size = int(header[48:58].decode().strip() or "0") + except ValueError: + break + data_offset = f.tell() + if name == "//": + longnames = f.read(size) + else: + if name.startswith("/") and name[1:].isdigit(): + start = int(name[1:]) + end = longnames.find(b"\n", start) + name = longnames[start:end].decode(errors="replace").rstrip("/") + elif name.endswith("/"): + name = name[:-1] + members.append((name, data_offset, size)) + f.seek(size, 1) + if size % 2 == 1: + f.seek(1, 1) + return members + + +def expand_rlib_archives(objects: List[str], workdir: Path) -> List[str]: + """Replace Rust rlib archives with their extracted object members. + + llvm-cov rejects rlib archives ("no coverage data found") because of the + leading lib.rmeta member, even though the .o members carry the coverage + mapping. Non-rlib entries (C++ .a archives, executables) pass through + unchanged. + """ + result = [] + extracted = 0 + for obj in objects: + members = _read_ar_members(obj) if obj.endswith((".a", ".rlib")) else [] + if not any(name == "lib.rmeta" for name, _, _ in members): + result.append(obj) + continue + workdir.mkdir(parents=True, exist_ok=True) + with open(obj, "rb") as f: + for index, (name, offset, size) in enumerate(members): + if not name.endswith(".o"): + continue + f.seek(offset) + out_path = workdir / f"{Path(obj).stem}.{index}.o" + out_path.write_bytes(f.read(size)) + result.append(str(out_path)) + extracted += 1 + if extracted: + print(f"INFO: Expanded {extracted} object(s) from Rust rlib baseline archives.", file=sys.stderr) + return result + + +def resolve_tool( + runfiles: Optional[Runfiles], + flag_value: Optional[str], + fallback_rlocation: str, +) -> Optional[Path]: + """Resolve an llvm tool path. + + Preference order: the explicit rlocation path passed by the + reporter_wrapper (consumer-supplied toolchain label), then the legacy + "llvm_toolchain/..." runfiles location, then the raw value as a plain path. + """ + for candidate in (flag_value, fallback_rlocation): + if not candidate: + continue + if runfiles: + location = runfiles.Rlocation(candidate) + if location and Path(location).exists(): + return Path(location) + if Path(candidate).exists(): + return Path(candidate) + return None + + +def find_cxxfilt( + llvm_bin_path: Path, + runfiles: Optional[Runfiles] = None, + explicit: Optional[str] = None, +) -> str: + """Locate llvm-cxxfilt for demangling (C++ Itanium and Rust v0/legacy symbols). + + Tries the explicit rlocation path from the reporter_wrapper first, then the + directory of llvm-cov, then the @llvm_toolchain_llvm distribution via + runfiles (toolchains_llvm declares no alias for llvm-cxxfilt). + Returns an empty string when unavailable (demangling is cosmetic). + """ + if explicit: + resolved = resolve_tool(runfiles, explicit, "") + if resolved: + return str(resolved) + sibling = llvm_bin_path.parent / "llvm-cxxfilt" + if sibling.exists(): + return str(sibling) + r = runfiles or Runfiles.Create() + if r: + location = r.Rlocation("llvm_toolchain_llvm/bin/llvm-cxxfilt") + if location and Path(location).exists(): + return location + return "" + + +def load_coverage_allowlist(runfiles: Runfiles, rlocation_path: str) -> List[str]: + """Load coverage allowlist (package paths) from a file via Bazel runfiles.""" + path = runfiles.Rlocation(rlocation_path) + if not path or not Path(path).exists(): + return [] + + lines = Path(path).read_text(encoding="utf-8").splitlines() + return [line.strip() for line in lines if line.strip() and not line.strip().startswith("#")] + + +def load_baseline_objects( + runfiles: Runfiles, + rlocation_path: str, + workspace_root: str, +) -> List[str]: + """Load baseline object archive paths and resolve them to absolute paths. + + The objects manifest lists relative paths to .a files. When the reporter runs + in the exec config, the manifest paths use the exec config dir + (e.g., k8-opt-exec-*). + """ + if not rlocation_path: + return [] + + path = runfiles.Rlocation(rlocation_path) + if not path or not Path(path).exists(): + print(f"WARNING: Baseline objects manifest not found: {rlocation_path}", file=sys.stderr) + return [] + + lines = Path(path).read_text(encoding="utf-8").splitlines() + resolved = [] + for line in lines: + line = line.strip() + if not line or line.startswith("#"): + continue + # The manifest lists short_paths of files built by the CONSUMER + # repository, which is always the root module ("_main") in a coverage + # run. Do NOT use runfiles.CurrentRepository() here: this script lives + # in score_tooling, so that would resolve against the wrong repo. + path = runfiles.Rlocation(os.path.join("_main", line)) + if os.path.exists(path): + resolved.append(path) + else: + print(f"ERROR: Baseline object not found: {line}", file=sys.stderr) + sys.exit(-1) + return sorted(resolved) + + +def run_command(cmd: List[str], separate_stderr: bool = False) -> subprocess.CompletedProcess: + """Run a command and exit on failure. + + With separate_stderr the child's stderr is captured separately and + forwarded to our stderr — required when stdout is machine-consumed data + (LCOV) that llvm-cov warnings must not corrupt. + """ + try: + result = subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE if separate_stderr else subprocess.STDOUT, + text=True, + ) + if separate_stderr and result.stderr: + print(result.stderr, file=sys.stderr) + return result + except subprocess.CalledProcessError as e: + print(f"ERROR: Command failed with code {e.returncode}:", file=sys.stderr) + print(f" {' '.join(cmd[:10])}{'...' if len(cmd) > 10 else ''}", file=sys.stderr) + if e.stdout: + print(e.stdout, file=sys.stderr) + if e.stderr: + print(e.stderr, file=sys.stderr) + sys.exit(1) + + +def write_empty_output(output_file: Path) -> None: + """Write an empty (but valid) zip so Bazel's coverage action still succeeds. + + Matches Bazel's own behaviour for runs that produce no coverage data + (e.g. a coverage invocation whose tests were all filtered out). + """ + with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED): + pass + + +def create_zip(root: Path, directories: List[Path], output_file: Path) -> None: + """Create a zip file from the given directories relative to root.""" + with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf: + for directory in directories: + if not directory.exists(): + continue + for dirpath, _, files in os.walk(directory): + for filename in files: + file_path = Path(dirpath) / filename + arcname = file_path.relative_to(root) + zf.write(file_path, arcname) + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments matching the Bazel coverage_report_generator interface.""" + parser = argparse.ArgumentParser(description="LLVM coverage reporter for Bazel") + parser.add_argument("--output_file", type=Path, required=True) + parser.add_argument("--reports_file", type=Path, required=True) + parser.add_argument( + "--coverage_allowlist", + type=str, + default=None, + help="Rlocation path to the coverage allowlist file (preferred over filter_regexes)", + ) + parser.add_argument( + "--baseline_objects", + type=str, + default=None, + help="Rlocation path to the baseline objects manifest (archive .a files)", + ) + parser.add_argument( + "--workspace_root", type=str, required=True, help="Real workspace root path for source path mapping" + ) + parser.add_argument( + "--llvm_cov", type=str, default=None, help="Rlocation path to llvm-cov (supplied by score_coverage_reporter)" + ) + parser.add_argument( + "--llvm_profdata", + type=str, + default=None, + help="Rlocation path to llvm-profdata (supplied by score_coverage_reporter)", + ) + parser.add_argument( + "--llvm_cxxfilt", + type=str, + default=None, + help="Rlocation path to llvm-cxxfilt (supplied by score_coverage_reporter)", + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/coverage/reporter_wrapper.bzl b/coverage/reporter_wrapper.bzl new file mode 100644 index 00000000..69189a64 --- /dev/null +++ b/coverage/reporter_wrapper.bzl @@ -0,0 +1,155 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +"""Executable wrapper rule for the coverage reporter. + +Instantiated in the CONSUMER repository (via the score_coverage_reporter +macro) so that the consumer's coverage_scope, MODULE.bazel and LLVM tool +labels can be wired into the reporter, which itself lives in score_tooling. +""" + +def _rlocation_path(ctx, file): + """Return the Runfiles.Rlocation()-compatible path for a Bazel File. + + External-repo files have short_path = "..//" — strip the + "../". Main-workspace files have short_path = "/" — prepend + the workspace name. Required because this rule mixes files from the + consumer repo (_main) and from score_tooling / toolchain repos. + """ + if file.short_path.startswith("../"): + return file.short_path[3:] + return ctx.workspace_name + "/" + file.short_path + +def _reporter_wrapper_impl(ctx): + launcher = ctx.actions.declare_file(ctx.label.name + ".sh") + + reporter = ctx.executable.reporter + module_bazel = ctx.file.module_bazel + coverage_scope = ctx.attr.coverage_scope + allowlist_group = coverage_scope[OutputGroupInfo].allowlist.to_list() + objects_group = coverage_scope[OutputGroupInfo].objects.to_list() + object_files = coverage_scope[OutputGroupInfo].object_files + + if len(allowlist_group) != 1: + fail("coverage_scope must provide exactly one allowlist file") + if len(objects_group) != 1: + fail("coverage_scope must provide exactly one objects manifest file") + + allowlist = allowlist_group[0] + baseline_objects = objects_group[0] + + cxxfilt_line = "" + if ctx.file.llvm_cxxfilt: + cxxfilt_line = ' --llvm_cxxfilt="{}" \\\n'.format( + _rlocation_path(ctx, ctx.file.llvm_cxxfilt), + ) + + # Bazel invokes the coverage report generator from within its coverage + # machinery, where an inherited RUNFILES_DIR may point at ANOTHER tool's + # runfiles tree. Derive our own runfiles directory from $0 first and only + # fall back to the inherited value. + script = """#!/usr/bin/env bash +set -euo pipefail +SELF_RUNFILES_DIR="$(cd "$(dirname "$0")" && pwd)/$(basename "$0").runfiles" +if [[ -d "${{SELF_RUNFILES_DIR}}" ]]; then + RUNFILES_DIR="${{SELF_RUNFILES_DIR}}" +elif [[ -z "${{RUNFILES_DIR:-}}" || ! -d "${{RUNFILES_DIR}}" ]]; then + echo "ERROR: could not locate the reporter_wrapper runfiles directory" >&2 + exit 1 +fi +export RUNFILES_DIR +WORKSPACE_ROOT="$(cd "$(dirname "$(readlink -f "${{RUNFILES_DIR}}/{module_bazel}")")" && pwd)/" +exec "${{RUNFILES_DIR}}/{reporter}" \\ + --coverage_allowlist="{allowlist}" \\ + --baseline_objects="{baseline_objects}" \\ + --workspace_root="${{WORKSPACE_ROOT}}" \\ + --llvm_cov="{llvm_cov}" \\ + --llvm_profdata="{llvm_profdata}" \\ +{cxxfilt_line} "$@" +""".format( + module_bazel = _rlocation_path(ctx, module_bazel), + reporter = _rlocation_path(ctx, reporter), + allowlist = _rlocation_path(ctx, allowlist), + baseline_objects = _rlocation_path(ctx, baseline_objects), + llvm_cov = _rlocation_path(ctx, ctx.file.llvm_cov), + llvm_profdata = _rlocation_path(ctx, ctx.file.llvm_profdata), + cxxfilt_line = cxxfilt_line, + ) + + ctx.actions.write( + output = launcher, + content = script, + is_executable = True, + ) + + direct_files = [ + reporter, + allowlist, + baseline_objects, + module_bazel, + ctx.file.llvm_cov, + ctx.file.llvm_profdata, + ] + if ctx.file.llvm_cxxfilt: + direct_files.append(ctx.file.llvm_cxxfilt) + + runfiles = ctx.runfiles( + files = direct_files, + transitive_files = object_files, + ).merge(ctx.attr.reporter[DefaultInfo].default_runfiles) + for tool in (ctx.attr.llvm_cov, ctx.attr.llvm_profdata, ctx.attr.llvm_cxxfilt): + if tool: + runfiles = runfiles.merge(tool[DefaultInfo].default_runfiles) + + return [DefaultInfo( + executable = launcher, + runfiles = runfiles, + )] + +reporter_wrapper = rule( + implementation = _reporter_wrapper_impl, + executable = True, + attrs = { + "reporter": attr.label( + executable = True, + cfg = "exec", + default = Label("//coverage:reporter"), + doc = "The coverage reporter binary (defaults to score_tooling's).", + ), + "coverage_scope": attr.label( + cfg = "target", + mandatory = True, + doc = "A score_coverage_scope target defining the in-scope sources/archives.", + ), + "module_bazel": attr.label( + allow_single_file = True, + mandatory = True, + doc = "The consumer's root MODULE.bazel; used to locate the real workspace root.", + ), + "llvm_cov": attr.label( + allow_single_file = True, + mandatory = True, + doc = "llvm-cov binary, e.g. @llvm_toolchain//:llvm-cov.", + ), + "llvm_profdata": attr.label( + allow_single_file = True, + mandatory = True, + doc = "llvm-profdata binary, e.g. @llvm_toolchain//:llvm-profdata.", + ), + "llvm_cxxfilt": attr.label( + allow_single_file = True, + doc = "Optional llvm-cxxfilt for demangling, e.g. " + + "@llvm_toolchain_llvm//:bin/llvm-cxxfilt.", + ), + }, +) diff --git a/coverage/requirements.in b/coverage/requirements.in new file mode 100644 index 00000000..c3726e8b --- /dev/null +++ b/coverage/requirements.in @@ -0,0 +1 @@ +pyyaml diff --git a/coverage/requirements_3_11.txt b/coverage/requirements_3_11.txt new file mode 100644 index 00000000..5017dc3e --- /dev/null +++ b/coverage/requirements_3_11.txt @@ -0,0 +1,81 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# bazel run //coverage:requirements_3_11.update +# +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r coverage/requirements.in diff --git a/coverage/requirements_3_12.txt b/coverage/requirements_3_12.txt new file mode 100644 index 00000000..1e800401 --- /dev/null +++ b/coverage/requirements_3_12.txt @@ -0,0 +1,81 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# bazel run //coverage:requirements_3_12.update +# +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via -r coverage/requirements.in diff --git a/coverage/scripts/normalize_symbol_report.py b/coverage/scripts/normalize_symbol_report.py deleted file mode 100755 index 646342c2..00000000 --- a/coverage/scripts/normalize_symbol_report.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -import json -import sys -from pathlib import Path - -if len(sys.argv) < 2: - print( - "usage: normalize_symbol_report.py [roots...]", - file=sys.stderr, - ) - sys.exit(2) - -json_path = Path(sys.argv[1]) -roots = [] -for arg in sys.argv[2:]: - if arg: - roots.append(Path(arg).resolve()) - -with json_path.open("r", encoding="utf-8") as fh: - data = json.load(fh) - - -def relativize(p: Path): - if not p.is_absolute(): - return p.as_posix() - for root in roots: - try: - return p.relative_to(root).as_posix() - except ValueError: - pass - try: - rp = p.resolve() - except Exception: - return None - for root in roots: - try: - return rp.relative_to(root).as_posix() - except ValueError: - pass - return None - - -changed = False -symbols = [] -for sym in data.get("symbols", []): - fname = sym.get("filename") - if not fname: - continue - rel = relativize(Path(fname)) - if rel is None: - changed = True - continue - if rel != fname: - sym["filename"] = rel - changed = True - symbols.append(sym) - -if symbols != data.get("symbols", []): - data["symbols"] = symbols - -if changed: - with json_path.open("w", encoding="utf-8") as fh: - json.dump(data, fh) diff --git a/coverage/tests/BUILD b/coverage/tests/BUILD index 87c99b2e..68ca4de9 100644 --- a/coverage/tests/BUILD +++ b/coverage/tests/BUILD @@ -1,5 +1,5 @@ # ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation +# Copyright (c) 2026 Contributors to the Eclipse Foundation # # See the NOTICE file(s) distributed with this work for additional # information regarding copyright ownership. @@ -11,32 +11,16 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("@rules_python//python:defs.bzl", "py_test") -package(default_visibility = ["//visibility:public"]) - -sh_test( - name = "parse_line_coverage_test", - srcs = ["parse_line_coverage_test.sh"], - data = [ - "fixtures/blanket_index.html", - "//coverage:scripts/parse_line_coverage.py", - ], -) - -sh_test( - name = "ferrocene_report_help_test", - srcs = ["ferrocene_report_help_test.sh"], - data = [ - "//coverage:ferrocene_report", - ], +py_test( + name = "merger_test", + srcs = ["merger_test.py"], + deps = ["//coverage:merger_lib"], ) -sh_test( - name = "normalize_symbol_report_test", - srcs = ["normalize_symbol_report_test.sh"], - data = [ - "fixtures/symbol_report.json", - "//coverage:scripts/normalize_symbol_report.py", - ], +py_test( + name = "reporter_test", + srcs = ["reporter_test.py"], + deps = ["//coverage:reporter_lib"], ) diff --git a/coverage/tests/fixtures/blanket_index.html b/coverage/tests/fixtures/blanket_index.html deleted file mode 100644 index 793b2d23..00000000 --- a/coverage/tests/fixtures/blanket_index.html +++ /dev/null @@ -1,293 +0,0 @@ - - -

Core library line coverage report

Go back to the documentation index

100.00% (8/8 lines)

Below is a list of all functions within the certified subset. Use the expander to review line coverage of any function.

To filter for specific coverage status, select below:
Line-through annotated functions

0 Partially Tested

0 Fully Untested

2 Fully Tested

coverage_demo::add
File: src/lib.rs
pub fn add(a: i32, b: i32) -> i32 {
-    a + b
-}
-
coverage_demo::maybe_div
File: src/lib.rs
pub fn maybe_div(a: i32, b: i32) -> Option<i32> {
-    if b == 0 {
-        None
-    } else {
-        Some(a / b)
-    }
-}
-

0 Fully Ignored

diff --git a/coverage/tests/fixtures/symbol_report.json b/coverage/tests/fixtures/symbol_report.json deleted file mode 100644 index 7766b58a..00000000 --- a/coverage/tests/fixtures/symbol_report.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "symbols": [ - {"filename": "/workspace/src/lib.rs"}, - {"filename": "/other/location/ignored.rs"}, - {"filename": "src/rel.rs"} - ] -} diff --git a/coverage/tests/merger_test.py b/coverage/tests/merger_test.py new file mode 100644 index 00000000..eea24a45 --- /dev/null +++ b/coverage/tests/merger_test.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Unit tests for the per-test coverage merger.""" + +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from coverage.merger import find_llvm_profdata, get_object_files_from_manifest, is_elf + + +class IsElfTest(unittest.TestCase): + def test_elf_magic_is_detected(self): + with tempfile.NamedTemporaryFile(suffix=".bin") as f: + f.write(b"\x7fELF" + b"\x00" * 12) + f.flush() + self.assertTrue(is_elf(Path(f.name))) + + def test_text_file_is_not_elf(self): + with tempfile.NamedTemporaryFile(suffix=".txt") as f: + f.write(b"just some text") + f.flush() + self.assertFalse(is_elf(Path(f.name))) + + def test_missing_file_is_not_elf(self): + self.assertFalse(is_elf(Path("/nonexistent/path/binary"))) + + +class FindLlvmProfdataTest(unittest.TestCase): + def test_llvm_profdata_env_wins(self): + with tempfile.NamedTemporaryFile() as f: + with mock.patch.dict(os.environ, {"LLVM_PROFDATA": f.name}, clear=True): + self.assertEqual(find_llvm_profdata(), f.name) + + def test_rust_llvm_profdata_resolved_against_root(self): + with tempfile.TemporaryDirectory() as root: + tool = Path(root) / "bin" / "llvm-profdata" + tool.parent.mkdir() + tool.write_bytes(b"\x7fELF") + env = {"RUST_LLVM_PROFDATA": "bin/llvm-profdata", "ROOT": root} + with mock.patch.dict(os.environ, env, clear=True): + self.assertEqual(find_llvm_profdata(), str(tool)) + + def test_returns_empty_when_nothing_resolves(self): + env = {"RUST_LLVM_PROFDATA": "does/not/exist"} + with mock.patch.dict(os.environ, env, clear=True): + self.assertEqual(find_llvm_profdata(), "") + + +class GetObjectFilesFromManifestTest(unittest.TestCase): + def test_missing_root_env_is_a_hard_error(self): + """Without ROOT the merger cannot resolve manifest paths — must exit.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt") as manifest: + manifest.write("some/path\n") + manifest.flush() + with mock.patch.dict(os.environ, {}, clear=True): + with self.assertRaises(SystemExit): + get_object_files_from_manifest(Path(manifest.name)) + + def test_rust_elf_manifest_entry_is_collected(self): + """rules_rust lists the instrumented test executable in the manifest.""" + with tempfile.TemporaryDirectory() as root: + binary = Path(root) / "pkg" / "my_test" + binary.parent.mkdir() + binary.write_bytes(b"\x7fELF" + b"\x00" * 12) + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt") as manifest: + manifest.write("pkg/my_test\n") + manifest.flush() + with mock.patch.dict(os.environ, {"ROOT": root}, clear=True): + objects = get_object_files_from_manifest(Path(manifest.name)) + self.assertEqual(objects, {str(binary)}) + + def test_external_manifest_entries_are_skipped(self): + """Toolchain-provided metadata files (external/) are not instrumented objects.""" + with tempfile.TemporaryDirectory() as root: + binary = Path(root) / "external" / "tool" + binary.parent.mkdir() + binary.write_bytes(b"\x7fELF" + b"\x00" * 12) + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt") as manifest: + manifest.write("external/tool\n") + manifest.flush() + with mock.patch.dict(os.environ, {"ROOT": root}, clear=True): + objects = get_object_files_from_manifest(Path(manifest.name)) + self.assertEqual(objects, set()) + + def test_objects_list_entries_are_resolved(self): + """C++ tests provide an objects_list.txt with one object path per line.""" + with tempfile.TemporaryDirectory() as root: + obj = Path(root) / "bazel-out" / "lib.a" + obj.parent.mkdir() + obj.write_bytes(b"!\n") + objects_list = Path(root) / "objects_list.txt" + objects_list.write_text("bazel-out/lib.a\n") + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt") as manifest: + manifest.write(f"{objects_list}\n") + manifest.flush() + with mock.patch.dict(os.environ, {"ROOT": root}, clear=True): + objects = get_object_files_from_manifest(Path(manifest.name)) + self.assertEqual(objects, {str(obj)}) + + +if __name__ == "__main__": + unittest.main() diff --git a/coverage/tests/normalize_symbol_report_test.sh b/coverage/tests/normalize_symbol_report_test.sh deleted file mode 100755 index d7431372..00000000 --- a/coverage/tests/normalize_symbol_report_test.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# ******************************************************************************* -# Copyright (c) 2026 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -set -euo pipefail - -fixture="${TEST_SRCDIR}/${TEST_WORKSPACE}/coverage/tests/fixtures/symbol_report.json" -script="${TEST_SRCDIR}/${TEST_WORKSPACE}/coverage/scripts/normalize_symbol_report.py" - -workdir="$(mktemp -d)" -trap 'rm -rf "${workdir}"' EXIT - -cp "${fixture}" "${workdir}/symbol_report.json" - -python3 "${script}" "${workdir}/symbol_report.json" "/workspace" "/execroot" - -python3 - "${workdir}/symbol_report.json" <<'PY' -import json -import sys - -with open(sys.argv[1], "r", encoding="utf-8") as fh: - data = json.load(fh) - -files = sorted({s.get("filename") for s in data.get("symbols", [])}) -expected = ["src/lib.rs", "src/rel.rs"] -if files != expected: - raise SystemExit(f"unexpected filenames: {files}") - -print("ok") -PY diff --git a/coverage/tests/reporter_test.py b/coverage/tests/reporter_test.py new file mode 100644 index 00000000..6f1e4ca2 --- /dev/null +++ b/coverage/tests/reporter_test.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Unit tests for the final coverage reporter.""" + +import tempfile +import unittest +import zipfile +from pathlib import Path + +from coverage.reporter import ( + _filter_lcov, + _make_html_paths_relative, + _make_lcov_paths_relative, + _read_ar_members, + expand_rlib_archives, + write_empty_output, +) + + +def _ar_header(name: str, size: int) -> bytes: + """Build a 60-byte Unix ar member header.""" + return (f"{name:<16}{'0':<12}{'0':<6}{'0':<6}{'100644':<8}{size:<10}`\n").encode() + + +def _make_archive(members) -> bytes: + """Build a Unix ar archive from (name, data) tuples.""" + blob = b"!\n" + for name, data in members: + blob += _ar_header(name, len(data)) + data + if len(data) % 2 == 1: + blob += b"\n" + return blob + + +class ReadArMembersTest(unittest.TestCase): + def test_non_archive_returns_empty(self): + with tempfile.NamedTemporaryFile(suffix=".a") as f: + f.write(b"\x7fELF not an archive") + f.flush() + self.assertEqual(_read_ar_members(f.name), []) + + def test_members_are_listed_with_sizes(self): + blob = _make_archive([("lib.rmeta/", b"META"), ("foo.o/", b"OBJDATA")]) + with tempfile.NamedTemporaryFile(suffix=".a") as f: + f.write(blob) + f.flush() + members = _read_ar_members(f.name) + self.assertEqual([(m[0], m[2]) for m in members], [("lib.rmeta", 4), ("foo.o", 7)]) + + def test_gnu_long_name_table_is_resolved(self): + longnames = b"a_very_long_object_file_name.o/\n" + blob = _make_archive([("//", longnames), ("/0", b"LONGOBJ")]) + with tempfile.NamedTemporaryFile(suffix=".a") as f: + f.write(blob) + f.flush() + members = _read_ar_members(f.name) + self.assertEqual([m[0] for m in members], ["a_very_long_object_file_name.o"]) + + +class ExpandRlibArchivesTest(unittest.TestCase): + def test_rlib_is_expanded_to_object_members(self): + """Archives with a lib.rmeta member are replaced by their .o members.""" + blob = _make_archive([("lib.rmeta/", b"META"), ("crate.o/", b"OBJ1"), ("notes.txt/", b"TXT")]) + with tempfile.TemporaryDirectory() as tmp: + rlib = Path(tmp) / "libcrate.a" + rlib.write_bytes(blob) + workdir = Path(tmp) / "extracted" + result = expand_rlib_archives([str(rlib)], workdir) + self.assertEqual(len(result), 1) + self.assertTrue(result[0].endswith(".o")) + self.assertEqual(Path(result[0]).read_bytes(), b"OBJ1") + + def test_plain_cc_archive_passes_through(self): + blob = _make_archive([("mylib.o/", b"OBJ1")]) + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "libcc.a" + archive.write_bytes(blob) + result = expand_rlib_archives([str(archive)], Path(tmp) / "x") + self.assertEqual(result, [str(archive)]) + + def test_executable_passes_through(self): + with tempfile.TemporaryDirectory() as tmp: + binary = Path(tmp) / "my_tool" + binary.write_bytes(b"\x7fELF" + b"\x00" * 12) + result = expand_rlib_archives([str(binary)], Path(tmp) / "x") + self.assertEqual(result, [str(binary)]) + + +class FilterLcovTest(unittest.TestCase): + LCOV = "SF:src/foo.cpp\nDA:1,1\nLF:1\nLH:1\nend_of_record\nSF:src/bar.cpp\nDA:1,0\nLF:1\nLH:0\nend_of_record\n" + + def test_only_target_records_survive(self): + result = _filter_lcov(self.LCOV, {"src/bar.cpp"}) + self.assertIn("SF:src/bar.cpp", result) + self.assertNotIn("SF:src/foo.cpp", result) + + def test_suffix_matching(self): + result = _filter_lcov("SF:/abs/prefix/src/foo.cpp\nend_of_record\n", {"src/foo.cpp"}) + self.assertIn("SF:/abs/prefix/src/foo.cpp", result) + + +class MakeLcovPathsRelativeTest(unittest.TestCase): + def test_workspace_paths_become_relative(self): + lcov = "SF:/ws/root/src/foo.cpp\nDA:1,1\nend_of_record\n" + result = _make_lcov_paths_relative(lcov, "/ws/root/") + self.assertEqual(result, "SF:src/foo.cpp\nDA:1,1\nend_of_record\n") + + def test_workspace_root_without_trailing_slash(self): + lcov = "SF:/ws/root/src/foo.cpp\nend_of_record\n" + result = _make_lcov_paths_relative(lcov, "/ws/root") + self.assertEqual(result, "SF:src/foo.cpp\nend_of_record\n") + + def test_external_paths_are_unchanged(self): + lcov = "SF:/other/place/dep.cpp\nend_of_record\n" + self.assertEqual(_make_lcov_paths_relative(lcov, "/ws/root/"), lcov) + + def test_non_sf_lines_are_preserved(self): + lcov = "TN:test\nSF:/ws/root/a.cpp\nDA:5,0\nLF:1\nLH:0\nend_of_record\n" + result = _make_lcov_paths_relative(lcov, "/ws/root/") + self.assertIn("TN:test\n", result) + self.assertIn("DA:5,0\n", result) + + +class MakeHtmlPathsRelativeTest(unittest.TestCase): + def test_source_title_is_rewritten_and_hrefs_untouched(self): + html = ( + "
/ws/root/src/foo.cpp
" + "link" + ) + with tempfile.TemporaryDirectory() as tmp: + page = Path(tmp) / "page.html" + page.write_text(html) + _make_html_paths_relative(Path(tmp), "/ws/root/") + result = page.read_text() + self.assertIn("
src/foo.cpp
", result) + # The href embeds the path components without a leading slash and must + # never be rewritten. + self.assertIn("href='coverage/ws/root/src/foo.cpp.html'", result) + + def test_missing_dir_is_a_noop(self): + _make_html_paths_relative(Path("/nonexistent/html_dir"), "/ws/root/") + + +class WriteEmptyOutputTest(unittest.TestCase): + def test_produces_valid_empty_zip(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "out.zip" + write_empty_output(out) + self.assertTrue(zipfile.is_zipfile(out)) + with zipfile.ZipFile(out) as zf: + self.assertEqual(zf.namelist(), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/defs.bzl b/defs.bzl index c90e26f9..375d2329 100644 --- a/defs.bzl +++ b/defs.bzl @@ -17,7 +17,11 @@ load("//cli_helper:cli_helper.bzl", _cli_helper = "cli_helper") # --- coverage --- -load("//coverage:coverage.bzl", _rust_coverage_report = "rust_coverage_report") +load( + "//coverage:defs.bzl", + _score_coverage_reporter = "score_coverage_reporter", + _score_coverage_scope = "score_coverage_scope", +) # --- dash --- load("//dash:dash.bzl", _dash_license_checker = "dash_license_checker") @@ -44,6 +48,7 @@ score_py_pytest = _score_py_pytest dash_license_checker = _dash_license_checker cli_helper = _cli_helper setup_starpls = _setup_starpls -rust_coverage_report = _rust_coverage_report +score_coverage_scope = _score_coverage_scope +score_coverage_reporter = _score_coverage_reporter trlc_requirements_ai_test = _trlc_requirements_ai_test architecture_ai_test = _architecture_ai_test