From d8a74ae15b9d0cefd6588c787b52672a6bcb8da0 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Mon, 10 Aug 2026 21:01:47 -0700 Subject: [PATCH 1/4] refactor(coerce): bring sig() onto the same fuzzy-matching rigor as the column coercions sig() derived statistical_significance_qualifier from a p-value column chosen by a naive substring selector ([c for c in names if col in c] + fuzz.ratio). The four coerce_* steps already share a rigorous pattern: a *_target() regex classifier, a canonical-name-wins guard, and fuzz.ratio against target.replace('_',' '). sig() now reuses pvalue_target() for the same classification, prefers a raw p_value column over adjusted_p_value, and applies canonical-wins. Five-band cascade and Biolink class rule (qualifier omitted without a p-value column) are unchanged. Coerce_pvalue_columns runs before sig in the pipeline, so realistic builds are unaffected; only a contrived p_value-substring column that pvalue_target rejects now correctly omits the qualifier. Adds three rigor tests. --- src/tablassert/coerce.py | 34 +++++++++++++++++++++++++------ tests/test_lib.py | 44 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/tablassert/coerce.py b/src/tablassert/coerce.py index f0efabd..7f07f26 100644 --- a/src/tablassert/coerce.py +++ b/src/tablassert/coerce.py @@ -15,12 +15,20 @@ def sig(lf: pl.LazyFrame, col: str = "p_value", out: str = "statistical_significance_qualifier") -> pl.LazyFrame: """Create the ``statistical_significance_qualifier`` column (Biolink PR #1766). - Picks the closest fuzzy-matching p-value-like column when the exact name - is missing, then buckets the value into one of five significance bands. + Picks the p-value column via ``pvalue_target`` classification (raw ``p_value`` + preferred, canonical-wins), then buckets the value into one of five significance bands. + + Candidate selection mirrors :func:`coerce_pvalue_columns`: a column counts + only when :func:`pvalue_target` accepts it (not by naive substring), an + existing canonical column always wins over a higher-scoring spaced alias, + and ties break by ``rapidfuzz`` ratio against ``target.replace("_", " ")``. + A raw ``p_value`` column is preferred over ``adjusted_p_value``. Args: lf: Source LazyFrame. - col: Reference column name to fuzzy-match against. + col: Preferred canonical target (``"p_value"`` raw or + ``"adjusted_p_value"``); the other bucket is the fallback. Kept for + API compatibility — selection still runs through :func:`pvalue_target`. out: Output qualifier column name. Returns: @@ -40,11 +48,25 @@ def sig(lf: pl.LazyFrame, col: str = "p_value", out: str = "statistical_signific from rapidfuzz import fuzz names: list[str] = lf.collect_schema().names() - candidates: list[str] = [c for c in names if col in c] - chosen: str | None = max(candidates, key=lambda c: fuzz.ratio(c, col)) if candidates else None - if chosen is None: + # Same rigorous classification as ``coerce_pvalue_columns``: a column is a significance + # source only when ``pvalue_target`` accepts it — NOT by naive substring — so non-p-value + # columns that merely contain the reference text stay out of the qualifier. + buckets: dict[str, list[str]] = {} + for name in names: + target: str | None = pvalue_target(name) + if target: + buckets.setdefault(target, []).append(name) + if not buckets: # Biolink class rule: qualifier may only be set when p_value/adjusted_p_value is populated. return lf + # Prefer the requested target (raw ``p_value`` by default); fall back to whichever p-value + # bucket is present. Raw p-value is the canonical significance source; adjusted is the fallback. + preferred: str = col if col in buckets else next(iter(buckets)) + candidates: list[str] = buckets[preferred] + reference: str = preferred.replace("_", " ") + # An existing canonical column always wins; fuzzy ranking only picks among aliases + # (same rule as ``coerce_pvalue_columns``). + chosen: str = preferred if preferred in candidates else max(candidates, key=lambda c: fuzz.ratio(c, reference)) expr: pl.Expr = pl.col(chosen).cast(pl.Float64, strict=False) band: pl.Expr = ( pl.when(expr.is_null()) diff --git a/tests/test_lib.py b/tests/test_lib.py index d9ccb06..420f743 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -737,10 +737,11 @@ def test_sig_uses_non_exact_p_value_column() -> None: def test_sig_picks_closest_non_exact_match() -> None: - """sig picks closest match when multiple non-exact columns present.""" + """sig prefers a raw p-value bucket over an adjusted one when both are non-exact.""" lf: pl.LazyFrame = pl.DataFrame({"log_p_value": [0.01], "adjusted_p_value_corrected": [0.5]}).lazy() result: pl.DataFrame = lib.sig(lf).collect() - # "log_p_value" has higher fuzz.ratio to "p_value" than "adjusted_p_value_corrected" + # "log_p_value" -> raw p_value bucket; "adjusted_p_value_corrected" -> adjusted bucket. + # The raw bucket is preferred, so 0.01 -> strongly_significant (not 0.5 -> not_significant). assert list(result["statistical_significance_qualifier"]) == ["biolink:strongly_significant"] @@ -792,6 +793,45 @@ def test_sig_not_significant_band() -> None: assert list(result["statistical_significance_qualifier"]) == ["biolink:not_significant", "biolink:not_significant"] +def test_sig_prefers_raw_p_value_over_adjusted_bucket() -> None: + """sig derives the qualifier from a raw p-value column, not a co-present adjusted one. + + ``"P"`` classifies as the raw ``p_value`` bucket (bare-P token) and ``"FDR"`` as the + ``adjusted_p_value`` bucket. The qualifier must follow the raw column: 0.01 maps to + ``strongly_significant``, whereas the adjusted 0.001 would wrongly yield + ``very_strongly_significant``. + """ + lf: pl.LazyFrame = pl.DataFrame({"P": [0.01], "FDR": [0.001]}).lazy() + result: pl.DataFrame = lib.sig(lf).collect() + assert list(result["statistical_significance_qualifier"]) == ["biolink:strongly_significant"] + + +def test_sig_canonical_column_wins_over_higher_scoring_alias() -> None: + """An existing canonical ``p_value`` column wins over a higher-scoring spaced alias. + + Both ``"p_value"`` and ``"p vals"`` land in the raw bucket; the canonical column is + chosen directly (no fuzzy tiebreak), so banding follows ``p_value``=0.05 + (``significant``) rather than ``p vals``=0.001 (``very_strongly_significant``). + """ + lf: pl.LazyFrame = pl.DataFrame({"p_value": [0.05], "p vals": [0.001]}).lazy() + result: pl.DataFrame = lib.sig(lf).collect() + assert list(result["statistical_significance_qualifier"]) == ["biolink:significant"] + + +def test_sig_excludes_substring_only_non_pvalue_column() -> None: + """sig uses pvalue_target, not a naive substring, so a look-alike column is ignored. + + ``"xp_value_x"`` contains the literal ``p_value`` substring (the old selector would + grab it) but ``pvalue_target`` rejects it: the ``p`` is glued to an alphanumeric on + both sides, so neither the value token nor the bare-P token matches. With no real + p-value column the qualifier is omitted (Biolink class rule). + """ + lf: pl.LazyFrame = pl.DataFrame({"xp_value_x": [0.01], "gene": ["BRCA1"]}).lazy() + result: pl.DataFrame = lib.sig(lf).collect() + assert "statistical_significance_qualifier" not in result.columns + assert result.height == 1 + + def test_drop_not_significant_removes_band_keeps_nulls() -> None: """drop_not_significant removes biolink:not_significant rows while keeping null qualifiers.""" lf: pl.LazyFrame = pl.DataFrame( From d9344f6c77c10422de8612ebc7fa47aa973a9b54 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Mon, 10 Aug 2026 21:01:47 -0700 Subject: [PATCH 2/4] test(e2e): smoke the statistical-column coercion pipeline end-to-end Extends the real-redb build_pipeline smoke to prove the whole coercion pipeline wires through end-to-end: raw annotation names (p value, sample size, odds ratio, effect type) normalize to canonical Biolink edge fields (p_value as a JSON number, effect_size in controlled notation, effect_type with the alias mapped to the EffectTypes enum), while supporting_study_size and the auto-derived statistical_significance_qualifier route into the inlined Study. Builds a tiny redb, not a real build-fullmap. --- tests/test_e2e_smoke.py | 80 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/test_e2e_smoke.py b/tests/test_e2e_smoke.py index 7edff84..4a72ff6 100644 --- a/tests/test_e2e_smoke.py +++ b/tests/test_e2e_smoke.py @@ -133,3 +133,83 @@ def test_validate_command_happy_path(tmp_path: Path) -> None: assert validate_pipeline(config, PipelineProgress(total_stages=3)) is None # The cyclopts command wrapper (cli.py validate -> run(3, validate_pipeline, ...)). assert validate(config, schema="table") is None + + +def test_build_pipeline_coerces_statistical_annotations(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The real pipeline normalizes raw statistical column names to canonical Biolink fields. + + Declares annotations with non-canonical source spellings (``p value``, ``sample size``, + ``odds ratio``, ``effect type``) and asserts the emitted KGX edge carries the coerced + canonical fields flat on the edge (``p_value`` as a JSON number, ``effect_size`` in + controlled notation, ``effect_type`` with the alias mapped to the ``EffectTypes`` enum) + and routes the auto-derived ``statistical_significance_qualifier`` plus + ``supporting_study_size`` into the inlined Study. Proves the whole coercion pipeline + (``coerce_pvalue_columns`` / ``coerce_study_size_columns`` / ``coerce_effect_size_columns`` + / ``coerce_effect_type_columns`` / ``sig``) wires through ``build_pipeline`` end-to-end. + """ + monkeypatch.chdir(tmp_path) + (tmp_path / ".tablassert" / "store").mkdir(parents=True) + + fullmap: Path = _build_real_redb(tmp_path / "fullmap") + + # A=subject B=object C=p value D=sample size E=odds ratio(effect size) F=effect type + data: Path = tmp_path / "data.tsv" + data.write_text("brca1\tmapk1\t0.01\t450\t0.85\tSpearman\n") + + table: Path = tmp_path / "table.yaml" + table_config: dict[str, Any] = { + "template": { + "source": {"kind": "text", "local": str(data), "url": ["https://example.com/data.tsv"], "delimiter": "\t"}, + "statement": { + "subject": {"method": "column", "encoding": "A"}, + "predicate": "associated_with", + "object": {"method": "column", "encoding": "B"}, + }, + "provenance": {"repo": "PMC", "publication": "PMC0000000"}, + "annotations": [ + {"annotation": "p value", "method": "column", "encoding": "C"}, + {"annotation": "sample size", "method": "column", "encoding": "D"}, + {"annotation": "odds ratio", "method": "column", "encoding": "E"}, + {"annotation": "effect type", "method": "column", "encoding": "F"}, + ], + } + } + to_yaml(table, table_config) + + graph: Path = tmp_path / "graph.yaml" + graph_config: dict[str, Any] = { + "name": "COERCE_KG", + "version": "1.0.0", + "description": "coercion smoke graph", + "tables": [str(table)], + "fullmap": str(fullmap), + } + to_yaml(graph, graph_config) + + build_pipeline(graph, PipelineProgress(total_stages=6)) + + edges_path: Path = tmp_path / "COERCE_KG_1.0.0.edges.ndjson" + assert edges_path.is_file() + edge_text: str = edges_path.read_text() + edges: list[dict[str, Any]] = [json.loads(line) for line in edge_text.splitlines() if line.strip()] + assert len(edges) == 1 + edge: dict[str, Any] = edges[0] + + # Raw annotation names normalized to canonical Biolink fields flat on the edge. + # p_value is a numeric Biolink float slot (emitted as a real JSON number), while + # effect_size has no numeric slot and keeps controlled {:.4g} string notation. + assert isinstance(edge["p_value"], float) + assert isinstance(edge["effect_size"], str) + assert float(edge["p_value"]) == 0.01 + assert edge["effect_size"] == "0.85" + assert edge["effect_type"] == "spearmans_rho" # "Spearman" alias mapped to the EffectTypes enum + assert "sample size" not in edge + assert "odds ratio" not in edge + assert "effect type" not in edge + + # supporting_study_size + the auto-derived statistical_significance_qualifier are + # UNSATISFIABLE edge fields, so they ride the inlined Study rather than the edge. + assert "supporting_study_size" not in edge + assert "statistical_significance_qualifier" not in edge + assert "supporting_study_size=450" in edge_text + assert "statistical_significance_qualifier=biolink:strongly_significant" in edge_text From 1caa8bf13d2e7241f2c2bd69cffc22adccc742c9 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Mon, 10 Aug 2026 21:01:47 -0700 Subject: [PATCH 3/4] docs(table): document automatic column coercion The p_value / supporting_study_size / effect_size / effect_type auto-normalization, effect_type value-to-enum mapping, relationship_strength forward-rename, the auto-derived statistical_significance_qualifier bands, and the Biolink class rules were previously undocumented behavior. Adds an 'Automatic column coercion' subsection to the table configuration reference. --- docs/configuration/table.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/configuration/table.md b/docs/configuration/table.md index 4dc6e4d..dc71c02 100644 --- a/docs/configuration/table.md +++ b/docs/configuration/table.md @@ -446,6 +446,24 @@ This means nothing in your source data is silently dropped: context that doesn't In addition to user-declared annotations, every edge automatically carries `extracted_from_row_number`, a 1-based index into the original source table (matching Excel-style row numbering). It is not declared as an annotation — tablassert emits it internally so each edge always carries its source-row provenance, and it folds into `supporting_text` like any other non-allow-list column (e.g. `"extracted_from_row_number: 42"`). +#### Automatic column coercion + +Before the allow-list sweep runs, tablassert renames statistical columns to their canonical Biolink names so source headers do not have to match exactly. Recognition is delimiter-anchored (spaces, `_`, `-`, `.` are interchangeable) and the **best fuzzy match per target wins, with an existing canonical column always preferred** over a higher-scoring spaced alias. + +| Recognized as | Canonical name | Typical source spellings | +|---|---|---| +| P value (raw) | `p_value` | `p value`, `p-value`, `pvalue`, `P`, `gwas p`, `raw_p`, `pvalue1` | +| Adjusted P value | `adjusted_p_value` | `padj`, `p.adj`, `adj.P.Val`, `FDR`, `Bonferroni`, `Holm`, `q value` | +| Study size | `supporting_study_size` | `n`, `sample_size`, `study size`, `cohort_size`, `participants_n`, `enrollment` | +| Effect size | `effect_size` | `effect size`, `odds ratio`, `hazard ratio`, `beta`, `log2FC`, `correlation`, `rho` (and the legacy `relationship_strength`) | +| Effect type | `effect_type` | `effect type`, `effect metric`, `statistic type`, `metric` | + +- **`effect_type` values are also coerced.** Each cell is matched case/separator-insensitively against an alias table (e.g. `"OR"` → `odds_ratio`, `"Cohen's d"` → `cohens_d`, `"Spearman"` → `spearmans_rho`), then by `rapidfuzz` fallback against the 25 permissible `EffectTypes` values; anything matching nothing is dropped to `null` rather than carried through (the Biolink range is the enum). +- **`statistical_significance_qualifier` is auto-derived** from the p-value column into five bands — `biolink:very_strongly_significant` (p ≤ 0.001), `biolink:strongly_significant` (≤ 0.01), `biolink:significant` (≤ 0.05), `biolink:suggestive` (≤ 0.10), `biolink:not_significant` (> 0.10). The same rigorous selection picks the source column: a raw `p_value` column is preferred, `adjusted_p_value` is the fallback, and the qualifier is omitted entirely when no p-value column is present. +- **Biolink class rules are enforced.** `effect_type` is nulled on every row where `effect_size` is null (and nulled entirely when no `effect_size` column exists); `statistical_significance_qualifier` is only set when `p_value`/`adjusted_p_value` is populated, and null p-values yield a null qualifier. + +This is why declaring an annotation like `{annotation: p value, method: column, encoding: E}` still produces a top-level `p_value` edge field — the header is normalized to the Biolink name before folding is considered. + ## Next Steps - **[Advanced Example](advanced-example.md)** - Real-world configuration with complex transformations From f6afa66a41375e765ffd2dbc8d460ce20fe5bfb2 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Mon, 10 Aug 2026 21:01:48 -0700 Subject: [PATCH 4/4] chore(release): 8.2.1 Bumps version 8.2.0 -> 8.2.1 (pyproject.toml, uv.lock self-package, CITATION.cff) and folds the unreleased section into 8.2.1: the multi-section source.url list breaking change, the build-fullmap --aria2c opt-in, the sig() rigor unification, the coercion docs, and the new tests. --- CHANGELOG.md | 9 ++++++++- CITATION.cff | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beed3c8..e477c95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project are documented in this file. -## Unreleased +## 8.2.1 - 2026-08-10 ### Breaking Changes - **`source.url` is now a list of URLs (`url: list[HttpUrl]`).** A table-config section may declare one or more remote source URLs, all recorded as provenance (emitted in the edge `source_record_urls` list and the RIG). The legacy scalar form `url: https://example.com/x.tsv` is no longer accepted — wrap it in a list. Update existing configs from `url: https://...` to a sequence: @@ -15,6 +15,13 @@ All notable changes to this project are documented in this file. ### Added - **`tablassert build-fullmap --aria2c` / `-a`** opt-in downloader acceleration. When requested, the BABEL download stage uses the installed `aria2c` executable with segmented HTTP downloads plus resume/retry flags (`--continue=true`, `--max-tries`, `--retry-wait`) while keeping the existing Python downloader as the default. Missing or failing `aria2c` fails loud instead of silently falling back, and aria2 `.aria2` control files are preserved so interrupted downloads can resume on rerun. +- New regression tests: three `sig()` rigor tests (raw-`p_value`-over-adjusted preference, canonical-column-wins-over-alias, and `pvalue_target`-based exclusion of a look-alike substring column) plus an end-to-end smoke proving the real pipeline normalizes raw statistical annotation names (`p value`, `sample size`, `odds ratio`, `effect type`) to canonical Biolink edge fields and routes `supporting_study_size` / `statistical_significance_qualifier` into the inlined Study. + +### Changed +- **`sig()` now applies the same fuzzy-matching rigor as the column coercions.** The `statistical_significance_qualifier` is derived from a p-value column chosen by `pvalue_target()` — the same delimiter-anchored classifier `coerce_pvalue_columns` uses — rather than a naive substring. A raw `p_value` column is preferred over `adjusted_p_value`, and an existing canonical column always wins over a higher-scoring spaced alias, mirroring the selection rule every other `coerce_*` step already uses. The five-band cascade and the Biolink class rule (qualifier omitted when no p-value column is present) are unchanged. No realistic build is affected: `coerce_pvalue_columns` canonicalizes every p-value column before `sig` runs, so the old and new selectors pick the same column; the lone divergence is a contrived column that merely contains a `p_value` substring but is not a real p-value column, which now correctly omits the qualifier instead of deriving a bogus one. + +### Documentation +- **Documented automatic column coercion** in the table-configuration reference: p-value / study-size / effect-size / effect-type columns are auto-normalized to canonical Biolink names before the edge allow-list sweep (with `effect_type` values mapped to the `EffectTypes` enum, unmatched values dropped to `null`, and the legacy `relationship_strength` renamed forward to `effect_size`), `statistical_significance_qualifier` is auto-derived into five significance bands, and the Biolink class rules that null `effect_type` where `effect_size` is absent and omit the qualifier without a p-value column. ## 8.2.0 - 2026-08-10 diff --git a/CITATION.cff b/CITATION.cff index d470200..9738338 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ cff-version: 1.2.0 message: "If you use Tablassert, please cite it as below." type: software title: Tablassert -version: 8.2.0 +version: 8.2.1 license: Apache-2.0 repository-code: https://github.com/SkyeAv/Tablassert abstract: Tablassert is a highly performant declarative knowledge graph backend for bioinformatics that extracts knowledge assertions from tabular data, performs entity resolution and data quality control, and exports NCATS Translator-compliant Knowledge Graph Exchange (KGX) NDJSON. diff --git a/pyproject.toml b/pyproject.toml index 0200126..11c4f2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tablassert" -version = "8.2.0" +version = "8.2.1" description = "Extract knowledge assertions from tabular data into NCATS Translator-compliant KGX NDJSON — declaratively, with entity resolution and quality control built in." authors = [ { name = "Skye Lane Goetz", email = "sgoetz@isbscience.org" } diff --git a/uv.lock b/uv.lock index 6e50d0a..81e2471 100644 --- a/uv.lock +++ b/uv.lock @@ -4098,7 +4098,7 @@ wheels = [ [[package]] name = "tablassert" -version = "8.2.0" +version = "8.2.1" source = { editable = "." } dependencies = [ { name = "biolink-model" },