feat: external_needs attribute

BUILD

@@ -17,8 +17,8 @@ package(default_visibility = ["//visibility:public"])
 exports_files(["pyproject.toml"])
 
 docs(
-    data = [
-        "@score_process//:needs_json",
+    external_needs = [
+        "@score_process//:needs_json_file",
     ],
     bundles = [
         {

docs.bzl

@@ -141,6 +141,7 @@ def docs(
         source_dir = "docs",
         data = [],
         deps = [],
+        external_needs = [],
         scan_code = [],
         test_sources = [],
         known_good = None,
@@ -155,6 +156,7 @@ def docs(
       source_dir: The source directory containing documentation files. Defaults to "docs".
       data: Additional data files to include in the documentation build.
       deps: Additional dependencies for the documentation build.
+      external_needs: List of external needs targets to include in the documentation build.
       scan_code: List of code targets to scan for source code links.
       test_sources: Optional list of repo-relative directory paths which will be used to filter testcases for documentation generation.
                     When empty (default), all testcases found in `bazel-testlogs` will be used.
@@ -171,6 +173,7 @@ def docs(
               Note: a bundle label may also point at another module's auto-exposed
               bundle, e.g. "@score_process//:docs_bundle".
     """
+    # HINT: keep documentation sync docs/reference/bazel_macros.rst
 
     source_config = ":" + ("" if source_dir == "." else source_dir + "/") + "conf.py"
 
@@ -205,7 +208,7 @@ def docs(
     sphinx_build_binary(
         name = "sphinx_build",
         visibility = ["//visibility:private"],
-        data = data + metamodel_label + [":docs_bundle"],
+        data = data + external_needs + metamodel_label + [":docs_bundle"],
         deps = deps,
     )
 
@@ -236,13 +239,14 @@ def docs(
     # complete bundle here would add those files to runfiles and could collide
     # with the executable target name (for example ``docs`` and ``docs/``).
     # External bundles do need runfiles, so keep only those sources.
-    docs_data = data + metamodel_label + [":sourcelinks_json", ":_external_docs_runfiles"] + mounts_manifest_label
+    docs_data = data + external_needs + metamodel_label + [":sourcelinks_json", ":_external_docs_runfiles"] + mounts_manifest_label
 
     docs_env = {
         "SOURCE_DIRECTORY": source_dir,
         "PACKAGE_DIR": native.package_name(),
         "TEST_SOURCES": str(test_sources),
         "DATA": str(data),
+        "EXTERNAL_NEEDS_FILES": str(external_needs),
         # `bazel run` starts from a runfiles tree, so this logical path is
         # resolved by score_mounts through ``RUNFILES_DIR``.
         "MOUNTS_MANIFEST": "$(rlocationpath :_mounts_manifest)" if bundles else "",
@@ -317,7 +321,7 @@ def docs(
             "-T",  # show more details in case of errors
             "--jobs",
             "auto",
-            "--define=external_needs_source=" + str(data),
+            "--define=external_needs_source=" + str(data + external_needs),
             "--define=score_sourcelinks_json=$(location :sourcelinks_json)",
             "--define=score_source_code_linker_plain_links=1",
         ] + (
@@ -327,7 +331,7 @@ def docs(
         ) + (["--define=score_metamodel_yaml=$(location " + str(metamodel) + ")"] if metamodel else []),
         formats = ["needs"],
         sphinx = ":sphinx_build",
-        tools = data + metamodel_label + [":sourcelinks_json", ":docs_bundle"] + mounts_manifest_label,
+        tools = data + external_needs + metamodel_label + [":sourcelinks_json", ":docs_bundle"] + mounts_manifest_label,
         visibility = ["//visibility:public"],
         # Persistent workers cause stale symlinks after dependency version
         # changes, corrupting the Bazel cache.
@@ -342,6 +346,16 @@ def docs(
         visibility = ["//visibility:public"],
     )
 
+    native.genrule(
+        # In contrast to the "needs_json" target represents *only* the needs.json file,
+        # not the whole needs build output.
+        name = "needs_json_file",
+        srcs = [":needs_json"],
+        outs = ["needs.json"],
+        cmd = "cp $(location :needs_json)/needs.json $@",
+        visibility = ["//visibility:public"],
+    )
+
     native.alias(
         name = "traceability_gate",
         actual = Label("//scripts_bazel:traceability_gate"),

docs/concepts/bidirectional_traceability.rst

@@ -73,9 +73,8 @@ Module links vs Bazel target deps
 
 Remember: Bazel target dependencies must not be circular.
 Module and document references may be circular.
-Use the combo build and the copy strategy to produce release documentation that contains all needed pages while keeping Bazel's graph acyclic.
-
-
+Use bundles to produce release documentation that contains all needed pages
+while keeping Bazel's graph acyclic.
 
 .. plantuml::
 

docs/how-to/other_modules.rst

@@ -41,24 +41,23 @@ A minimal example (add or extend the existing `bazel_deps` stanza):
 2a) Import the other module's built inventory
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The documentation build in this project is exposed via a Bazel macro/rule that accepts a `data` parameter.
-Add the external module's ``:needs_json`` target to that list
-to have their needs elements available for cross-referencing.
+The documentation build is exposed via a Bazel macro that accepts an ``external_needs`` parameter
+for external ``:needs_json_file`` targets.
+Use ``external_needs`` instead of ``data`` when the target produces needs JSON —
+``data`` is meant for non-needs runfiles (e.g. custom tool outputs).
 
 Example `BUILD` snippet (consumer module):
 
 .. code-block:: starlark
 
     load("@score_docs_as_code//:docs.bzl", "docs")
     docs(
-      data = [
+      external_needs = [
          "@score_process//:needs_json",
       ],
       source_dir = "docs",
     )
 
-More details in :ref:`docs_bidirectional_traceability`.
-
 
 2b) Mount the external module's documentation bundle
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/how-to/setup.md

@@ -73,19 +73,7 @@ docs(
 )
 ```
 
-
-#### Configuration Options
-
-The `docs()` macro accepts the following arguments:
-
-| Parameter | Description | Required |
-|-----------|-------------|----------|
-| `source_dir` | Directory of documentation source files (RST, MD) | Yes |
-| `data` | List of `needs_json` targets that should be included in the documentation | No |
-| `deps` | Additional Bazel Python dependencies | No |
-| `scan_code` | Source code targets to scan for traceability tags | No |
-| `known_good` | Label to a "known good" JSON file for source links | No |
-| `metamodel` | Label to a custom `metamodel.yaml` that replaces the default metamodel | No |
+For configuration options see {ref}`docs_bazel-macros`.
 
 ### 4. Copy conf.py
 

docs/reference/bazel_macros.rst

@@ -76,6 +76,11 @@ Minimal example (root ``BUILD``)
   Source code targets to scan for traceability tags (``req-Id:`` annotations).
   Used to generate the source-code-link JSON that maps tags back to source files.
 
+- ``external_needs`` (list of bazel labels)
+  External ``:needs_json_file`` targets from other modules/repositories
+  for referencing their needs.
+  Do not use ``:needs_json`` targets.
+
 - ``metamodel`` (bazel label, optional)
   Path to a custom ``metamodel.yaml`` file.
   When set, the ``score_metamodel`` extension loads **this file instead of** the default metamodel.

docs/reference/commands.md

@@ -24,7 +24,6 @@
 | `bazel run //:docs`                            | Builds documentation (also writes `metrics.json`)                                                 |
 | `bazel run //:docs_check`                      | Verifies documentation correctness                                                                |
 | `bazel run //:docs_link_check`                 | Lists broken links                                                                                |
-| `bazel run //:docs_combo`                      | Builds combined documentation with all external dependencies included                             |
 | `bazel run //:traceability_gate -- --metrics-json bazel-bin/needs_json/_build/needs/metrics.json --min-req-code 70 --min-req-test 70 --min-req-fully-linked 60 --min-tests-linked 70` | Reads the pre-computed metrics.json and fails if coverage thresholds are not met                   |
 | `bazel run //:live_preview`                    | Creates a live_preview of the documentation viewable in a local server                            |
 | `bazel run //:ide_support`                     | Sets up a Python venv for esbonio (Remember to restart VS Code!)                                  |

src/extensions/score_metamodel/external_needs.py

@@ -54,7 +54,7 @@ def _parse_bazel_external_need(s: str) -> ExternalNeedsSource | None:
     repo, path_to_target = repo_and_path.split("//", 1)
     repo = repo.lstrip("@")  # empty for same-repo `//pkg:needs_json`
 
-    if target in ("needs_json", "docs_sources"):
+    if target in ("needs_json", "needs_json_file", "docs_sources"):
         return ExternalNeedsSource(
             bazel_module=repo,
             path_to_target=path_to_target,
@@ -159,12 +159,11 @@ def temp(self: NeedsList):
 def get_external_needs_source(external_needs_source: str) -> list[ExternalNeedsSource]:
     if external_needs_source:
         # Path taken for all invocations via `bazel`
-        external_needs = parse_external_needs_sources_from_DATA(external_needs_source)
+        return parse_external_needs_sources_from_DATA(external_needs_source)
     else:
         # This is the path taken for anything that doesn't
         # run via `bazel`  e.g. esbonio or other direct executions
-        external_needs = parse_external_needs_sources_from_bazel_query()  # pyright: ignore[reportAny]
-    return external_needs
+        return parse_external_needs_sources_from_bazel_query()  # pyright: ignore[reportAny]
 
 
 def add_external_needs_json(e: ExternalNeedsSource, config: Config):
@@ -182,9 +181,9 @@ def add_external_needs_json(e: ExternalNeedsSource, config: Config):
         needs_json_data = json.loads(Path(json_file).read_text(encoding="utf-8"))  # pyright: ignore[reportAny]
     except FileNotFoundError:
         logger.error(
-            f"Could not find external needs JSON file at {json_file}. "
-            + "Something went terribly wrong. "
-            + "Try running `bazel clean --async && rm -rf _build`."
+            "Could not find external needs JSON file at %s from target %s.",
+            json_file,
+            e.target,
         )
         # Attempt to continue, exit code will be non-zero after a logged error anyway.
         return
@@ -227,6 +226,7 @@ def add_external_docs_sources(e: ExternalNeedsSource, config: Config):
 def connect_external_needs(app: Sphinx, config: Config):
     extend_needs_json_exporter(config, ["project_url"])
 
+    # Local external needs from DATA (e.g. :needs_json or :docs_sources)
     external_needs = get_external_needs_source(app.config.external_needs_source)
 
     # this sets the default value - required for the needs-config-writer
@@ -236,9 +236,41 @@ def connect_external_needs(app: Sphinx, config: Config):
     for e in external_needs:
         if e.target == "needs_json":
             add_external_needs_json(e, app.config)
+        elif e.target == "needs_json_file":
+            _add_needs_json_file(e, app.config)
         elif e.target == "docs_sources":
             add_external_docs_sources(e, app.config)
         else:
             raise ValueError(
                 f"Internal Error. Unknown external needs target: {e.target}"
             )
+
+
+def _add_needs_json_file(ext_needs: ExternalNeedsSource, config: Config) -> None:
+    """Resolve a needs_json_file target from runfiles and register it."""
+    json_file_raw = (
+        Path(_runfiles_module_dir(ext_needs)) / ext_needs.path_to_target / "needs.json"
+    )
+    r = get_runfiles_dir()
+    json_file = r / json_file_raw
+    logger.debug(f"External needs_json_file: {json_file}")
+    try:
+        needs_json_data = json.loads(
+            Path(json_file).read_text(encoding="utf-8")  # pyright: ignore[reportAny]
+        )
+    except FileNotFoundError:
+        logger.error(
+            "Could not find external needs JSON file at %s from target %s.",
+            json_file,
+            ext_needs.target,
+        )
+        return
+    except json.JSONDecodeError as exc:
+        logger.error(f"Failed to parse external needs JSON file {json_file}: {exc}")
+        return
+    config.needs_external_needs.append(
+        {  # pyright: ignore[reportUnknownMemberType]
+            "base_url": needs_json_data.get("project_url", "") + "/main",
+            "json_path": json_file,
+        }
+    )

src/extensions/score_metamodel/tests/test_external_needs.py

@@ -24,6 +24,7 @@
 import score_metamodel.external_needs as ext_needs
 from score_metamodel.external_needs import (
     ExternalNeedsSource,
+    _add_needs_json_file,
     add_external_docs_sources,
     add_external_needs_json,
     get_external_needs_source,
@@ -202,6 +203,41 @@ def test_add_external_needs_json_appends_entry_local(
     assert Path(entry["json_path"]) == json_path
 
 
+def test_add_needs_json_file_appends_entry(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """_add_needs_json_file should load from a :needs_json_file target."""
+    # Arrange: create the needs.json at the runfiles path
+    rel_json = Path("ext_mod+/needs.json")
+    json_path = tmp_path / rel_json
+    json_path.parent.mkdir(parents=True, exist_ok=True)
+    json_path.write_text(
+        json.dumps({"project_url": "https://example.test/json-file"}), encoding="utf-8"
+    )
+
+    runfiles_dir = tmp_path
+    config = Config()
+    config.needs_external_needs = []
+
+    monkeypatch.setattr(ext_needs, "get_runfiles_dir", lambda: runfiles_dir)
+
+    # Act
+    e = ExternalNeedsSource(
+        bazel_module="ext_mod",
+        target="needs_json_file",
+        path_to_target="",
+        is_local=False,
+    )
+    _add_needs_json_file(e, config)
+
+    # Assert
+    assert config.needs_external_needs is not None
+    assert len(config.needs_external_needs) == 1
+    entry = config.needs_external_needs[0]
+    assert entry["base_url"] == "https://example.test/json-file/main"
+    assert Path(entry["json_path"]) == json_path
+
+
 def test_add_external_needs_json_missing_file_keeps_list_empty(
     monkeypatch: pytest.MonkeyPatch, tmp_path: Path
 ) -> None:

src/incremental.py

@@ -13,6 +13,7 @@
 
 import argparse
 import hashlib
+import json
 import logging
 import os
 import shutil
@@ -42,6 +43,17 @@ def get_env(name: str) -> str:
     return val
 
 
+def _merged_external_needs() -> str:
+    """Combine DATA and EXTERNAL_NEEDS_FILES into one JSON label list.
+
+    Both env vars hold JSON lists of Bazel labels; the extension parses the
+    resulting `external_needs_source` define uniformly.
+    """
+    data = json.loads(get_env("DATA") or "[]")
+    external = json.loads(os.environ.get("EXTERNAL_NEEDS_FILES", "[]") or "[]")
+    return json.dumps(data + external)
+
+
 def _compute_hash(files: list[Path]) -> str:
     h = hashlib.sha256()
     for f in sorted(files, key=str):
@@ -142,7 +154,11 @@ def _mounted_watch_dirs(
         "-T",  # show details in case of errors in extensions
         "--jobs",
         "auto",
-        f"--define=external_needs_source={get_env('DATA')}",
+        # Merge DATA (:needs_json / :docs_sources) with EXTERNAL_NEEDS_FILES
+        # (:needs_json_file) into a single define. The sphinx_docs rule cannot
+        # receive per-target env vars, so --define is the only channel that
+        # works for both the py_binary and the needs_json target.
+        f"--define=external_needs_source={_merged_external_needs()}",
         f"--define=testcase_source_dirs={os.environ.get('TEST_SOURCES', '[]')}",
         # Path to the Bazel-emitted mounts manifest (empty when no mounts are
         # configured); consumed by the score_mounts extension.

src/tests/docs_bzl/scenarios/external_needs/consumer/BUILD

@@ -18,6 +18,6 @@ load("//:docs.bzl", "docs")
 # avoiding the workspace-wide bazel-testlogs scan during the html build.
 docs(
     source_dir = "docs",
-    data = ["//src/tests/docs_bzl/scenarios/external_needs/producer:needs_json"],
+    external_needs = ["//src/tests/docs_bzl/scenarios/external_needs/producer:needs_json"],
     test_sources = ["src/tests/docs_bzl/scenarios/external_needs/consumer"],
 )