Normalize OWASP cheat sheet references

application/tests/cheatsheets_parser_test.py

@@ -8,6 +8,7 @@
 import tempfile
 from unittest.mock import patch
 import os
+import subprocess
 
 
 class TestCheatsheetsParser(unittest.TestCase):
@@ -34,7 +35,13 @@ class Repo:
         repo.working_dir = loc
         cre = defs.CRE(name="blah", id="223-780")
         self.collection.add_cre(cre)
-        with open(os.path.join(os.path.join(loc, "cheatsheets"), "cs.md"), "w") as mdf:
+        with open(
+            os.path.join(
+                os.path.join(loc, "cheatsheets"),
+                "Secrets_Management_Cheat_Sheet.md",
+            ),
+            "w",
+        ) as mdf:
             mdf.write(cs)
         mock_clone.return_value = repo
         entries = cheatsheets_parser.Cheatsheets().parse(
@@ -45,22 +52,149 @@ class Repo:
         # verify the external tagging convention, not just enum wiring.
         expected = defs.Standard(
             name="OWASP Cheat Sheets",
-            hyperlink="https://github.com/foo/bar/tree/master/cs.md",
+            hyperlink="https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html",
             section="Secrets Management Cheat Sheet",
-            links=[defs.Link(document=cre, ltype=defs.LinkTypes.LinkedTo)],
+            links=[defs.Link(document=cre, ltype=defs.LinkTypes.AutomaticallyLinkedTo)],
             tags=[
                 "family:guidance",
                 "subtype:cheatsheet",
-                "source:owasp_cheatsheets",
                 "audience:developer",
                 "maturity:stable",
+                "source:owasp_cheatsheets",
             ],
         )
         self.maxDiff = None
-        for name, nodes in entries.results.items():
-            self.assertEqual(name, parser.name)
-            self.assertEqual(len(nodes), 1)
-            self.assertCountEqual(expected.todict(), nodes[0].todict())
+        # Ensure the parser name exists in results before inspecting
+        self.assertIn(parser.name, entries.results)
+        nodes = entries.results[parser.name]
+        sections = {node.section for node in nodes}
+        self.assertIn("Secrets Management Cheat Sheet", sections)
+        secret_entry = next(
+            (
+                node
+                for node in nodes
+                if node.section == "Secrets Management Cheat Sheet"
+            ),
+            None,
+        )
+        self.assertIsNotNone(secret_entry)
+        self.assertEqual(expected.todict(), secret_entry.todict())
+
+    def test_register_supplemental_cheatsheets(self) -> None:
+        for cre_id, name in [
+            ("118-110", "API/web services"),
+            ("724-770", "Technical application access control"),
+            ("623-550", "Denial Of Service protection"),
+        ]:
+            self.collection.add_cre(defs.CRE(name=name, id=cre_id))
+
+        entries = cheatsheets_parser.Cheatsheets().register_supplemental_cheatsheets(
+            cache=self.collection
+        )
+        rest = next(
+            entry for entry in entries if entry.section == "REST Security Cheat Sheet"
+        )
+        for link in rest.links:
+            self.assertEqual(link.ltype, defs.LinkTypes.AutomaticallyLinkedTo)
+        self.assertEqual(
+            "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html",
+            rest.hyperlink,
+        )
+        self.assertEqual(
+            ["118-110", "724-770", "623-550"],
+            [link.document.id for link in rest.links],
+        )
+
+    @patch.object(git, "clone")
+    def test_parse_returns_supplemental_entries_when_clone_fails(
+        self, mock_clone
+    ) -> None:
+        for cre_id, name in [
+            ("118-110", "API/web services"),
+            ("724-770", "Technical application access control"),
+            ("623-550", "Denial Of Service protection"),
+        ]:
+            self.collection.add_cre(defs.CRE(name=name, id=cre_id))
+
+        mock_clone.side_effect = subprocess.CalledProcessError(
+            returncode=1,
+            cmd=["git", "clone"],
+        )
+
+        entries = cheatsheets_parser.Cheatsheets().parse(
+            cache=self.collection, ph=PromptHandler(database=self.collection)
+        )
+        mock_clone.assert_called_once()
+
+        rest = next(
+            node
+            for node in entries.results["OWASP Cheat Sheets"]
+            if node.section == "REST Security Cheat Sheet"
+        )
+        self.assertEqual(
+            "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html",
+            rest.hyperlink,
+        )
+        self.assertEqual(
+            ["118-110", "724-770", "623-550"],
+            [link.document.id for link in rest.links],
+        )
+
+    @patch.object(git, "clone")
+    def test_parse_merges_repo_and_supplemental_duplicate_entries(
+        self, mock_clone
+    ) -> None:
+        cs = self.rest_cheatsheet_md
+
+        class Repo:
+            working_dir = ""
+
+        repo = Repo()
+        temp_dir = tempfile.TemporaryDirectory()
+        self.addCleanup(temp_dir.cleanup)
+        loc = temp_dir.name
+        os.mkdir(os.path.join(loc, "cheatsheets"))
+        repo.working_dir = loc
+        for cre_id, name in [
+            ("223-780", "REST security repo link"),
+            ("118-110", "API/web services"),
+            ("724-770", "Technical application access control"),
+            ("623-550", "Denial Of Service protection"),
+        ]:
+            self.collection.add_cre(defs.CRE(name=name, id=cre_id))
+
+        with open(
+            os.path.join(
+                os.path.join(loc, "cheatsheets"),
+                "REST_Security_Cheat_Sheet.md",
+            ),
+            "w",
+        ) as mdf:
+            mdf.write(cs)
+        mock_clone.return_value = repo
+
+        entries = cheatsheets_parser.Cheatsheets().parse(
+            cache=self.collection, ph=PromptHandler(database=self.collection)
+        )
+
+        rest_entries = [
+            node
+            for node in entries.results["OWASP Cheat Sheets"]
+            if node.section == "REST Security Cheat Sheet"
+            and node.hyperlink
+            == "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html"
+        ]
+        self.assertEqual(1, len(rest_entries))
+        self.assertCountEqual(
+            ["223-780", "118-110", "724-770", "623-550"],
+            [link.document.id for link in rest_entries[0].links],
+        )
+        self.assertTrue(
+            all(
+                link.ltype == defs.LinkTypes.AutomaticallyLinkedTo
+                for link in rest_entries[0].links
+            )
+        )
 
     cheatsheets_md = """ # Secrets Management Cheat Sheet
 
@@ -119,4 +253,11 @@ class Repo:
 - [NIST SP 800-57 Recommendation for Key Management](https://csrc.nist.gov/publications/detail/sp/800-57-part-1/rev-5/final)
 
 
+"""
+
+    rest_cheatsheet_md = """# REST Security Cheat Sheet
+
+## Authentication
+
+For access-control recommendations see [OpenCRE REST reference](https://www.opencre.org/cre/223-780).
 """

application/utils/external_project_parsers/parsers/cheatsheets_parser.py

@@ -1,11 +1,15 @@
 # script to parse cheatsheet md files find the links to opencre.org and add the cheatsheets to CRE
 from typing import List
+import subprocess
 from application.database import db
 from application.utils import git
 from application.defs import cre_defs as defs
 import os
 import re
 from application.utils.external_project_parsers import base_parser_defs
+import json
+from pathlib import Path
+import logging
 from application.utils.external_project_parsers.base_parser_defs import (
     ParserInterface,
     ParseResult,
@@ -15,6 +19,15 @@
 
 class Cheatsheets(ParserInterface):
     name = "OWASP Cheat Sheets"
+    cheatsheetseries_base_url = "https://cheatsheetseries.owasp.org/cheatsheets"
+    supplement_data_file = (
+        Path(__file__).resolve().parents[3]
+        / "tests"
+        / "fixtures"
+        / "owasp_mappings"
+        / "owasp_cheatsheets_supplement.json"
+    )
+    logger = logging.getLogger(__name__)
 
     def cheatsheet(
         self, section: str, hyperlink: str, tags: List[str]
@@ -33,13 +46,31 @@ def cheatsheet(
             hyperlink=hyperlink,
         )
 
+    def official_cheatsheet_url(self, markdown_filename: str) -> str:
+        html_name = os.path.splitext(markdown_filename)[0] + ".html"
+        return f"{self.cheatsheetseries_base_url}/{html_name}"
+
     def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler):
         c_repo = "https://github.com/OWASP/CheatSheetSeries.git"
         cheatsheets_path = "cheatsheets/"
-        repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True)
-        cheatsheets = self.register_cheatsheets(
-            repo=repo, cache=cache, cheatsheets_path=cheatsheets_path, repo_path=c_repo
-        )
+        cheatsheets = []
+        repo = None
+        try:
+            repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True)
+        except (subprocess.SubprocessError, OSError) as exc:
+            self.logger.warning(
+                "Unable to clone OWASP CheatSheetSeries, continuing with supplemental cheat sheets only: %s",
+                exc,
+            )
+        if repo:
+            cheatsheets = self.register_cheatsheets(
+                repo=repo,
+                cache=cache,
+                cheatsheets_path=cheatsheets_path,
+                repo_path=c_repo,
+            )
+        cheatsheets.extend(self.register_supplemental_cheatsheets(cache=cache))
+        cheatsheets = self.deduplicate_entries(cheatsheets)
         results = {self.name: cheatsheets}
         base_parser_defs.validate_classification_tags(results)
         return ParseResult(results=results)
@@ -65,7 +96,7 @@ def register_cheatsheets(
                     name = title.group("title")
                     cre_id = cre.group("cre")
                     cres = cache.get_CREs(external_id=cre_id)
-                    hyperlink = f"{repo_path.replace('.git','')}/tree/master/{cheatsheets_path}{mdfile}"
+                    hyperlink = self.official_cheatsheet_url(mdfile)
                     cs = self.cheatsheet(section=name, hyperlink=hyperlink, tags=[])
                     for cre in cres:
                         cs.add_link(
@@ -75,3 +106,78 @@ def register_cheatsheets(
                         )
                     standard_entries.append(cs)
         return standard_entries
+
+    def register_supplemental_cheatsheets(self, cache: db.Node_collection):
+        # Check if file exists
+        if not self.supplement_data_file.exists():
+            self.logger.warning(
+                "Supplemental cheatsheet file not found at %s – skipping.",
+                self.supplement_data_file,
+            )
+            return []
+
+        # Try to load and parse JSON
+        try:
+            with self.supplement_data_file.open("r", encoding="utf-8") as handle:
+                supplement_entries = json.load(handle)
+        except (json.JSONDecodeError, OSError) as exc:
+            self.logger.error(
+                "Failed to load supplemental cheatsheet file %s: %s – skipping.",
+                self.supplement_data_file,
+                exc,
+            )
+            return []
+
+        standard_entries = []
+        for entry in supplement_entries:
+            # Validate required keys
+            if not all(k in entry for k in ("section", "hyperlink")):
+                self.logger.warning(
+                    "Skipping malformed supplemental entry (missing 'section' or 'hyperlink'): %s",
+                    entry,
+                )
+                continue
+
+            cs = self.cheatsheet(
+                section=entry["section"],
+                hyperlink=entry["hyperlink"],
+                tags=[],
+            )
+            add_link_failures = False
+            for cre_id in entry.get("cre_ids", []):
+                cres = cache.get_CREs(external_id=cre_id)
+                for cre in cres:
+                    link = defs.Link(
+                        document=cre.shallow_copy(),
+                        ltype=defs.LinkTypes.AutomaticallyLinkedTo,
+                    )
+                    try:
+                        cs.add_link(link)
+                    except ValueError as exc:
+                        self.logger.warning(
+                            "Failed to add link for cre_id %s to cheatsheet %s: %s",
+                            cre_id,
+                            entry.get("section", "<unknown>"),
+                            exc,
+                        )
+                        add_link_failures = True
+            if cs.links and not add_link_failures:
+                standard_entries.append(cs)
+        return standard_entries
+
+    def deduplicate_entries(self, entries: List[defs.Standard]) -> List[defs.Standard]:
+        deduped = {}
+        for entry in entries:
+            key = (entry.section, entry.hyperlink)
+            if key in deduped:
+                # Merge duplicates: union links into existing entry
+                existing_entry = deduped[key]
+                existing_link_ids = {link.document.id for link in existing_entry.links}
+                for link in entry.links:
+                    if link.document.id not in existing_link_ids:
+                        existing_entry.add_link(link)
+                        existing_link_ids.add(link.document.id)
+            else:
+                # First occurrence: store the entry
+                deduped[key] = entry
+        return list(deduped.values())