fix(jcs): reject lone surrogates in canonicalization (RFC 8785)

src/agent_passport/canonical.py

@@ -13,6 +13,45 @@
 import re
 
 
+class JCSCanonicalizationError(ValueError):
+    """A value cannot be canonicalized under RFC 8785.
+
+    Subclasses ValueError so existing fail-closed handlers that catch ValueError
+    around canonicalization keep working (they fail closed rather than crash).
+
+    Carries a stable machine-readable ``category`` and ``reason`` matching the
+    TypeScript and Go SDKs, so cross-language callers can branch on the failure
+    without parsing the message.
+    """
+
+    #: Stable machine-readable category, shared across the APS SDKs.
+    category = "invalid_unicode"
+
+    def __init__(self, message: str, reason: str = "lone_surrogate") -> None:
+        super().__init__(message)
+        #: Specific failure within the category, e.g. "lone_surrogate".
+        self.reason = reason
+
+
+def _assert_no_lone_surrogate(s: str) -> None:
+    """Reject a string containing an unpaired UTF-16 surrogate.
+
+    A lone surrogate (U+D800..U+DFFF) is not a valid Unicode scalar and has no
+    UTF-8 encoding, so RFC 8785 requires rejecting the input rather than escaping
+    it or replacing it with U+FFFD. Python's JSON parser resolves valid surrogate
+    PAIRS to their single non-BMP code point, so any character that remains in the
+    surrogate range is unpaired.
+    """
+    for ch in s:
+        o = ord(ch)
+        if 0xD800 <= o <= 0xDFFF:
+            raise JCSCanonicalizationError(
+                "canonicalize_jcs: string contains an unpaired UTF-16 surrogate "
+                f"(U+{o:04X}); a lone surrogate has no valid UTF-8 encoding and "
+                "RFC 8785 requires rejection"
+            )
+
+
 def has_non_finite(obj) -> bool:
     """Return True if obj contains a NaN or Infinity float anywhere within it.
 
@@ -149,11 +188,17 @@ def canonicalize_jcs(obj) -> str:
             raise ValueError(f"Cannot canonicalize {obj}")
         return _es_number(obj)
     if isinstance(obj, str):
+        _assert_no_lone_surrogate(obj)
         return json.dumps(obj, ensure_ascii=False)
     if isinstance(obj, list):
         return "[" + ",".join(canonicalize_jcs(item) for item in obj) + "]"
     if isinstance(obj, dict):
         # Preserve null values in objects (unlike legacy canonicalize)
+        # Validate keys before sorting: _canonical_keys encodes each key as
+        # utf-16-be, which itself raises on a lone surrogate. Check first so the
+        # failure is the clean typed error, not a raw UnicodeEncodeError.
+        for key in obj.keys():
+            _assert_no_lone_surrogate(key)
         pairs = []
         for key in _canonical_keys(obj.keys()):
             pairs.append(

tests/test_jcs_lone_surrogate.py

@@ -0,0 +1,145 @@
+# Copyright 2026 Tymofii Pidlisnyi. Apache-2.0 license. See LICENSE.
+"""RFC 8785: canonicalize_jcs must reject lone/unpaired UTF-16 surrogates.
+
+A lone surrogate is not a valid Unicode scalar and has no UTF-8 encoding, so the
+input is invalid and must be rejected, not escaped or replaced. A valid surrogate
+pair (a non-BMP character) must still canonicalize to its raw UTF-8 bytes.
+"""
+
+import pytest
+
+from agent_passport.canonical import JCSCanonicalizationError, canonicalize_jcs
+
+HIGH = chr(0xD800)  # lone high surrogate
+LOW = chr(0xDFFF)  # lone low surrogate
+EMOJI = chr(0x1F600)  # valid non-BMP scalar (U+1F600)
+
+
+def test_reject_lone_high_surrogate():
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({"v": HIGH})
+
+
+def test_reject_lone_low_surrogate():
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({"v": LOW})
+
+
+def test_reject_lone_surrogate_after_valid_pair():
+    # A valid non-BMP scalar followed by a lone surrogate: detection must not be
+    # fooled by the earlier valid character.
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({"v": EMOJI + HIGH})
+
+
+def test_reject_lone_surrogate_in_key():
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({HIGH: "x"})
+
+
+def test_reject_bare_lone_surrogate_string():
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs(HIGH)
+
+
+def test_valid_non_bmp_unchanged():
+    # A valid non-BMP character canonicalizes to raw UTF-8, unchanged by this fix.
+    assert canonicalize_jcs(EMOJI) == '"' + EMOJI + '"'
+    assert canonicalize_jcs({"v": EMOJI}) == '{"v":"' + EMOJI + '"}'
+    # The canonical bytes are the raw 4-byte UTF-8 of U+1F600.
+    assert canonicalize_jcs({"v": EMOJI}).encode("utf-8") == b'{"v":"\xf0\x9f\x98\x80"}'
+
+
+def test_error_is_valueerror_subclass():
+    # Existing fail-closed handlers that catch ValueError keep working.
+    assert issubclass(JCSCanonicalizationError, ValueError)
+
+
+# --- property-name and structural coverage ---
+
+
+def test_reject_lone_surrogate_nested_value():
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({"a": {"b": HIGH}})
+
+
+def test_reject_lone_surrogate_array_element():
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({"a": [HIGH]})
+
+
+def test_reject_lone_surrogate_nested_key():
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({"a": {LOW: "x"}})
+
+
+def test_reject_valid_pair_followed_by_lone_low():
+    # Off-by-one guard: a valid non-BMP scalar then a lone low surrogate rejects.
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs({"v": EMOJI + LOW})
+
+
+# --- error contract ---
+
+
+def test_error_contract_category_reason_and_no_leak():
+    # Caught by the documented type (ValueError), carries the stable category and
+    # reason, and does not leak the offending string into the message.
+    with pytest.raises(ValueError) as exc:
+        canonicalize_jcs({"v": HIGH})
+    err = exc.value
+    assert isinstance(err, JCSCanonicalizationError)
+    assert err.category == "invalid_unicode"
+    assert err.reason == "lone_surrogate"
+    assert HIGH not in str(err)
+
+
+# --- signing boundary ---
+
+
+def test_sign_preimage_api_fails_closed_on_lone_surrogate():
+    # A real content-id API (compute_action_ref) routes through canonicalize_jcs;
+    # a lone surrogate in the signed preimage must fail closed with the typed
+    # error and produce no output, with no fallback to the legacy canonicalizer.
+    from agent_passport.action_ref import compute_action_ref
+
+    with pytest.raises(JCSCanonicalizationError):
+        compute_action_ref("agent-1", "read", [HIGH], "2026-07-13T00:00:00Z")
+
+
+# --- adversarial raw payloads reach the same terminal state as Go ---
+# The same raw JSON text pinned in the Go scanner tests. Python preserves a lone
+# surrogate through json.loads and rejects it at canonicalize_jcs, so it reaches
+# the same accept/reject terminal state as the Go raw-JSON path.
+
+import json  # noqa: E402
+
+_REJECT = {
+    "space-separated-non-adjacent": r'{"v":"\uD800 \uDC00"}',
+    "newline-separated-non-adjacent": r'{"v":"\uD800\n\uDC00"}',
+    "lone-low-first": r'{"v":"\uDC00"}',
+    "low-then-high": r'{"v":"\uDC00\uD800"}',
+    "high-then-literal-low": r'{"v":"\uD800\\uDC00"}',
+    "lone-in-key": r'{"\uD800":"x"}',
+    "lowercase-hex": r'{"v":"\ud800"}',
+    "literal-backslash-then-lone": r'{"v":"\\\uD800"}',
+}
+
+_ACCEPT = {
+    "valid-adjacent-pair": r'{"v":"😀"}',
+    "escaped-backslash-literal": r'{"v":"\\uD800"}',
+    "double-backslash-literal": r'{"v":"\\\\uD800"}',
+    "genuine-replacement-char": r'{"v":"�"}',
+}
+
+
+@pytest.mark.parametrize("raw", list(_REJECT.values()), ids=list(_REJECT.keys()))
+def test_adversarial_raw_reject(raw):
+    with pytest.raises(JCSCanonicalizationError):
+        canonicalize_jcs(json.loads(raw))
+
+
+@pytest.mark.parametrize("raw", list(_ACCEPT.values()), ids=list(_ACCEPT.keys()))
+def test_adversarial_raw_accept(raw):
+    # Must not raise.
+    canonicalize_jcs(json.loads(raw))