From dd271c2595ff6db847ddaa99f3e6b97c3f6ffe99 Mon Sep 17 00:00:00 2001
From: He-Pin <hepin1989@gmail.com>
Date: Thu, 6 Aug 2026 11:33:41 +0800
Subject: [PATCH] fix: std.parseJson replaces lone surrogates with U+FFFD
 instead of corrupting to '?'

Motivation:
std.parseJson kept unpaired UTF-16 surrogates in the parsed string
(like CPython/JS), but every UTF-8 output path then replaced the
unencodable unit with '?': std.parseJson('"\\ud800"') rendered as "?"
while go-jsonnet renders the Unicode replacement character U+FFFD.
No implementation emits '?', and sjsonnet already follows the U+FFFD
policy elsewhere (std.char and %c replace surrogate codepoints with
U+FFFD; UnicodeHandlingTests documents the alignment), so parseJson
contradicted the project's own convention and silently lost data.

Modification:
Sanitize strings in ValVisitor.visitString and object keys in
visitKeyValue via a new ValVisitor.replaceLoneSurrogates helper:
a fast-path scan that returns the input unchanged without allocation
when it contains no surrogates, and otherwise replaces each unpaired
high/low surrogate with U+FFFD while preserving valid pairs.

Result:
std.parseJson('"\\ud800"') == "\ufffd", byte-identical output with
go-jsonnet (22 ef bf bd 22). Valid surrogate pairs pass through
unchanged. The JSON import path (ujson ByteParser) is unaffected: it
already drops or rejects lone surrogates before any visitor runs
(pre-existing behavior, unchanged by this fix).

References:
Found by four-way differential testing (sjsonnet vs go-jsonnet vs
jrsonnet vs spec).
---
 sjsonnet/src/sjsonnet/ValVisitor.scala        | 47 ++++++++++++++++++-
 .../parseJson_lone_surrogate.jsonnet          | 10 ++++
 .../parseJson_lone_surrogate.jsonnet.golden   |  1 +
 3 files changed, 56 insertions(+), 2 deletions(-)
 create mode 100644 sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet
 create mode 100644 sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet.golden

diff --git a/sjsonnet/src/sjsonnet/ValVisitor.scala b/sjsonnet/src/sjsonnet/ValVisitor.scala
index 33eda06c6..0ab5487b6 100644
--- a/sjsonnet/src/sjsonnet/ValVisitor.scala
+++ b/sjsonnet/src/sjsonnet/ValVisitor.scala
@@ -27,7 +27,7 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
     var key: String = _
     def subVisitor: Visitor[?, ?] = self
     def visitKey(index: Int): upickle.core.StringVisitor.type = upickle.core.StringVisitor
-    def visitKeyValue(s: Any): Unit = key = s.toString
+    def visitKeyValue(s: Any): Unit = key = ValVisitor.replaceLoneSurrogates(s.toString)
     def visitValue(v: Val, index: Int): Unit = {
       cache.put(key, v)
       allKeys.put(key, false)
@@ -52,5 +52,48 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
       }
     )
 
-  def visitString(s: CharSequence, index: Int): Val = Val.Str(pos, s.toString)
+  def visitString(s: CharSequence, index: Int): Val =
+    Val.Str(pos, ValVisitor.replaceLoneSurrogates(s.toString))
+}
+
+object ValVisitor {
+
+  /**
+   * Replace unpaired UTF-16 surrogates with U+FFFD. JSON inputs may contain lone
+   * surrogate escapes (RFC 8259 lets implementations accept them); keeping them
+   * would corrupt to '?' on UTF-8 output. Matches the replacement policy of
+   * std.char / %c and go-jsonnet's JSON decoding. Strings without surrogates are
+   * returned as-is without allocation.
+   */
+  private[sjsonnet] def replaceLoneSurrogates(s: String): String = {
+    val len = s.length
+    var i = 0
+    while (i < len) {
+      val c = s.charAt(i)
+      if (c >= 0xd800 && c <= 0xdfff) {
+        val sb = new java.lang.StringBuilder(len)
+        sb.append(s, 0, i)
+        while (i < len) {
+          val ch = s.charAt(i)
+          if (
+            Character.isHighSurrogate(ch) && i + 1 < len &&
+            Character.isLowSurrogate(s.charAt(i + 1))
+          ) {
+            sb.append(ch)
+            sb.append(s.charAt(i + 1))
+            i += 2
+          } else if (ch >= 0xd800 && ch <= 0xdfff) {
+            sb.append('\ufffd')
+            i += 1
+          } else {
+            sb.append(ch)
+            i += 1
+          }
+        }
+        return sb.toString
+      }
+      i += 1
+    }
+    s
+  }
 }
diff --git a/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet
new file mode 100644
index 000000000..d3d757823
--- /dev/null
+++ b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet
@@ -0,0 +1,10 @@
+// Lone surrogates in JSON strings are replaced with U+FFFD, matching
+// go-jsonnet and sjsonnet's own std.char / %c surrogate policy. Keeping
+// them would corrupt to '?' on UTF-8 output.
+std.assertEqual(std.parseJson('"\\ud800"'), "\ufffd") &&
+std.assertEqual(std.parseJson('"\\ud800\\ud800"'), "\ufffd\ufffd") &&
+std.assertEqual(std.parseJson('"a\\udc00b"'), "a\ufffdb") &&
+// a valid surrogate pair is preserved
+std.assertEqual(std.parseJson('"\\ud83d\\ude00"'), "\ud83d\ude00") &&
+// object keys are sanitized too
+std.assertEqual(std.parseJson('{"\\ud800": 1}')['\ufffd'], 1)
diff --git a/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet.golden b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet.golden
new file mode 100644
index 000000000..27ba77dda
--- /dev/null
+++ b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet.golden
@@ -0,0 +1 @@
+true