diff --git a/sjsonnet/src/sjsonnet/ValVisitor.scala b/sjsonnet/src/sjsonnet/ValVisitor.scala index 33eda06c..0ab5487b 100644 --- a/sjsonnet/src/sjsonnet/ValVisitor.scala +++ b/sjsonnet/src/sjsonnet/ValVisitor.scala @@ -27,7 +27,7 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self => var key: String = _ def subVisitor: Visitor[?, ?] = self def visitKey(index: Int): upickle.core.StringVisitor.type = upickle.core.StringVisitor - def visitKeyValue(s: Any): Unit = key = s.toString + def visitKeyValue(s: Any): Unit = key = ValVisitor.replaceLoneSurrogates(s.toString) def visitValue(v: Val, index: Int): Unit = { cache.put(key, v) allKeys.put(key, false) @@ -52,5 +52,48 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self => } ) - def visitString(s: CharSequence, index: Int): Val = Val.Str(pos, s.toString) + def visitString(s: CharSequence, index: Int): Val = + Val.Str(pos, ValVisitor.replaceLoneSurrogates(s.toString)) +} + +object ValVisitor { + + /** + * Replace unpaired UTF-16 surrogates with U+FFFD. JSON inputs may contain lone + * surrogate escapes (RFC 8259 lets implementations accept them); keeping them + * would corrupt to '?' on UTF-8 output. Matches the replacement policy of + * std.char / %c and go-jsonnet's JSON decoding. Strings without surrogates are + * returned as-is without allocation. + */ + private[sjsonnet] def replaceLoneSurrogates(s: String): String = { + val len = s.length + var i = 0 + while (i < len) { + val c = s.charAt(i) + if (c >= 0xd800 && c <= 0xdfff) { + val sb = new java.lang.StringBuilder(len) + sb.append(s, 0, i) + while (i < len) { + val ch = s.charAt(i) + if ( + Character.isHighSurrogate(ch) && i + 1 < len && + Character.isLowSurrogate(s.charAt(i + 1)) + ) { + sb.append(ch) + sb.append(s.charAt(i + 1)) + i += 2 + } else if (ch >= 0xd800 && ch <= 0xdfff) { + sb.append('\ufffd') + i += 1 + } else { + sb.append(ch) + i += 1 + } + } + return sb.toString + } + i += 1 + } + s + } } diff --git a/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet new file mode 100644 index 00000000..d3d75782 --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet @@ -0,0 +1,10 @@ +// Lone surrogates in JSON strings are replaced with U+FFFD, matching +// go-jsonnet and sjsonnet's own std.char / %c surrogate policy. Keeping +// them would corrupt to '?' on UTF-8 output. +std.assertEqual(std.parseJson('"\\ud800"'), "\ufffd") && +std.assertEqual(std.parseJson('"\\ud800\\ud800"'), "\ufffd\ufffd") && +std.assertEqual(std.parseJson('"a\\udc00b"'), "a\ufffdb") && +// a valid surrogate pair is preserved +std.assertEqual(std.parseJson('"\\ud83d\\ude00"'), "\ud83d\ude00") && +// object keys are sanitized too +std.assertEqual(std.parseJson('{"\\ud800": 1}')['\ufffd'], 1) diff --git a/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet.golden b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet.golden new file mode 100644 index 00000000..27ba77dd --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/parseJson_lone_surrogate.jsonnet.golden @@ -0,0 +1 @@ +true