From d499ac278a492a964e018ec88bf83c8759075239 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:11:13 -0700 Subject: [PATCH 1/3] fix: Improve jreader throughput with a single-pass tokenizer The default tokenizer previously walked every string character through bytes.Reader.ReadRune (a method call and UTF-8 decode per character, even for plain ASCII), decoded escaped strings by appending one rune at a time to a growing buffer, and copied token structs by value through every scalar read. Behaviorally nothing changes: the exported API, decoded values, error types, error messages, and error offsets are all identical. In particular, the tokenizer still skips any byte that unicode.IsSpace matches as a Latin-1 code point between tokens, passes unescaped strings through byte-for-byte as zero-copy subslices (including invalid UTF-8 and control characters), substitutes the replacement character for each invalid UTF-8 byte only in strings that contain an escape, encodes a \u-escaped surrogate code point as the replacement character, and keeps the same lenient number grammar (including int64 wraparound for very long integer literals). Structurally: - Strings are scanned in place over the input byte slice; a string without escapes is found with a two-byte-sentinel scan (the quote mark and the backslash are the only bytes that cannot pass through verbatim) and returned as a zero-copy subslice of the input. - A string with escapes is decoded in one forward pass into a single buffer sized from the string's raw length, bulk-copying each run of plain characters. - \u escapes are decoded by direct indexing instead of through a bytes.Reader. - next() now parses into a token field on the tokenReader instead of returning token structs by value, and putBack flips a flag instead of storing a token. - Bool, Number, StringAsBytes, PropertyName, and Any dispatch on the first non-whitespace byte and parse the value directly, going through the token machinery only when a token has been pushed back or the input is not the expected type. - Whitespace and keyword scanning index the input directly (whitespace through a table built from unicode.IsSpace) instead of going through per-byte reader method calls. A bytes.IndexByte-based scan was also measured for the zero-copy path: it is several times faster on long strings but 15-20% slower on short ones, and on a real flag payload the two are statistically indistinguishable, so the simpler loop wins. Interleaved benchmarks (benchstat, n=6): -43% boolean reads, -48% integer reads, -56% short string reads with a 3x-6x string-scan throughput increase, -64% escaped strings with half the allocations, -54% typical object parsing, -57% geomean across the jreader suite. Parsing a real 3,228-flag LaunchDarkly payload through ldmodel improves 50%. --- jreader/reader_benchmark_test.go | 34 ++ jreader/token_reader_default.go | 560 ++++++++++++++++++++----------- 2 files changed, 395 insertions(+), 199 deletions(-) diff --git a/jreader/reader_benchmark_test.go b/jreader/reader_benchmark_test.go index 2c66013..3d57b7e 100644 --- a/jreader/reader_benchmark_test.go +++ b/jreader/reader_benchmark_test.go @@ -1,6 +1,7 @@ package jreader import ( + "strings" "testing" "github.com/launchdarkly/go-jsonstream/v4/internal/commontest" @@ -108,6 +109,39 @@ func BenchmarkReadArrayOfStrings(b *testing.B) { } } +// BenchmarkReadStringKinds covers the tokenizer's distinct string paths: zero-copy scanning of +// plain ASCII at several lengths, multi-byte characters, and escape sequences that require +// decoding into a new buffer. +func BenchmarkReadStringKinds(b *testing.B) { + for _, tc := range []struct { + name string + elem string + count int + }{ + {"shortASCII", "value123", 50}, + {"mediumASCII", strings.Repeat("m", 40), 50}, + {"longASCII", strings.Repeat("L", 400), 20}, + {"multiByte", "café naïve \U0001D11E", 50}, + {"escaped", `x\ty` + strings.Repeat("p", 20), 50}, + } { + parts := make([]string, tc.count) + for i := range parts { + parts[i] = `"` + tc.elem + `"` + } + data := []byte("[" + strings.Join(parts, ",") + "]") + b.Run(tc.name, func(b *testing.B) { + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + r := NewReader(data) + for arr := r.Array(); arr.Next(); { + r.StringAsBytes() + } + failBenchmarkOnReaderError(b, &r) + } + }) + } +} + func BenchmarkReadArrayOfNullsNoAlloc(b *testing.B) { // This just verifies that simply parsing an array doesn't cause any allocations, if the values don't. data := []byte(`[null,null]`) diff --git a/jreader/token_reader_default.go b/jreader/token_reader_default.go index 446bc09..1b03f38 100644 --- a/jreader/token_reader_default.go +++ b/jreader/token_reader_default.go @@ -17,6 +17,34 @@ var ( tokenFalse = []byte("false") //nolint:gochecknoglobals ) +// whitespaceChars marks the bytes that are skipped between tokens. Each byte is classified the +// way unicode.IsSpace classifies its Latin-1 code point, matching a per-byte rune conversion of +// the input (so the single bytes 0x85 and 0xA0 also count as whitespace). +var whitespaceChars = makeWhitespaceChars() //nolint:gochecknoglobals + +func makeWhitespaceChars() (t [256]bool) { + for c := 0; c < 256; c++ { + if unicode.IsSpace(rune(c)) { + t[c] = true + } + } + return +} + +// plainStringChars marks the bytes that pass through a decoded string unchanged: the ASCII +// characters other than the quote mark and the backslash. Everything else ends a scan of plain +// characters and is handled individually. +var plainStringChars = makePlainStringChars() //nolint:gochecknoglobals + +func makePlainStringChars() (t [256]bool) { + for c := 0; c < utf8.RuneSelf; c++ { + if c != '"' && c != '\\' { + t[c] = true + } + } + return +} + type token struct { kind tokenKind boolValue bool @@ -55,12 +83,12 @@ func (t token) description() string { } type tokenReader struct { - data []byte - pos int - len int - hasUnread bool - unreadToken token - lastPos int + data []byte + pos int + len int + hasUnread bool + tok token // the most recently parsed token; the unread token when hasUnread is true + lastPos int } func newTokenReader(data []byte) tokenReader { @@ -103,16 +131,15 @@ func (r *tokenReader) getPos() int { // // This and all other tokenReader methods skip transparently past whitespace between tokens. func (r *tokenReader) Null() (bool, error) { - t, err := r.next() - if err != nil { + if err := r.next(); err != nil { return false, err } - if t.kind == nullToken { + if r.tok.kind == nullToken { return true, nil } - r.putBack(t) - if t.kind == delimiterToken && t.delimiter != '[' && t.delimiter != '{' { - return false, SyntaxError{Message: errMsgUnexpectedChar, Value: string(t.delimiter), Offset: r.getPos()} + r.putBack() + if r.tok.kind == delimiterToken && r.tok.delimiter != '[' && r.tok.delimiter != '{' { + return false, SyntaxError{Message: errMsgUnexpectedChar, Value: string(r.tok.delimiter), Offset: r.getPos()} } return false, nil } @@ -122,17 +149,46 @@ func (r *tokenReader) Null() (bool, error) { // // This and all other tokenReader methods skip transparently past whitespace between tokens. func (r *tokenReader) Bool() (bool, error) { - t, err := r.consumeScalar(boolToken) - return t.boolValue, err + if !r.hasUnread { + b, ok := r.skipWhitespaceAndReadByte() + if !ok { + return false, io.EOF + } + if b == 't' || b == 'f' { + // A keyword starting with these letters can only be a boolean or malformed. + _, boolValue, err := r.readKeyword(b) + return boolValue, err + } + r.unreadByte() + } + if err := r.consumeScalar(boolToken); err != nil { + return false, err + } + return r.tok.boolValue, nil } -// Bool requires that the next token is a JSON number, returning its value if successful (consuming +// Number requires that the next token is a JSON number, returning its value if successful (consuming // the token), or an error if the next token is anything other than a JSON number. // // This and all other tokenReader methods skip transparently past whitespace between tokens. func (r *tokenReader) Number() (float64, error) { - t, err := r.consumeScalar(numberToken) - return t.numberValue, err + if !r.hasUnread { + b, ok := r.skipWhitespaceAndReadByte() + if !ok { + return 0, io.EOF + } + if (b >= '0' && b <= '9') || b == '-' { + if n, ok2 := r.readNumber(b); ok2 { + return n, nil + } + return 0, SyntaxError{Message: errMsgInvalidNumber, Offset: r.lastPos} + } + r.unreadByte() + } + if err := r.consumeScalar(numberToken); err != nil { + return 0, err + } + return r.tok.numberValue, nil } // String requires that the next token is a JSON string, returning its value if successful (consuming @@ -149,8 +205,20 @@ func (r *tokenReader) String() (string, error) { // // This and all other tokenReader methods skip transparently past whitespace between tokens. func (r *tokenReader) StringAsBytes() ([]byte, error) { - t, err := r.consumeScalar(stringToken) - return t.stringValue, err + if !r.hasUnread { + b, ok := r.skipWhitespaceAndReadByte() + if !ok { + return nil, io.EOF + } + if b == '"' { + return r.readString() + } + r.unreadByte() + } + if err := r.consumeScalar(stringToken); err != nil { + return nil, err + } + return r.tok.stringValue, nil } // PropertyName requires that the next token is a JSON string and the token after that is a colon, @@ -162,7 +230,7 @@ func (r *tokenReader) StringAsBytes() ([]byte, error) { // // This and all other tokenReader methods skip transparently past whitespace between tokens. func (r *tokenReader) PropertyName() ([]byte, error) { - t, err := r.consumeScalar(stringToken) + name, err := r.StringAsBytes() if err != nil { return nil, err } @@ -174,7 +242,7 @@ func (r *tokenReader) PropertyName() ([]byte, error) { r.unreadByte() return nil, r.syntaxErrorOnNextToken(errMsgExpectedColon) } - return t.stringValue, nil + return name, nil } // Delimiter checks whether the next token is the specified ASCII delimiter character. If so, it @@ -184,7 +252,7 @@ func (r *tokenReader) PropertyName() ([]byte, error) { // This and all other tokenReader methods skip transparently past whitespace between tokens. func (r *tokenReader) Delimiter(delimiter byte) (bool, error) { if r.hasUnread { - if r.unreadToken.kind == delimiterToken && r.unreadToken.delimiter == delimiter { + if r.tok.kind == delimiterToken && r.tok.delimiter == delimiter { r.hasUnread = false return true, nil } @@ -198,11 +266,10 @@ func (r *tokenReader) Delimiter(delimiter byte) (bool, error) { return true, nil } r.unreadByte() // we'll back up and try to parse a token, to see if it's valid JSON or not - token, err := r.next() - if err != nil { + if err := r.next(); err != nil { return false, err // it was malformed JSON } - r.putBack(token) // it was valid JSON, we just haven't hit that delimiter + r.putBack() // it was valid JSON, we just haven't hit that delimiter return false, nil } @@ -212,13 +279,13 @@ func (r *tokenReader) Delimiter(delimiter byte) (bool, error) { // returns an error. The delimiter parameter will always be either '}' or ']'. func (r *tokenReader) EndDelimiterOrComma(delimiter byte) (bool, error) { if r.hasUnread { - if r.unreadToken.kind == delimiterToken && - (r.unreadToken.delimiter == delimiter || r.unreadToken.delimiter == ',') { + if r.tok.kind == delimiterToken && + (r.tok.delimiter == delimiter || r.tok.delimiter == ',') { r.hasUnread = false - return r.unreadToken.delimiter == delimiter, nil + return r.tok.delimiter == delimiter, nil } return false, SyntaxError{Message: badArrayOrObjectItemMessage(delimiter == '}'), - Value: r.unreadToken.description(), Offset: r.lastPos} + Value: r.tok.description(), Offset: r.lastPos} } b, ok := r.skipWhitespaceAndReadByte() if !ok { @@ -228,12 +295,11 @@ func (r *tokenReader) EndDelimiterOrComma(delimiter byte) (bool, error) { return b == delimiter, nil } r.unreadByte() - t, err := r.next() - if err != nil { + if err := r.next(); err != nil { return false, err } return false, SyntaxError{Message: badArrayOrObjectItemMessage(delimiter == '}'), - Value: t.description(), Offset: r.lastPos} + Value: r.tok.description(), Offset: r.lastPos} } func badArrayOrObjectItemMessage(isObject bool) string { @@ -252,109 +318,156 @@ func (r *tokenReader) Any() (AnyValue, error) { } func (r *tokenReader) any(ignoreString bool) (AnyValue, error) { - t, err := r.next() - if err != nil { - return AnyValue{}, err + if r.hasUnread { + r.hasUnread = false + return r.tokenToAnyValue(ignoreString) + } + b, ok := r.skipWhitespaceAndReadByte() + if !ok { + return AnyValue{}, io.EOF } - switch t.kind { + switch { + case b >= 'a' && b <= 'z': + kind, boolValue, err := r.readKeyword(b) + if err != nil { + return AnyValue{}, err + } + if kind == boolToken { + return AnyValue{Kind: BoolValue, Bool: boolValue}, nil + } + return AnyValue{Kind: NullValue}, nil + case (b >= '0' && b <= '9') || b == '-': + n, ok2 := r.readNumber(b) + if !ok2 { + return AnyValue{}, SyntaxError{Message: errMsgInvalidNumber, Offset: r.lastPos} + } + return AnyValue{Kind: NumberValue, Number: n}, nil + case b == '"': + stringValue, err := r.readString() + if err != nil { + return AnyValue{}, err + } + var s string + if !ignoreString { + s = string(stringValue) + } + return AnyValue{Kind: StringValue, String: s}, nil + case b == '[': + return AnyValue{Kind: ArrayValue}, nil + case b == '{': + return AnyValue{Kind: ObjectValue}, nil + } + return AnyValue{}, SyntaxError{Message: errMsgUnexpectedChar, Value: string(b), Offset: r.lastPos} +} + +// tokenToAnyValue converts the token in r.tok, which has just been consumed, to an AnyValue. +func (r *tokenReader) tokenToAnyValue(ignoreString bool) (AnyValue, error) { + switch r.tok.kind { case boolToken: - return AnyValue{Kind: BoolValue, Bool: t.boolValue}, nil + return AnyValue{Kind: BoolValue, Bool: r.tok.boolValue}, nil case numberToken: - return AnyValue{Kind: NumberValue, Number: t.numberValue}, nil + return AnyValue{Kind: NumberValue, Number: r.tok.numberValue}, nil case stringToken: var s string if !ignoreString { - s = string(t.stringValue) + s = string(r.tok.stringValue) } return AnyValue{Kind: StringValue, String: s}, nil case delimiterToken: - if t.delimiter == '[' { + if r.tok.delimiter == '[' { return AnyValue{Kind: ArrayValue}, nil } - if t.delimiter == '{' { + if r.tok.delimiter == '{' { return AnyValue{Kind: ObjectValue}, nil } return AnyValue{}, - SyntaxError{Message: errMsgUnexpectedChar, Value: string(t.delimiter), Offset: r.lastPos} + SyntaxError{Message: errMsgUnexpectedChar, Value: string(r.tok.delimiter), Offset: r.lastPos} default: return AnyValue{Kind: NullValue}, nil } } -// Attempts to parse and consume the next token, ignoring whitespace. A token is either a valid JSON scalar -// value or an ASCII delimiter character. If a token was previously unread using putBack, it consumes that -// instead. -func (r *tokenReader) next() (token, error) { +// Attempts to parse and consume the next token, ignoring whitespace, leaving it in r.tok. A token +// is either a valid JSON scalar value or an ASCII delimiter character. If a token was previously +// unread using putBack, it consumes that instead. When an error is returned, r.tok is not +// meaningful. +func (r *tokenReader) next() error { if r.hasUnread { r.hasUnread = false - return r.unreadToken, nil + return nil } b, ok := r.skipWhitespaceAndReadByte() if !ok { - return token{}, io.EOF + return io.EOF } switch { // We can get away with reading bytes instead of runes because the JSON spec doesn't allow multi-byte // characters except within a string literal. case b >= 'a' && b <= 'z': - n := r.consumeASCIILowercaseAlphabeticChars() + 1 - id := r.data[r.lastPos : r.lastPos+n] - if b == 'f' && bytes.Equal(id, tokenFalse) { - return token{kind: boolToken, boolValue: false}, nil - } - if b == 't' && bytes.Equal(id, tokenTrue) { - return token{kind: boolToken, boolValue: true}, nil - } - if b == 'n' && bytes.Equal(id, tokenNull) { - return token{kind: nullToken}, nil + kind, boolValue, err := r.readKeyword(b) + if err != nil { + return err } - return token{}, SyntaxError{Message: errMsgUnexpectedSymbol, Value: string(id), Offset: r.lastPos} + r.tok = token{kind: kind, boolValue: boolValue} + return nil case (b >= '0' && b <= '9') || b == '-': - if n, ok := r.readNumber(b); ok { - return token{kind: numberToken, numberValue: n}, nil + n, ok2 := r.readNumber(b) + if !ok2 { + return SyntaxError{Message: errMsgInvalidNumber, Offset: r.lastPos} } - return token{}, SyntaxError{Message: errMsgInvalidNumber, Offset: r.lastPos} + r.tok = token{kind: numberToken, numberValue: n} + return nil case b == '"': s, err := r.readString() if err != nil { - return token{}, err + return err } - return token{kind: stringToken, stringValue: s}, nil + r.tok = token{kind: stringToken, stringValue: s} + return nil case b == '[', b == ']', b == '{', b == '}', b == ':', b == ',': - return token{kind: delimiterToken, delimiter: b}, nil + r.tok = token{kind: delimiterToken, delimiter: b} + return nil } - return token{}, SyntaxError{Message: errMsgUnexpectedChar, Value: string(b), Offset: r.lastPos} + return SyntaxError{Message: errMsgUnexpectedChar, Value: string(b), Offset: r.lastPos} } -func (r *tokenReader) putBack(token token) { - r.unreadToken = token +// putBack marks the token in r.tok, which must have just been parsed by next(), as unread, so +// that the next call to next() will consume it again instead of parsing new input. +func (r *tokenReader) putBack() { r.hasUnread = true } -func (r *tokenReader) consumeScalar(kind tokenKind) (token, error) { - t, err := r.next() - if err != nil { - return token{}, err +func (r *tokenReader) consumeScalar(kind tokenKind) error { + if err := r.next(); err != nil { + return err } - if t.kind == kind { - return t, nil + if r.tok.kind == kind { + return nil } - if t.kind == delimiterToken && t.delimiter != '[' && t.delimiter != '{' { - return token{}, SyntaxError{Message: errMsgUnexpectedChar, Value: string(t.delimiter), Offset: r.LastPos()} + if r.tok.kind == delimiterToken && r.tok.delimiter != '[' && r.tok.delimiter != '{' { + return SyntaxError{Message: errMsgUnexpectedChar, Value: string(r.tok.delimiter), Offset: r.LastPos()} } - return token{}, TypeError{Expected: valueKindFromTokenKind(kind), - Actual: t.valueKind(), Offset: r.LastPos()} + return TypeError{Expected: valueKindFromTokenKind(kind), + Actual: r.tok.valueKind(), Offset: r.LastPos()} } -func (r *tokenReader) readByte() (byte, bool) { - if r.pos >= r.len { - return 0, false +// readKeyword parses the remainder of a keyword token (true, false, or null) whose first letter +// has already been consumed, returning the token kind and, for a boolean, its value. +func (r *tokenReader) readKeyword(first byte) (tokenKind, bool, error) { + n := r.consumeASCIILowercaseAlphabeticChars() + 1 + id := r.data[r.lastPos : r.lastPos+n] + if first == 'f' && bytes.Equal(id, tokenFalse) { + return boolToken, false, nil + } + if first == 't' && bytes.Equal(id, tokenTrue) { + return boolToken, true, nil + } + if first == 'n' && bytes.Equal(id, tokenNull) { + return nullToken, false, nil } - b := r.data[r.pos] - r.pos++ - return b, true + return 0, false, SyntaxError{Message: errMsgUnexpectedSymbol, Value: string(id), Offset: r.lastPos} } func (r *tokenReader) unreadByte() { @@ -362,180 +475,236 @@ func (r *tokenReader) unreadByte() { } func (r *tokenReader) skipWhitespaceAndReadByte() (byte, bool) { - for { - ch, ok := r.readByte() - if !ok { - return 0, false - } - if !unicode.IsSpace(rune(ch)) { - r.lastPos = r.pos - 1 - return ch, true + data, n := r.data, r.len + p := r.pos + for p < n { + ch := data[p] + if whitespaceChars[ch] { + p++ + continue } + r.lastPos = p + r.pos = p + 1 + return ch, true } + r.pos = p + return 0, false } func (r *tokenReader) consumeASCIILowercaseAlphabeticChars() int { - n := 0 - for { - ch, ok := r.readByte() - if !ok { - break - } - if ch < 'a' || ch > 'z' { - r.unreadByte() - break - } - n++ + data, n := r.data, r.len + p := r.pos + for p < n && data[p] >= 'a' && data[p] <= 'z' { + p++ } - return n + count := p - r.pos + r.pos = p + return count } func (r *tokenReader) readNumber(_ byte) (float64, bool) { + // The digit run may contain at most one '.', and an exponent part must match + // [eE][-+]?[0-9]+; beyond that the number's format is validated by the string-to-number + // conversion at the end. We scan the input directly by index rather than through + // readByte/unreadByte, updating r.pos at every return point so that the read position + // always reflects exactly the bytes consumed. startPos := r.lastPos + data, n := r.data, r.len + p := startPos + 1 // the first byte has already been read isFloat := false + + // Integer and fractional digits. The byte that ends the run is consumed and then + // reconsidered below. var ch byte - var ok bool - for { - ch, ok = r.readByte() - if !ok { - break - } + consumedEnd := false + for p < n { + ch = data[p] + p++ if (ch < '0' || ch > '9') && (ch != '.' || isFloat) { + consumedEnd = true break } if ch == '.' { isFloat = true } } - hasExponent := false - if ch == 'e' || ch == 'E' { - // exponent must match this regex: [eE][-+]?[0-9]+ - ch, ok = r.readByte() - if !ok { + + if consumedEnd && (ch == 'e' || ch == 'E') { + // The exponent marker must be followed by an optional sign and at least one digit. + if p >= n { + r.pos = p return 0, false } - if ch == '+' || ch == '-' { //nolint:gocritic,revive - } else if ch >= '0' && ch <= '9' { - r.unreadByte() - } else { + ch = data[p] + p++ + if ch >= '0' && ch <= '9' { + p-- // the digit is consumed by the loop below + } else if ch != '+' && ch != '-' { + r.pos = p return 0, false } - for { - ch, ok = r.readByte() - if !ok { - break - } - if ch < '0' || ch > '9' { - r.unreadByte() - break - } + hasExponent := false + for p < n && data[p] >= '0' && data[p] <= '9' { + p++ hasExponent = true } if !hasExponent { + r.pos = p return 0, false } isFloat = true - } else { //nolint:gocritic - if ok { - r.unreadByte() - } + } else if consumedEnd { + p-- // the byte that ended the digit run is not part of the number } - chars := r.data[startPos:r.pos] + + r.pos = p + chars := data[startPos:p] if isFloat { // Unfortunately, strconv.ParseFloat requires a string - there is no []byte equivalent. This means we can't // avoid a heap allocation here. Easyjson works around this by creating an unsafe string that points directly // at the existing bytes, but in our default implementation we can't use unsafe. - n, err := strconv.ParseFloat(string(chars), 64) - return n, err == nil - } else { //nolint:revive - n, ok := parseIntFromBytes(chars) - return float64(n), ok + num, err := strconv.ParseFloat(string(chars), 64) + return num, err == nil } + num, ok := parseIntFromBytes(chars) + return float64(num), ok } func (r *tokenReader) readString() ([]byte, error) { - startPos := r.pos // the opening quote mark has already been read - var chars []byte - haveEscaped := false - var reader bytes.Reader // bytes.Reader understands multi-byte characters - reader.Reset(r.data) - _, _ = reader.Seek(int64(r.pos), io.SeekStart) + data, n := r.data, r.len + start := r.pos // the opening quote mark has already been read + p := start + // Everything except the closing quote and the escape character passes through verbatim + // (a quote or backslash byte can never occur inside a multi-byte UTF-8 sequence, so byte + // comparisons are safe), which makes the whole string a zero-copy subslice of the input + // unless it contains an escape. + for p < n { + b := data[p] + if b == '"' { + r.pos = p + 1 + if p == start { + return nil, nil + } + return data[start:p], nil + } + if b == '\\' { + return r.decodeString(start, p) + } + p++ + } + return nil, r.syntaxErrorOnLastToken(errMsgInvalidString) +} +// decodeString handles a string that cannot be returned as a subslice of the input: the escape +// sequence at position p ends the plain prefix that began at start (just past the opening quote +// mark). It decodes the rest of the string into a new buffer in one forward scan, bulk-copying +// each run of plain characters. +func (r *tokenReader) decodeString(start, p int) ([]byte, error) { + data, n := r.data, r.len + buf := make([]byte, 0, decodedStringCapacity(data, start, n)) + buf = append(buf, data[start:p]...) for { - ch, _, err := reader.ReadRune() - if err != nil { - return nil, r.syntaxErrorOnLastToken(errMsgInvalidString) + runStart := p + for p < n && plainStringChars[data[p]] { + p++ } - if ch == '"' { - break + buf = append(buf, data[runStart:p]...) + if p >= n { + return nil, r.syntaxErrorOnLastToken(errMsgInvalidString) } - if ch != '\\' { - if haveEscaped { - chars = appendRune(chars, ch) + b := data[p] + if b == '"' { + r.pos = p + 1 + return buf, nil + } + if b >= utf8.RuneSelf { + // Decoding and re-encoding a character passes it through unchanged, except that + // each invalid UTF-8 byte becomes the Unicode replacement character. + _, size := utf8.DecodeRune(data[p:]) + if size > 1 { + buf = append(buf, data[p:p+size]...) + p += size + } else { + buf = utf8.AppendRune(buf, utf8.RuneError) + p++ } continue } - if !haveEscaped { - pos := (r.len - reader.Len()) - 1 // don't include the backslash we just read - chars = make([]byte, pos-startPos, pos-startPos+20) - if pos > startPos { - copy(chars, r.data[startPos:pos]) - } - haveEscaped = true - } - ch, _, err = reader.ReadRune() - if err != nil { + // b == '\\' + p++ + if p >= n { return nil, r.syntaxErrorOnLastToken(errMsgInvalidString) } - switch ch { + // All valid escape characters are ASCII, so reading a byte here is equivalent to reading + // a character: any multi-byte or invalid sequence takes the default (error) branch. + esc := data[p] + p++ + switch esc { case '"', '\\', '/': - chars = appendRune(chars, ch) + buf = append(buf, esc) case 'b': - chars = appendRune(chars, '\b') + buf = append(buf, '\b') case 'f': - chars = appendRune(chars, '\f') + buf = append(buf, '\f') case 'n': - chars = appendRune(chars, '\n') + buf = append(buf, '\n') case 'r': - chars = appendRune(chars, '\r') + buf = append(buf, '\r') case 't': - chars = appendRune(chars, '\t') + buf = append(buf, '\t') case 'u': - if ch, ok := readHexChar(&reader); ok { - chars = appendRune(chars, ch) - } else { + ch, ok := readHex4(data, p, n) + if !ok { return nil, r.syntaxErrorOnLastToken(errMsgInvalidString) } + // AppendRune encodes a surrogate code point as the replacement character. + buf = utf8.AppendRune(buf, ch) + p += 4 default: return nil, r.syntaxErrorOnLastToken(errMsgInvalidString) } } - r.pos = r.len - reader.Len() - if haveEscaped { - if len(chars) == 0 { - return nil, nil - } - return chars, nil - } else { //nolint:revive - pos := r.pos - 1 - if pos <= startPos { - return nil, nil +} + +// decodedStringCapacity returns a buffer capacity for decoding the string whose content starts at +// start (just past the opening quote mark): the string's raw length in the input. Decoding never +// needs more bytes than the raw form, except when invalid UTF-8 bytes are replaced (three bytes +// for one); the buffer grows in that rare case. +func decodedStringCapacity(data []byte, start, n int) int { + p := start + for p < n { + switch data[p] { + case '\\': + p += 2 + case '"': + return p - start + default: + p++ } - return r.data[startPos:pos], nil } + return n - start } -func readHexChar(reader *bytes.Reader) (rune, bool) { - var digits [4]byte +// readHex4 parses the four hex digits of a \u escape starting at data[p]. +func readHex4(data []byte, p, n int) (rune, bool) { + if p+4 > n { + return 0, false + } + var v rune for i := 0; i < 4; i++ { - ch, err := reader.ReadByte() - if err != nil || ((ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') && (ch < 'A' || ch > 'F')) { + c := data[p+i] + switch { + case c >= '0' && c <= '9': + v = v<<4 + rune(c-'0') + case c >= 'a' && c <= 'f': + v = v<<4 + rune(c-'a'+10) + case c >= 'A' && c <= 'F': + v = v<<4 + rune(c-'A'+10) + default: return 0, false } - digits[i] = ch //nolint:gosec // G602 false positive: i is bounded by [0,4) matching the array size } - n, _ := strconv.ParseUint(string(digits[:]), 16, 32) - return rune(n), true + return v, true } func (r *tokenReader) syntaxErrorOnLastToken(msg string) error { //nolint:unparam @@ -543,11 +712,10 @@ func (r *tokenReader) syntaxErrorOnLastToken(msg string) error { //nolint:unpara } func (r *tokenReader) syntaxErrorOnNextToken(msg string) error { - t, err := r.next() - if err != nil { + if err := r.next(); err != nil { return err } - return SyntaxError{Message: msg, Value: t.description(), Offset: r.LastPos()} + return SyntaxError{Message: msg, Value: r.tok.description(), Offset: r.LastPos()} } // This is faster than creating a string to pass to strconv.Atoi. @@ -575,12 +743,6 @@ func parseIntFromBytes(chars []byte) (int64, bool) { return ret, true } -func appendRune(out []byte, ch rune) []byte { - var encodedRune [10]byte - n := utf8.EncodeRune(encodedRune[0:10], ch) - return append(out, encodedRune[0:n]...) -} - func valueKindFromTokenKind(k tokenKind) ValueKind { switch k { case nullToken: From 830628af5ae7b92ab1762a772f54fbf0efb59bb1 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:57:12 -0700 Subject: [PATCH 2/3] test: pin tokenizer edge-case behavior with table-driven tests Table-driven cases for the paths the permutation suites do not reach: malformed numbers (exponent truncations, lone minus), string decoding failures (unterminated forms, bad escapes, bad or truncated unicode escapes) and pass-through of invalid UTF-8 after an escape, missing colons after property names, wrong tokens where an array or object needs a comma or end delimiter, and each entry point's interaction with a pushed-back token. Errors are asserted exactly: type, message, value, and offset. Three defensive helper branches are covered directly because no input reaches them through the entry points. token_reader_default.go is at 100% statement coverage with these tests. --- jreader/token_reader_default_test.go | 257 +++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) diff --git a/jreader/token_reader_default_test.go b/jreader/token_reader_default_test.go index 5b80a5e..b2106ac 100644 --- a/jreader/token_reader_default_test.go +++ b/jreader/token_reader_default_test.go @@ -1 +1,258 @@ package jreader + +import ( + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +// These tests pin the default tokenReader's behavior on inputs that the broader permutation +// suites do not reach: malformed numbers and strings, wrong tokens where a structural +// delimiter is required, and the interaction of each entry point with a pushed-back token. +// Expected errors are asserted exactly (type, message, value, and offset), since all of +// those are part of the reader's observable behavior. + +// parkToken parses the next token and pushes it back, leaving it as the tokenReader's +// unread token. Null does exactly that for any token that is not a null (including +// returning an error for a lone punctuation token, which still leaves the token parked). +func parkToken(tr *tokenReader) { + _, _ = tr.Null() +} + +func TestTokenReaderNumberEdgeCases(t *testing.T) { + for _, tc := range []struct { + name string + input string + want float64 + wantErr error + }{ + {name: "uppercase exponent marker", input: "2E3", want: 2000}, + {name: "negative zero", input: "-0", want: 0}, + {name: "exponent marker at end of input", input: "1e", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, + {name: "exponent marker followed by non-digit", input: "1ex", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, + {name: "exponent sign at end of input", input: "1e+", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, + {name: "exponent sign followed by non-digit", input: "1e-x", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, + {name: "minus sign alone", input: "-", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, + {name: "error offset counts leading whitespace", input: " 1e", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 1}}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTokenReader([]byte(tc.input)) + n, err := tr.Number() + if tc.wantErr != nil { + require.Equal(t, tc.wantErr, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, n) + }) + } +} + +func TestTokenReaderStringDecodingEdgeCases(t *testing.T) { + for _, tc := range []struct { + name string + input string + want string + wantErr error + }{ + {name: "control-character escapes", input: `"\b\f\n\r"`, want: "\b\f\n\r"}, + {name: "uppercase hex digits in unicode escape", input: `"\u00AF"`, want: "\u00af"}, + {name: "multi-byte character after an escape", input: "\"\\t\u00e9\"", want: "\t\u00e9"}, + {name: "invalid UTF-8 byte after an escape becomes replacement character", + input: "\"\\t\xffx\"", want: "\t\ufffdx"}, + {name: "unterminated with no escape", input: `"abc`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + {name: "unterminated after an escape", input: `"a\tb`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + {name: "backslash at end of input", input: `"a\`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + {name: "invalid escape character", input: `"\q"`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + {name: "non-hex digit in unicode escape", input: `"\u12G4"`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + {name: "unicode escape truncated by end of input", input: `"\u12`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTokenReader([]byte(tc.input)) + s, err := tr.String() + if tc.wantErr != nil { + require.Equal(t, tc.wantErr, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, s) + }) + } +} + +func TestTokenReaderPropertyNameEdgeCases(t *testing.T) { + for _, tc := range []struct { + name string + input string + want string + wantErr error + }{ + {name: "name followed by colon", input: `"a":`, want: "a"}, + {name: "input ends after name", input: `"a"`, wantErr: io.EOF}, + {name: "value instead of colon", input: `"a" 1`, + wantErr: SyntaxError{Message: errMsgExpectedColon, Value: "number", Offset: 4}}, + {name: "malformed token instead of colon", input: `"a" tru`, + wantErr: SyntaxError{Message: errMsgUnexpectedSymbol, Value: "tru", Offset: 4}}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTokenReader([]byte(tc.input)) + name, err := tr.PropertyName() + if tc.wantErr != nil { + require.Equal(t, tc.wantErr, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, string(name)) + }) + } +} + +func TestTokenReaderEndDelimiterOrCommaEdgeCases(t *testing.T) { + for _, tc := range []struct { + name string + input string + park bool // parse and push back the first token before the call under test + delimiter byte + wantEnd bool + wantErr error + }{ + {name: "pushed-back end delimiter matches", input: "]", park: true, delimiter: ']', + wantEnd: true}, + {name: "pushed-back comma continues", input: ",", park: true, delimiter: ']'}, + {name: "pushed-back value token in array", input: "5", park: true, delimiter: ']', + wantErr: SyntaxError{Message: errMsgBadArrayItem, Value: "number", Offset: 0}}, + {name: "pushed-back value token in object", input: "5", park: true, delimiter: '}', + wantErr: SyntaxError{Message: errMsgBadObjectItem, Value: "number", Offset: 0}}, + {name: "pushed-back wrong punctuation", input: ":", park: true, delimiter: ']', + wantErr: SyntaxError{Message: errMsgBadArrayItem, Value: "':'", Offset: 0}}, + {name: "end of input", input: "", delimiter: ']', wantErr: io.EOF}, + {name: "value token", input: "5", delimiter: ']', + wantErr: SyntaxError{Message: errMsgBadArrayItem, Value: "number", Offset: 0}}, + {name: "array start where an array should end", input: "[", delimiter: ']', + wantErr: SyntaxError{Message: errMsgBadArrayItem, Value: "array", Offset: 0}}, + {name: "malformed token", input: "tru", delimiter: ']', + wantErr: SyntaxError{Message: errMsgUnexpectedSymbol, Value: "tru", Offset: 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTokenReader([]byte(tc.input)) + if tc.park { + parkToken(&tr) + } + isEnd, err := tr.EndDelimiterOrComma(tc.delimiter) + if tc.wantErr != nil { + require.Equal(t, tc.wantErr, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.wantEnd, isEnd) + }) + } +} + +func TestTokenReaderAnyEdgeCases(t *testing.T) { + for _, tc := range []struct { + name string + input string + park bool + want AnyValue + wantErr error + }{ + {name: "pushed-back array start", input: "[", park: true, + want: AnyValue{Kind: ArrayValue}}, + {name: "pushed-back object start", input: "{", park: true, + want: AnyValue{Kind: ObjectValue}}, + {name: "pushed-back number", input: "5", park: true, + want: AnyValue{Kind: NumberValue, Number: 5}}, + {name: "pushed-back punctuation", input: ":", park: true, + wantErr: SyntaxError{Message: errMsgUnexpectedChar, Value: ":", Offset: 0}}, + {name: "malformed number", input: "1e", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, + {name: "unterminated string", input: `"abc`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTokenReader([]byte(tc.input)) + if tc.park { + parkToken(&tr) + } + got, err := tr.Any() + if tc.wantErr != nil { + require.Equal(t, tc.wantErr, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +func TestTokenReaderNextTokenErrors(t *testing.T) { + // Null is the generic next()-driven entry point: any malformed token surfaces through it. + for _, tc := range []struct { + name string + input string + wantErr error + }{ + {name: "malformed number", input: "1e", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, + {name: "unterminated string", input: `"abc`, + wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTokenReader([]byte(tc.input)) + _, err := tr.Null() + require.Equal(t, tc.wantErr, err) + }) + } +} + +func TestTokenReaderEOFWithPushedBackToken(t *testing.T) { + tr := newTokenReader([]byte("false")) + parkToken(&tr) + require.False(t, tr.EOF(), "a pushed-back token means the input is not exhausted") + b, err := tr.Bool() + require.NoError(t, err) + require.False(t, b) + require.True(t, tr.EOF()) +} + +func TestTokenReaderDelimiterAtEndOfInput(t *testing.T) { + tr := newTokenReader([]byte(" ")) + found, err := tr.Delimiter('[') + require.NoError(t, err) + require.False(t, found) +} + +// The remaining tests cover defensive branches of internal helpers directly, because no +// input can reach them through the entry points: getPos is only called after a putBack, +// readNumber always passes parseIntFromBytes at least one byte, and valueKind filters out +// the delimiter kinds before calling valueKindFromTokenKind. + +func TestTokenReaderGetPosWithoutPushedBackToken(t *testing.T) { + tr := newTokenReader([]byte(" 5 ")) + _, err := tr.Number() + require.NoError(t, err) + require.Equal(t, 2, tr.getPos()) +} + +func TestParseIntFromBytesRejectsEmptyInput(t *testing.T) { + _, ok := parseIntFromBytes(nil) + require.False(t, ok) +} + +func TestValueKindFromTokenKindHasNoDelimiterMapping(t *testing.T) { + require.Equal(t, ValueKind(-1), valueKindFromTokenKind(delimiterToken)) +} From 63ffa7cdaab399618046fb6db77825b117fb1b8b Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:32:55 -0700 Subject: [PATCH 3/3] test: pin lenient parsing behaviors surfaced by an external conformance sweep A sweep of the JSON parsing minefield corpus confirmed every outcome matches this reader's deliberately lenient baseline, and surfaced behaviors no in-repo test pinned. Added: - number leniencies that parse successfully: leading zeros, a dot with no fractional digits (with and without an exponent), no integer part - the overflow split: integer literals beyond int64 wrap, while an exponent overflowing float64 range is rejected - raw control characters (NUL, tab, newline) passing through strings verbatim on the zero-copy path - the non-ASCII single whitespace bytes (0x0B, 0x0C, 0x85, 0xA0) skipped between tokens - byte order marks rejected as unexpected characters, reported as code points - deep nesting scanning in constant space (the tokenizer tracks no nesting state; 100k-deep brackets need no recursion) --- jreader/token_reader_default_test.go | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/jreader/token_reader_default_test.go b/jreader/token_reader_default_test.go index b2106ac..eacf062 100644 --- a/jreader/token_reader_default_test.go +++ b/jreader/token_reader_default_test.go @@ -29,6 +29,15 @@ func TestTokenReaderNumberEdgeCases(t *testing.T) { }{ {name: "uppercase exponent marker", input: "2E3", want: 2000}, {name: "negative zero", input: "-0", want: 0}, + {name: "leading zero accepted", input: "012", want: 12}, + {name: "negative leading zero accepted", input: "-01", want: -1}, + {name: "dot with no fractional digits accepted", input: "1.", want: 1}, + {name: "no integer part accepted", input: "-.123", want: -0.123}, + {name: "dot with no fractional digits before exponent accepted", input: "2.e3", want: 2000}, + {name: "integer beyond int64 wraps", input: "9223372036854775808", + want: -9223372036854775808}, + {name: "exponent overflowing float64 range rejected", input: "1e400", + wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, {name: "exponent marker at end of input", input: "1e", wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, {name: "exponent marker followed by non-digit", input: "1ex", @@ -67,6 +76,9 @@ func TestTokenReaderStringDecodingEdgeCases(t *testing.T) { {name: "multi-byte character after an escape", input: "\"\\t\u00e9\"", want: "\t\u00e9"}, {name: "invalid UTF-8 byte after an escape becomes replacement character", input: "\"\\t\xffx\"", want: "\t\ufffdx"}, + {name: "raw NUL character passes through", input: "\"a\x00a\"", want: "a\x00a"}, + {name: "raw tab passes through", input: "\"\t\"", want: "\t"}, + {name: "raw newline passes through", input: "\"new\nline\"", want: "new\nline"}, {name: "unterminated with no escape", input: `"abc`, wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, {name: "unterminated after an escape", input: `"a\tb`, @@ -210,6 +222,12 @@ func TestTokenReaderNextTokenErrors(t *testing.T) { wantErr: SyntaxError{Message: errMsgInvalidNumber, Offset: 0}}, {name: "unterminated string", input: `"abc`, wantErr: SyntaxError{Message: errMsgInvalidString, Offset: 0}}, + // The offending byte is reported as a code point (string(byte) converts as a rune), + // so 0xEF renders as U+00EF. + {name: "UTF-8 byte order mark", input: "\xef\xbb\xbf{}", + wantErr: SyntaxError{Message: errMsgUnexpectedChar, Value: "ï", Offset: 0}}, + {name: "UTF-16 byte order mark", input: "\xff\xfe{\x00}\x00", + wantErr: SyntaxError{Message: errMsgUnexpectedChar, Value: "ÿ", Offset: 0}}, } { t.Run(tc.name, func(t *testing.T) { tr := newTokenReader([]byte(tc.input)) @@ -219,6 +237,55 @@ func TestTokenReaderNextTokenErrors(t *testing.T) { } } +func TestTokenReaderNonASCIIWhitespaceBytes(t *testing.T) { + // The whitespace table classifies each byte the way unicode.IsSpace classifies its + // Latin-1 code point, so these single bytes are skipped between tokens along with the + // standard space, tab, CR, and LF. + for _, tc := range []struct { + name string + ws byte + }{ + {name: "vertical tab", ws: 0x0B}, + {name: "form feed", ws: 0x0C}, + {name: "next line", ws: 0x85}, + {name: "no-break space", ws: 0xA0}, + } { + t.Run(tc.name, func(t *testing.T) { + tr := newTokenReader([]byte{tc.ws, '5', tc.ws}) + n, err := tr.Number() + require.NoError(t, err) + require.Equal(t, float64(5), n) + require.Equal(t, 1, tr.LastPos()) + require.True(t, tr.EOF()) + }) + } +} + +func TestTokenReaderDeepNestingNeedsNoPerDepthState(t *testing.T) { + // The tokenizer tracks no nesting state (matching brackets is the caller's job), so + // arbitrarily deep structures scan in constant space with no recursion. + const depth = 100_000 + data := make([]byte, 0, 2*depth) + for i := 0; i < depth; i++ { + data = append(data, '[') + } + for i := 0; i < depth; i++ { + data = append(data, ']') + } + tr := newTokenReader(data) + for i := 0; i < depth; i++ { + found, err := tr.Delimiter('[') + require.NoError(t, err) + require.True(t, found) + } + for i := 0; i < depth; i++ { + isEnd, err := tr.EndDelimiterOrComma(']') + require.NoError(t, err) + require.True(t, isEnd) + } + require.True(t, tr.EOF()) +} + func TestTokenReaderEOFWithPushedBackToken(t *testing.T) { tr := newTokenReader([]byte("false")) parkToken(&tr)