From e078638bdb6dd978860eac6acf620c57de5cd01f Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Sun, 2 Aug 2026 04:08:37 -0400 Subject: [PATCH 01/30] Record a behavioral baseline for the JSON functions Nine new test files capture the current, unchanged behavior of the whole JSON surface so that any later change to it shows up as an explicit result diff rather than as a judgement call. - `func_json_format` - the exact output bytes of every producer, in several input formattings, including the chained calls where one JSON function feeds another, and `JSON_EXTRACT`'s dependence on whether it is evaluated in string or numeric context. - `func_json_invalid` - input that is not valid JSON, is NULL, is not a string, or carries bytes JSON cannot hold unescaped; the recorded observable is the whole trichotomy of value, warning and error code. - `func_json_depth` - where the nesting limit sits for each container mix, every function at and one past it, path depth, and the composed depth reached by splicing an acceptable document into another one. - `func_json_agg_limits` - `JSON_ARRAYAGG`'s length cut walked byte by byte across the interesting offsets, for string and for non-string members, next to `JSON_OBJECTAGG`. - `func_json_charset` - the character set and collation every function reports, the same bytes read under different labels, character sets in which JSON punctuation is a letter, wide character sets, arguments of mixed character sets meeting in one call, and what `CONVERT` and `CAST` each make of a value that is already a document. - `func_json_columns` - a JSON column and its check constraint in every shape the constraint can take, including what the functions make of a column whose stored bytes no longer satisfy it, the same column reached through a subquery and through merged and materialised views, a user-created temporary table, stores that truncate or convert on the way in, tables rebuilt by `ALTER`, generated columns, and the temporary tables the server makes for grouping, derived tables, union branches and window frames. - `func_json_sp` - JSON values in stored program variables: how a variable declared JSON differs from a column declared JSON, what the declared type does to a value on assignment, and a variable that is both source and target of the same statement. - `func_json_table` - what `JSON_TABLE` output columns are treated as when fed back into JSON functions, a document read from a column whose stored bytes are not valid, and a `DEFAULT` clause supplying a value for a column declared JSON. - `func_json_mislabel` - the same stored bytes read under a different character set label, reached by relabelling a column through `VARBINARY`. Converting a column repairs or rejects the bytes first, so this is the only way to put a document in front of the parser that is not valid in the character set it claims to be in. Two of these turn on something easy to get wrong about the shape of the test itself, so they are built to show it. A stored program runs under the `sql_mode` it was CREATED with and not the one in force at the call, so setting the mode around a `CALL` decides nothing. `func_json_sp` therefore gives each mode a procedure of its own and calls each under both modes, which puts that invariance on the record beside the behaviour. The non-strict one truncates an over-long assignment silently - no error and no warning - leaving `{"a": 9,` in a `VARCHAR(8)`. A document spliced into a document of another character set is written in the character set of the answer, and nothing refuses it. `func_json_charset` records the two ways that set is chosen: a mutator takes it from its document argument and converts the fragment into it, writing a latin1 fragment back out as utf8mb4, while a constructor aggregates over all of its arguments, so the fragment can decide the set and its bytes then go across unchanged. Output that is not printable ASCII is recorded through `HEX()` so the result files stay readable and diffable. `--echo` text is part of the recorded output, so the prose in these files is a gated expectation rather than a comment. One combination is deliberately left out: `JSON_OVERLAPS` crashes the server when an argument is an array nested past the scanner limit, so `func_json_columns` excludes that one row from that one statement, with a comment saying why. Every other function in the file takes it. Tests only; no server code is touched. --- mysql-test/main/func_json_agg_limits.result | 382 ++++++++ mysql-test/main/func_json_agg_limits.test | 195 ++++ mysql-test/main/func_json_charset.result | 584 ++++++++++++ mysql-test/main/func_json_charset.test | 267 ++++++ mysql-test/main/func_json_columns.result | 936 ++++++++++++++++++++ mysql-test/main/func_json_columns.test | 422 +++++++++ mysql-test/main/func_json_depth.result | 378 ++++++++ mysql-test/main/func_json_depth.test | 155 ++++ mysql-test/main/func_json_format.result | 508 +++++++++++ mysql-test/main/func_json_format.test | 235 +++++ mysql-test/main/func_json_invalid.result | 692 +++++++++++++++ mysql-test/main/func_json_invalid.test | 262 ++++++ mysql-test/main/func_json_mislabel.result | 86 ++ mysql-test/main/func_json_mislabel.test | 67 ++ mysql-test/main/func_json_sp.result | 457 ++++++++++ mysql-test/main/func_json_sp.test | 371 ++++++++ mysql-test/main/func_json_table.result | 87 ++ mysql-test/main/func_json_table.test | 74 ++ 18 files changed, 6158 insertions(+) create mode 100644 mysql-test/main/func_json_agg_limits.result create mode 100644 mysql-test/main/func_json_agg_limits.test create mode 100644 mysql-test/main/func_json_charset.result create mode 100644 mysql-test/main/func_json_charset.test create mode 100644 mysql-test/main/func_json_columns.result create mode 100644 mysql-test/main/func_json_columns.test create mode 100644 mysql-test/main/func_json_depth.result create mode 100644 mysql-test/main/func_json_depth.test create mode 100644 mysql-test/main/func_json_format.result create mode 100644 mysql-test/main/func_json_format.test create mode 100644 mysql-test/main/func_json_invalid.result create mode 100644 mysql-test/main/func_json_invalid.test create mode 100644 mysql-test/main/func_json_mislabel.result create mode 100644 mysql-test/main/func_json_mislabel.test create mode 100644 mysql-test/main/func_json_sp.result create mode 100644 mysql-test/main/func_json_sp.test create mode 100644 mysql-test/main/func_json_table.result create mode 100644 mysql-test/main/func_json_table.test diff --git a/mysql-test/main/func_json_agg_limits.result b/mysql-test/main/func_json_agg_limits.result new file mode 100644 index 0000000000000..6197735f35fb5 --- /dev/null +++ b/mysql-test/main/func_json_agg_limits.result @@ -0,0 +1,382 @@ +# +# Behavioral baseline: the JSON aggregate functions. +# +# JSON_ARRAYAGG inherits the GROUP_CONCAT length cap, so its result is +# chopped at group_concat_max_len bytes. Depending on where the cut +# lands, the value it returns is either still parseable or not, and +# the only signal is a warning. This test walks the cut across every +# interesting byte offset and records the exact result each time. +# JSON_OBJECTAGG is included next to it, including the fact that it +# does not honour the same cap. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES +(1, 'aaaaaaaaaa'), (1, 'bbbbbbbbbb'), (1, 'cccccccccc'), +(2, 'dd'), (2, 'ee'); +# +# 1. Uncut results. +# +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +["aaaaaaaaaa","bbbbbbbbbb","cccccccccc","dd","ee"] +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +h +5B2261616161616161616161222C2262626262626262626262222C2263636363636363636363222C226464222C226565225D +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1; +ok +1 +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"] +2 ["dd","ee"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +v +{"aaaaaaaaaa":"aaaaaaaaaa", "bbbbbbbbbb":"bbbbbbbbbb", "cccccccccc":"cccccccccc", "dd":"dd", "ee":"ee"} +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1; +h +7B2261616161616161616161223A2261616161616161616161222C202262626262626262626262223A2262626262626262626262222C202263636363636363636363223A2263636363636363636363222C20226464223A226464222C20226565223A226565227D +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1; +ok +1 +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 {"aaaaaaaaaa":"aaaaaaaaaa", "bbbbbbbbbb":"bbbbbbbbbb", "cccccccccc":"cccccccccc"} +2 {"dd":"dd", "ee":"ee"} +# the aggregates over an empty input +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +v +NULL +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +v +NULL +# a single row +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'dd'; +v +["dd"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'dd'; +v +{"dd":"dd"} +# NULL and non-string members +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT NULL) d; +v +[1,null] +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +v +[1,2] +SELECT JSON_OBJECTAGG(k, x) AS v +FROM (SELECT 'a' AS k, 1 AS x UNION ALL SELECT 'b', NULL) d; +v +{"a":1, "b":null} +# DISTINCT and ORDER BY inside the aggregate +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1 WHERE g = 1; +v +["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"] +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +v +["cccccccccc","bbbbbbbbbb","aaaaaaaaaa"] +# an aggregate over documents rather than strings +SELECT JSON_ARRAYAGG(JSON_OBJECT('k', a)) AS v FROM t1 WHERE g = 2; +v +[{"k": "dd"},{"k": "ee"}] +SELECT JSON_ARRAYAGG(JSON_COMPACT(CONCAT('{"k":"', a, '"}'))) AS v +FROM t1 WHERE g = 2; +v +[{"k":"dd"},{"k":"ee"}] +# +# 2. The cut, walked byte by byte. The uncut result for group 1 is +# ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"], 40 bytes. +# +SET SESSION group_concat_max_len = 40; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C2262626262626262626262222C2263636363636363636363225D 1 +SET SESSION group_concat_max_len = 39; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C2262626262626262626262222C2263636363636363636363225D 1 +SET SESSION group_concat_max_len = 30; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C2262626262626262626262222C226363225D 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C226262626262225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# the cut lands just after the separator, leaving an empty member +SET SESSION group_concat_max_len = 15; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C22225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# the cut lands on the opening quote of the next member +SET SESSION group_concat_max_len = 14; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C225D 0 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# the cut lands on the separator itself +SET SESSION group_concat_max_len = 13; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B226161616161616161616122225D 0 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 +# the cut lands inside the first member +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B22616161225D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B226161225D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +# the smallest the cap can be +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1 WHERE a = 'dd'; +h +5B226464225D +SET SESSION group_concat_max_len = DEFAULT; +# +# 3. A cut that falls inside an escape sequence, where the trailing +# bytes are not a complete character. +# +CREATE TABLE t2 (a VARCHAR(20)); +INSERT INTO t2 VALUES ('xx"yy'), ('zzzz'); +SET SESSION group_concat_max_len = 10; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B2278785C227979222C225D 0 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B2278785C22797922225D 0 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B2278785C227979225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B2278785C2279225D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B2278785C22225D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; +# +# 3a. A cut over members that are not quoted strings. The re-close +# path only fixes a split trailing quote, so a cut that ends on a +# digit or inside an object takes the other branch entirely. +# +CREATE TABLE t3 (n INT, j JSON); +INSERT INTO t3 VALUES (111, '{"k":1}'), (222, '{"k":2}'), (333, '{"k":3}'); +# numeric members: the cut can leave a shorter but still parseable array +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +h ok +5B3131312C3232322C335D 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +h ok +5B3131312C325D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# document members: the cut lands inside an object instead +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +h ok +5B7B226B223A317D2C7B226B225D 0 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 13 +SET SESSION group_concat_max_len = 17; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +h ok +5B7B226B223A317D2C7B226B223A327D2C7B5D 0 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 19 +# members built by a constructor rather than read from a column +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS ok FROM t3; +h ok +5B7B226B223A203131317D2C7B5D 0 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; +# +# 4. A cut result consumed by another JSON function. Today the cut +# value is handed on as a document argument like any other. +# +SET SESSION group_concat_max_len = 14; +SELECT JSON_SET(JSON_ARRAYAGG(a), '$[0]', 1) AS v FROM t1 WHERE g = 1; +v +NULL +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a), '$[0]') AS v FROM t1 WHERE g = 1; +v +NULL +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +[["aaaaaaaaaa","]] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_DEPTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +NULL +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_depth' +SELECT JSON_LENGTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +NULL +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_length' +SELECT JSON_COMPACT(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +NULL +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_compact' +# an uncut group and a cut group in the same statement +SET SESSION group_concat_max_len = 14; +SELECT g, HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 GROUP BY g ORDER BY g; +g h ok +1 5B2261616161616161616161222C225D 0 +2 5B226464222C226565225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET SESSION group_concat_max_len = DEFAULT; +# +# 5. JSON_OBJECTAGG does not honour group_concat_max_len. +# +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h, JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok +FROM t1 WHERE g = 1; +h ok +7B2261616161616161616161223A2261616161616161616161222C202262626262626262626262223A2262626262626262626262222C202263636363636363636363223A2263636363636363636363227D 1 +SET SESSION group_concat_max_len = DEFAULT; +# duplicate keys are kept, not merged +SELECT JSON_OBJECTAGG(g, a) AS v FROM t1 WHERE g = 1; +v +{"1":"aaaaaaaaaa", "1":"bbbbbbbbbb", "1":"cccccccccc"} +SELECT JSON_VALID(JSON_OBJECTAGG(g, a)) AS ok FROM t1 WHERE g = 1; +ok +1 +# a key that needs escaping +SELECT JSON_OBJECTAGG(k, 1) AS v +FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +v +{"a\"b":1, "c\\d":1} +SELECT JSON_VALID(JSON_OBJECTAGG(k, 1)) AS ok +FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +ok +1 +# a NULL key +SELECT JSON_OBJECTAGG(k, 1) AS v +FROM (SELECT NULL AS k UNION ALL SELECT 'b') d; +v +{"b":1} +# +# 6. The result metadata the aggregates report, which decides how the +# bytes are converted on their way to the client. +# +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 4194312 11 Y 0 0 45 +v +["dd","ee"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 4194304 22 Y 0 0 45 +v +{"dd":"dd", "ee":"ee"} +SELECT JSON_ARRAY(1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 40 3 Y 0 39 45 +v +[1] +SELECT JSON_SET('{}', '$.a', 1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 68 8 Y 0 39 45 +v +{"a": 1} +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_limits.test b/mysql-test/main/func_json_agg_limits.test new file mode 100644 index 0000000000000..6c08d153754d6 --- /dev/null +++ b/mysql-test/main/func_json_agg_limits.test @@ -0,0 +1,195 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: the JSON aggregate functions. +--echo # +--echo # JSON_ARRAYAGG inherits the GROUP_CONCAT length cap, so its result is +--echo # chopped at group_concat_max_len bytes. Depending on where the cut +--echo # lands, the value it returns is either still parseable or not, and +--echo # the only signal is a warning. This test walks the cut across every +--echo # interesting byte offset and records the exact result each time. +--echo # JSON_OBJECTAGG is included next to it, including the fact that it +--echo # does not honour the same cap. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES + (1, 'aaaaaaaaaa'), (1, 'bbbbbbbbbb'), (1, 'cccccccccc'), + (2, 'dd'), (2, 'ee'); + +--echo # +--echo # 1. Uncut results. +--echo # + +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1; +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +--echo # the aggregates over an empty input +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +--echo # a single row +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'dd'; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'dd'; +--echo # NULL and non-string members +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT NULL) d; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +SELECT JSON_OBJECTAGG(k, x) AS v + FROM (SELECT 'a' AS k, 1 AS x UNION ALL SELECT 'b', NULL) d; +--echo # DISTINCT and ORDER BY inside the aggregate +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +--echo # an aggregate over documents rather than strings +SELECT JSON_ARRAYAGG(JSON_OBJECT('k', a)) AS v FROM t1 WHERE g = 2; +SELECT JSON_ARRAYAGG(JSON_COMPACT(CONCAT('{"k":"', a, '"}'))) AS v + FROM t1 WHERE g = 2; + +--echo # +--echo # 2. The cut, walked byte by byte. The uncut result for group 1 is +--echo # ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"], 40 bytes. +--echo # + +SET SESSION group_concat_max_len = 40; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 39; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 30; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 20; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cut lands just after the separator, leaving an empty member +SET SESSION group_concat_max_len = 15; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cut lands on the opening quote of the next member +SET SESSION group_concat_max_len = 14; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cut lands on the separator itself +SET SESSION group_concat_max_len = 13; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cut lands inside the first member +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the smallest the cap can be +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1 WHERE a = 'dd'; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 3. A cut that falls inside an escape sequence, where the trailing +--echo # bytes are not a complete character. +--echo # + +CREATE TABLE t2 (a VARCHAR(20)); +INSERT INTO t2 VALUES ('xx"yy'), ('zzzz'); +SET SESSION group_concat_max_len = 10; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 6; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 3a. A cut over members that are not quoted strings. The re-close +--echo # path only fixes a split trailing quote, so a cut that ends on a +--echo # digit or inside an object takes the other branch entirely. +--echo # + +CREATE TABLE t3 (n INT, j JSON); +INSERT INTO t3 VALUES (111, '{"k":1}'), (222, '{"k":2}'), (333, '{"k":3}'); +--echo # numeric members: the cut can leave a shorter but still parseable array +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +--echo # document members: the cut lands inside an object instead +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +SET SESSION group_concat_max_len = 17; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +--echo # members built by a constructor rather than read from a column +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS ok FROM t3; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; + +--echo # +--echo # 4. A cut result consumed by another JSON function. Today the cut +--echo # value is handed on as a document argument like any other. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT JSON_SET(JSON_ARRAYAGG(a), '$[0]', 1) AS v FROM t1 WHERE g = 1; +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a), '$[0]') AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_DEPTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_LENGTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_COMPACT(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +--echo # an uncut group and a cut group in the same statement +SET SESSION group_concat_max_len = 14; +SELECT g, HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 GROUP BY g ORDER BY g; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 5. JSON_OBJECTAGG does not honour group_concat_max_len. +--echo # + +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h, JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; +--echo # duplicate keys are kept, not merged +SELECT JSON_OBJECTAGG(g, a) AS v FROM t1 WHERE g = 1; +SELECT JSON_VALID(JSON_OBJECTAGG(g, a)) AS ok FROM t1 WHERE g = 1; +--echo # a key that needs escaping +SELECT JSON_OBJECTAGG(k, 1) AS v + FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +SELECT JSON_VALID(JSON_OBJECTAGG(k, 1)) AS ok + FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +--echo # a NULL key +SELECT JSON_OBJECTAGG(k, 1) AS v + FROM (SELECT NULL AS k UNION ALL SELECT 'b') d; + +--echo # +--echo # 6. The result metadata the aggregates report, which decides how the +--echo # bytes are converted on their way to the client. +--echo # + +--enable_metadata +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 2; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +SELECT JSON_ARRAY(1) AS v; +SELECT JSON_SET('{}', '$.a', 1) AS v; +--disable_metadata + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result new file mode 100644 index 0000000000000..43575c1c933a8 --- /dev/null +++ b/mysql-test/main/func_json_charset.result @@ -0,0 +1,584 @@ +# +# Behavioral baseline: character sets. +# +# The same bytes are a different document in a different character +# set, and in some character sets the bytes that JSON uses as +# punctuation are ordinary letters. This test records the character +# set and collation every JSON function reports for its result, what +# each function makes of the same bytes under different labels, and +# what happens when arguments of different character sets meet in one +# call. +# +SET NAMES utf8mb4; +# +# 1. The character set and collation of every result. +# +SELECT CHARSET(JSON_ARRAY(1)) AS cs, COLLATION(JSON_ARRAY(1)) AS co; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_OBJECT('a',1)) AS cs, COLLATION(JSON_OBJECT('a',1)) AS co; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_SET('{}','$.a',1)) AS cs, COLLATION(JSON_SET('{}','$.a',1)) AS co; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_INSERT('{}','$.a',1)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_REPLACE('{"a":1}','$.a',2)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_REMOVE('{"a":1}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_ARRAY_APPEND('[1]','$',2)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_ARRAY_INSERT('[1]','$[0]',2)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_MERGE('[1]','[2]')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_MERGE_PATCH('{"a":1}','{"b":2}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_EXTRACT('{"a":1}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_QUERY('{"a":{"b":1}}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_VALUE('{"a":1}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_KEYS('{"a":1}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_SEARCH('["x"]','one','x')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_COMPACT('{}')) AS cs, CHARSET(JSON_LOOSE('{}')) AS cs2; +cs cs2 +utf8mb4 utf8mb4 +SELECT CHARSET(JSON_DETAILED('{}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_NORMALIZE('{}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_TYPE('{}')) AS cs; +cs +utf8mb3 +# JSON_QUOTE and JSON_UNQUOTE, whose collations differ from the rest +SELECT CHARSET(JSON_QUOTE('a')) AS cs, COLLATION(JSON_QUOTE('a')) AS co; +cs co +utf8mb4 utf8mb4_bin +SELECT CHARSET(JSON_UNQUOTE('"a"')) AS cs, COLLATION(JSON_UNQUOTE('"a"')) AS co; +cs co +utf8mb4 utf8mb4_bin +SELECT COLLATION(CONCAT(JSON_UNQUOTE('"a"'), '')) AS co; +co +utf8mb4_bin +SELECT JSON_UNQUOTE('"A"') = 'a' AS case_sensitive_compare; +case_sensitive_compare +0 +SELECT JSON_VALUE('{"a":"A"}','$.a') = 'a' AS case_sensitive_compare; +case_sensitive_compare +1 +# the aggregates +SELECT CHARSET(JSON_ARRAYAGG(x)) AS cs, COLLATION(JSON_ARRAYAGG(x)) AS co +FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_OBJECTAGG(x,x)) AS cs FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +cs +utf8mb4 +# +# 2. The same bytes under different labels. +# +# 0xFF is one character in latin1 and not a character at all in utf8mb4 +SELECT JSON_VALID(_latin1 X'227B7DFF22') AS as_latin1; +as_latin1 +1 +SELECT JSON_VALID(_binary X'227B7DFF22') AS as_binary; +as_binary +1 +SELECT HEX(JSON_UNQUOTE(_latin1 X'22FF22')) AS unq_latin1; +unq_latin1 +C3BF +SELECT HEX(JSON_UNQUOTE(_binary X'22FF22')) AS unq_binary; +unq_binary +C3BF +# the byte 0x5C is a backslash in latin1 and utf8mb4 +SELECT JSON_VALID(_latin1 X'225C6E22') AS as_latin1; +as_latin1 +1 +SELECT JSON_VALID(_utf8mb4 X'225C6E22') AS as_utf8mb4; +as_utf8mb4 +1 +# a two-byte character that ends in a byte JSON treats as punctuation +SELECT JSON_VALID(CONVERT(_latin1 X'227B7D22' USING utf8mb4)) AS v; +v +1 +# +# 3. A character set in which the JSON punctuation bytes are letters. +# In swe7 the bytes 5B 5C 5D 7B 7D are national characters, so the +# same bytes that encode an object elsewhere encode a word here. +# +SELECT JSON_VALID(_swe7 X'7B7D') AS braces_as_swe7; +braces_as_swe7 +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID(_swe7 X'5B5D') AS brackets_as_swe7; +brackets_as_swe7 +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID(_binary X'7B7D') AS braces_as_binary; +braces_as_binary +1 +SELECT HEX(CONVERT(_swe7 X'7B7D' USING utf8mb4)) AS swe7_to_utf8mb4; +swe7_to_utf8mb4 +C3A4C3A5 +SELECT HEX(JSON_ARRAY(_swe7 X'7B7D')) AS embedded; +embedded +5B227B7D225D +SELECT HEX(JSON_QUOTE(_swe7 X'7B7D')) AS quoted; +quoted +22C3A4C3A522 +SELECT JSON_VALID(JSON_ARRAY(_swe7 X'7B7D')) AS still_valid; +still_valid +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# a document written in swe7 itself +SELECT HEX(JSON_COMPACT(CONVERT('{"a":1}' USING swe7))) AS as_swe7; +as_swe7 +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_compact' at position 1 +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS as_swe7; +as_swe7 +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 1 +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS cs; +cs +swe7 +# a swe7 value spliced into a utf8mb4 document +SELECT HEX(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS spliced; +spliced +7B2261223A20312C202262223A2022C3A4C3A5227D +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS ok; +ok +1 +# +# 4. Wide character sets, where a character is not a byte. +# +SELECT HEX(CONVERT('{"a":1}' USING ucs2)) AS doc_ucs2; +doc_ucs2 +007B002200610022003A0031007D +SELECT JSON_VALID(CONVERT('{"a":1}' USING ucs2)) AS v; +v +1 +SELECT JSON_DEPTH(CONVERT('{"a":{"b":1}}' USING ucs2)) AS d; +d +3 +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS compacted; +compacted +007B002200610022003A0031007D +SELECT CHARSET(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS cs; +cs +ucs2 +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS setv; +setv +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 26 +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS cs; +cs +ucs2 +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), '$.a') AS v; +v +NULL +SELECT HEX(JSON_ARRAY(CONVERT('x' USING ucs2))) AS arr; +arr +005B002200780022005D +SELECT CHARSET(JSON_ARRAY(CONVERT('x' USING ucs2))) AS cs; +cs +ucs2 +# utf16 and utf32 +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf16)) AS v16; +v16 +1 +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf32)) AS v32; +v32 +1 +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING utf16))) AS c16; +c16 +007B002200610022003A0031007D +# +# 5. Arguments of different character sets meeting in one call. +# +SELECT CHARSET(JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4))) AS cs; +cs +utf8mb4 +SELECT JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4)) AS v; +v +["a", "b"] +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', +CONVERT('x' USING utf8mb4))) AS cs; +cs +latin1 +SELECT JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', +CONVERT('x' USING utf8mb4)) AS v; +v +{"a": 1, "b": "x"} +# a ucs2 document with a utf8mb4 value spliced into it +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +CONVERT('x' USING utf8mb4))) AS cs; +cs +ucs2 +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +CONVERT('x' USING utf8mb4))) AS v; +v +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 30 +# a utf8mb4 document with a ucs2 value spliced into it +SELECT CHARSET(JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2))) AS cs; +cs +utf8mb4 +SELECT JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2)) AS v; +v +{"a": 1, "b": "x"} +# a path argument in another character set than the document +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING latin1)) AS v; +v +1 +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING ucs2)) AS v; +v +NULL +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), CONVERT('$.a' USING latin1)) AS v; +v +NULL +# 5a. A value that is itself a document, spliced into a document of +# another character set. The fragment is written in the character +# set of the answer, and what differs between the functions is how +# that set is chosen: a mutator takes it from its document argument +# and converts the fragment into it, while a constructor aggregates +# over all of its arguments, so the fragment itself can decide the +# set and its bytes then go across unchanged. +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS typed_into_ucs2; +typed_into_ucs2 +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 38 +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS ok; +ok +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 38 +SELECT HEX(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS latin1_typed; +latin1_typed +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 18 +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS ok; +ok +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 18 +# the constructors take the same fragment without complaint +SELECT HEX(JSON_ARRAY( +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS embedded; +embedded +5B7B2278223A22E9227D5D +SELECT JSON_VALID(JSON_ARRAY( +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS still_valid; +still_valid +1 +SELECT CHARSET(JSON_ARRAY(JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) AS cs; +cs +latin1 +SELECT HEX(JSON_OBJECT('k', JSON_COMPACT(CONVERT('{"x":2}' USING ucs2)))) AS obj_embedded; +obj_embedded +007B0022006B0022003A00200000007B0000002200000078000000220000003A000000320000007D007D +# +# 6. Binary arguments, which are not text and are copied through. +# +CREATE TABLE t1 (b BLOB, v VARBINARY(10), c VARCHAR(10) CHARACTER SET latin1); +INSERT INTO t1 VALUES (X'FF', X'FF', X'FF'), (X'C3', X'C3', X'C3'); +SELECT CHARSET(JSON_ARRAY(b)) AS cs, COLLATION(JSON_ARRAY(b)) AS co FROM t1 LIMIT 1; +cs co +binary binary +SELECT HEX(JSON_ARRAY(b)) AS h, JSON_VALID(JSON_ARRAY(b)) AS ok FROM t1; +h ok +5B22FF225D 1 +5B22C3225D 1 +SELECT HEX(JSON_ARRAY(v)) AS h, JSON_VALID(JSON_ARRAY(v)) AS ok FROM t1; +h ok +5B22FF225D 1 +5B22C3225D 1 +SELECT HEX(JSON_ARRAY(c)) AS h, JSON_VALID(JSON_ARRAY(c)) AS ok FROM t1; +h ok +5B22FF225D 1 +5B22C3225D 1 +SELECT CHARSET(JSON_ARRAY(c)) AS cs FROM t1 LIMIT 1; +cs +latin1 +SELECT HEX(JSON_OBJECT('k', b)) AS h FROM t1; +h +7B226B223A2022FF227D +7B226B223A2022C3227D +SELECT HEX(JSON_SET('{}', '$.a', b)) AS h FROM t1; +h +7B2261223A2022C3BF227D +7B2261223A2022C383227D +SELECT CHARSET(JSON_SET('{}', '$.a', b)) AS cs FROM t1 LIMIT 1; +cs +utf8mb4 +SELECT HEX(JSON_QUOTE(b)) AS h FROM t1; +h +22C3BF22 +22C38322 +# the raw bytes survive a round trip through a document +SELECT HEX(JSON_VALUE(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +h +FF +C3 +SELECT HEX(JSON_EXTRACT(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +h +22FF22 +22C322 +DROP TABLE t1; +# 6a. A binary column holding a whole document, with bytes that are +# not valid utf8mb4 sitting in a value, in a key, and in the +# structural skeleton. Nothing converts these, so what the +# parser accepts is decided by the binary label alone. +CREATE TABLE t2 (b VARBINARY(60)); +INSERT INTO t2 VALUES +(X'7B2261223A22FF227D'), (X'7B22FF61223A317D'), (X'7B2261223A31FF7D'), +(X'7B2261223A22C3227D'), (X'7B2261223A22C3A9227D'); +SELECT HEX(b) AS stored, JSON_VALID(b) AS v FROM t2 ORDER BY stored; +stored v +7B2261223A22C3227D 1 +7B2261223A22C3A9227D 1 +7B2261223A22FF227D 1 +7B2261223A31FF7D 0 +7B22FF61223A317D 1 +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 6 +SELECT HEX(b) AS stored, HEX(JSON_EXTRACT(b, '$.a')) AS ext FROM t2 ORDER BY stored; +stored ext +7B2261223A22C3227D 22C322 +7B2261223A22C3A9227D 22C3A922 +7B2261223A22FF227D 22FF22 +7B2261223A31FF7D NULL +7B22FF61223A317D NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_extract' at position 6 +SELECT HEX(b) AS stored, HEX(JSON_VALUE(b, '$.a')) AS val FROM t2 ORDER BY stored; +stored val +7B2261223A22C3227D C3 +7B2261223A22C3A9227D C3A9 +7B2261223A22FF227D FF +7B2261223A31FF7D NULL +7B22FF61223A317D NULL +SELECT HEX(b) AS stored, HEX(JSON_KEYS(b)) AS ks FROM t2 ORDER BY stored; +stored ks +7B2261223A22C3227D 5B2261225D +7B2261223A22C3A9227D 5B2261225D +7B2261223A22FF227D 5B2261225D +7B2261223A31FF7D NULL +7B22FF61223A317D 5B22FF61225D +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_keys' at position 6 +SELECT HEX(b) AS stored, HEX(JSON_SET(b, '$.z', 1)) AS mutated FROM t2 ORDER BY stored; +stored mutated +7B2261223A22C3227D 7B2261223A2022C3222C20227A223A20317D +7B2261223A22C3A9227D 7B2261223A2022C3A9222C20227A223A20317D +7B2261223A22FF227D 7B2261223A2022FF222C20227A223A20317D +7B2261223A31FF7D NULL +7B22FF61223A317D 7B22FF61223A20312C20227A223A20317D +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 6 +SELECT CHARSET(JSON_EXTRACT(b, '$.a')) AS cs_ext, +CHARSET(JSON_SET(b, '$.z', 1)) AS cs_set FROM t2 LIMIT 1; +cs_ext cs_set +binary binary +DROP TABLE t2; +# +# 6b. CONVERT and CAST over a value that is already a document. +# CONVERT wraps the value and the wrapper is seen through, so the +# result is still treated as a document; CAST does not. +# +CREATE TABLE tc (j JSON); +INSERT INTO tc VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +SELECT HEX(JSON_ARRAY(CONVERT(j USING latin1))) AS h, +CHARSET(JSON_ARRAY(CONVERT(j USING latin1))) AS cs FROM tc; +h cs +5B7B2261223A20312C202262223A2022E9227D5D latin1 +SELECT JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tc; +ok +1 +# a mutator takes the same fragment and writes it back out in the +# character set of its document argument, which is where the answer's +# character set comes from +SELECT HEX(JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1))) AS h FROM tc; +h +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 27 +SELECT HEX(JSON_ARRAY(CONVERT(j USING binary))) AS h FROM tc; +h +5B7B2261223A20312C202262223A2022C3A9227D5D +# CAST produces a plain string, so it is quoted rather than embedded +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR))) AS h FROM tc; +h +5B227B5C22615C223A20312C205C22625C223A205C22C3A95C227D225D +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR CHARACTER SET latin1))) AS h FROM tc; +h +5B227B5C22615C223A20312C205C22625C223A205C22E95C227D225D +SELECT HEX(JSON_ARRAY(CONVERT(JSON_OBJECT('a', _utf8mb4 X'C3A9') USING latin1))) AS h; +h +5B7B2261223A2022E9227D5D +SELECT HEX(JSON_SET('{}', '$.a', CONVERT(JSON_OBJECT('k','v') USING latin1))) AS h; +h +7B2261223A207B226B223A202276227D7D +DROP TABLE tc; +# the same wrapper over a column whose stored bytes are not a document +CREATE TABLE tp (j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY(CONVERT(j USING latin1)) AS v, +JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tp; +v ok +[{"a":1,] 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1)) AS v FROM tp; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 20 +SELECT JSON_ARRAY(CAST(j AS CHAR)) AS v FROM tp; +v +["{\"a\":1,"] +DROP TABLE tp; +# +# 7. The metadata the client is told about, which decides whether the +# bytes are converted on the way out. +# +SELECT JSON_ARRAY(1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 40 3 Y 0 39 45 +v +[1] +SELECT JSON_OBJECT('a',1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 72 8 Y 0 39 45 +v +{"a": 1} +SELECT JSON_SET('{}','$.a',1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 68 8 Y 0 39 45 +v +{"a": 1} +SELECT JSON_EXTRACT('{"a":1}','$.a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 56 1 Y 0 39 45 +v +1 +SELECT JSON_UNQUOTE('"a"') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 12 1 Y 128 39 45 +v +a +SELECT JSON_QUOTE('a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 56 3 Y 128 39 45 +v +"a" +SELECT JSON_KEYS('{"a":1}') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 28 5 Y 0 39 45 +v +["a"] +SELECT JSON_COMPACT('{}') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 8 2 Y 0 0 45 +v +{} +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 16777224 5 Y 0 0 45 +v +[1,2] +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 16777216 14 Y 0 0 45 +v +{"1":1, "2":2} +# +# 8. A result delivered in a character set other than the one it was +# computed in. +# +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAY('a') AS v; +v +["a"] +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +["a","b"] +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +{"a":"a", "b":"b"} +SELECT JSON_SET('{}','$.a','b') AS v; +v +{"a": "b"} +SET SESSION character_set_results = DEFAULT; +# The values above are all ASCII, so a conversion on the way out is +# indistinguishable from no conversion. swe7 has no code point for +# the JSON punctuation, so a result that IS converted arrives with +# those characters replaced, and a result that is passed +# unconverted arrives intact. That makes the difference visible. +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('a') AS v; +v +?"a"? +SELECT JSON_OBJECT('a',1) AS v; +v +?"a": 1? +SELECT JSON_SET('{}','$.a','b') AS v; +v +?"a": "b"? +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +v +?1, 2? +SELECT JSON_KEYS('{"a":1}') AS v; +v +?"a"? +SELECT JSON_COMPACT('{"a": 1}') AS v; +v +?"a":1? +SELECT JSON_UNQUOTE('"[a]"') AS v; +v +?a? +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +["a","b"] +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +?"a":"a", "b":"b"? +SET SESSION character_set_results = DEFAULT; +SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_charset.test b/mysql-test/main/func_json_charset.test new file mode 100644 index 0000000000000..99ffd7464432d --- /dev/null +++ b/mysql-test/main/func_json_charset.test @@ -0,0 +1,267 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: character sets. +--echo # +--echo # The same bytes are a different document in a different character +--echo # set, and in some character sets the bytes that JSON uses as +--echo # punctuation are ordinary letters. This test records the character +--echo # set and collation every JSON function reports for its result, what +--echo # each function makes of the same bytes under different labels, and +--echo # what happens when arguments of different character sets meet in one +--echo # call. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. The character set and collation of every result. +--echo # + +SELECT CHARSET(JSON_ARRAY(1)) AS cs, COLLATION(JSON_ARRAY(1)) AS co; +SELECT CHARSET(JSON_OBJECT('a',1)) AS cs, COLLATION(JSON_OBJECT('a',1)) AS co; +SELECT CHARSET(JSON_SET('{}','$.a',1)) AS cs, COLLATION(JSON_SET('{}','$.a',1)) AS co; +SELECT CHARSET(JSON_INSERT('{}','$.a',1)) AS cs; +SELECT CHARSET(JSON_REPLACE('{"a":1}','$.a',2)) AS cs; +SELECT CHARSET(JSON_REMOVE('{"a":1}','$.a')) AS cs; +SELECT CHARSET(JSON_ARRAY_APPEND('[1]','$',2)) AS cs; +SELECT CHARSET(JSON_ARRAY_INSERT('[1]','$[0]',2)) AS cs; +SELECT CHARSET(JSON_MERGE('[1]','[2]')) AS cs; +SELECT CHARSET(JSON_MERGE_PATCH('{"a":1}','{"b":2}')) AS cs; +SELECT CHARSET(JSON_EXTRACT('{"a":1}','$.a')) AS cs; +SELECT CHARSET(JSON_QUERY('{"a":{"b":1}}','$.a')) AS cs; +SELECT CHARSET(JSON_VALUE('{"a":1}','$.a')) AS cs; +SELECT CHARSET(JSON_KEYS('{"a":1}')) AS cs; +SELECT CHARSET(JSON_SEARCH('["x"]','one','x')) AS cs; +SELECT CHARSET(JSON_COMPACT('{}')) AS cs, CHARSET(JSON_LOOSE('{}')) AS cs2; +SELECT CHARSET(JSON_DETAILED('{}')) AS cs; +SELECT CHARSET(JSON_NORMALIZE('{}')) AS cs; +SELECT CHARSET(JSON_TYPE('{}')) AS cs; +--echo # JSON_QUOTE and JSON_UNQUOTE, whose collations differ from the rest +SELECT CHARSET(JSON_QUOTE('a')) AS cs, COLLATION(JSON_QUOTE('a')) AS co; +SELECT CHARSET(JSON_UNQUOTE('"a"')) AS cs, COLLATION(JSON_UNQUOTE('"a"')) AS co; +SELECT COLLATION(CONCAT(JSON_UNQUOTE('"a"'), '')) AS co; +SELECT JSON_UNQUOTE('"A"') = 'a' AS case_sensitive_compare; +SELECT JSON_VALUE('{"a":"A"}','$.a') = 'a' AS case_sensitive_compare; +--echo # the aggregates +SELECT CHARSET(JSON_ARRAYAGG(x)) AS cs, COLLATION(JSON_ARRAYAGG(x)) AS co + FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +SELECT CHARSET(JSON_OBJECTAGG(x,x)) AS cs FROM (SELECT 1 AS x UNION ALL SELECT 2) d; + +--echo # +--echo # 2. The same bytes under different labels. +--echo # + +--echo # 0xFF is one character in latin1 and not a character at all in utf8mb4 +SELECT JSON_VALID(_latin1 X'227B7DFF22') AS as_latin1; +SELECT JSON_VALID(_binary X'227B7DFF22') AS as_binary; +SELECT HEX(JSON_UNQUOTE(_latin1 X'22FF22')) AS unq_latin1; +SELECT HEX(JSON_UNQUOTE(_binary X'22FF22')) AS unq_binary; +--echo # the byte 0x5C is a backslash in latin1 and utf8mb4 +SELECT JSON_VALID(_latin1 X'225C6E22') AS as_latin1; +SELECT JSON_VALID(_utf8mb4 X'225C6E22') AS as_utf8mb4; +--echo # a two-byte character that ends in a byte JSON treats as punctuation +SELECT JSON_VALID(CONVERT(_latin1 X'227B7D22' USING utf8mb4)) AS v; + +--echo # +--echo # 3. A character set in which the JSON punctuation bytes are letters. +--echo # In swe7 the bytes 5B 5C 5D 7B 7D are national characters, so the +--echo # same bytes that encode an object elsewhere encode a word here. +--echo # + +SELECT JSON_VALID(_swe7 X'7B7D') AS braces_as_swe7; +SELECT JSON_VALID(_swe7 X'5B5D') AS brackets_as_swe7; +SELECT JSON_VALID(_binary X'7B7D') AS braces_as_binary; +SELECT HEX(CONVERT(_swe7 X'7B7D' USING utf8mb4)) AS swe7_to_utf8mb4; +SELECT HEX(JSON_ARRAY(_swe7 X'7B7D')) AS embedded; +SELECT HEX(JSON_QUOTE(_swe7 X'7B7D')) AS quoted; +SELECT JSON_VALID(JSON_ARRAY(_swe7 X'7B7D')) AS still_valid; +--echo # a document written in swe7 itself +SELECT HEX(JSON_COMPACT(CONVERT('{"a":1}' USING swe7))) AS as_swe7; +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS as_swe7; +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS cs; +--echo # a swe7 value spliced into a utf8mb4 document +SELECT HEX(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS spliced; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS ok; + +--echo # +--echo # 4. Wide character sets, where a character is not a byte. +--echo # + +SELECT HEX(CONVERT('{"a":1}' USING ucs2)) AS doc_ucs2; +SELECT JSON_VALID(CONVERT('{"a":1}' USING ucs2)) AS v; +SELECT JSON_DEPTH(CONVERT('{"a":{"b":1}}' USING ucs2)) AS d; +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS compacted; +SELECT CHARSET(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS cs; +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS setv; +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS cs; +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), '$.a') AS v; +SELECT HEX(JSON_ARRAY(CONVERT('x' USING ucs2))) AS arr; +SELECT CHARSET(JSON_ARRAY(CONVERT('x' USING ucs2))) AS cs; +--echo # utf16 and utf32 +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf16)) AS v16; +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf32)) AS v32; +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING utf16))) AS c16; + +--echo # +--echo # 5. Arguments of different character sets meeting in one call. +--echo # + +SELECT CHARSET(JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4))) AS cs; +SELECT JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4)) AS v; +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', + CONVERT('x' USING utf8mb4))) AS cs; +SELECT JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', + CONVERT('x' USING utf8mb4)) AS v; +--echo # a ucs2 document with a utf8mb4 value spliced into it +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + CONVERT('x' USING utf8mb4))) AS cs; +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + CONVERT('x' USING utf8mb4))) AS v; +--echo # a utf8mb4 document with a ucs2 value spliced into it +SELECT CHARSET(JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2))) AS cs; +SELECT JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2)) AS v; +--echo # a path argument in another character set than the document +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING latin1)) AS v; +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING ucs2)) AS v; +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), CONVERT('$.a' USING latin1)) AS v; + +--echo # 5a. A value that is itself a document, spliced into a document of +--echo # another character set. The fragment is written in the character +--echo # set of the answer, and what differs between the functions is how +--echo # that set is chosen: a mutator takes it from its document argument +--echo # and converts the fragment into it, while a constructor aggregates +--echo # over all of its arguments, so the fragment itself can decide the +--echo # set and its bytes then go across unchanged. +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS typed_into_ucs2; +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS ok; +SELECT HEX(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS latin1_typed; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS ok; +--echo # the constructors take the same fragment without complaint +SELECT HEX(JSON_ARRAY( + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS embedded; +SELECT JSON_VALID(JSON_ARRAY( + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS still_valid; +SELECT CHARSET(JSON_ARRAY(JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) AS cs; +SELECT HEX(JSON_OBJECT('k', JSON_COMPACT(CONVERT('{"x":2}' USING ucs2)))) AS obj_embedded; + +--echo # +--echo # 6. Binary arguments, which are not text and are copied through. +--echo # + +CREATE TABLE t1 (b BLOB, v VARBINARY(10), c VARCHAR(10) CHARACTER SET latin1); +INSERT INTO t1 VALUES (X'FF', X'FF', X'FF'), (X'C3', X'C3', X'C3'); +SELECT CHARSET(JSON_ARRAY(b)) AS cs, COLLATION(JSON_ARRAY(b)) AS co FROM t1 LIMIT 1; +SELECT HEX(JSON_ARRAY(b)) AS h, JSON_VALID(JSON_ARRAY(b)) AS ok FROM t1; +SELECT HEX(JSON_ARRAY(v)) AS h, JSON_VALID(JSON_ARRAY(v)) AS ok FROM t1; +SELECT HEX(JSON_ARRAY(c)) AS h, JSON_VALID(JSON_ARRAY(c)) AS ok FROM t1; +SELECT CHARSET(JSON_ARRAY(c)) AS cs FROM t1 LIMIT 1; +SELECT HEX(JSON_OBJECT('k', b)) AS h FROM t1; +SELECT HEX(JSON_SET('{}', '$.a', b)) AS h FROM t1; +SELECT CHARSET(JSON_SET('{}', '$.a', b)) AS cs FROM t1 LIMIT 1; +SELECT HEX(JSON_QUOTE(b)) AS h FROM t1; +--echo # the raw bytes survive a round trip through a document +SELECT HEX(JSON_VALUE(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +SELECT HEX(JSON_EXTRACT(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +DROP TABLE t1; + +--echo # 6a. A binary column holding a whole document, with bytes that are +--echo # not valid utf8mb4 sitting in a value, in a key, and in the +--echo # structural skeleton. Nothing converts these, so what the +--echo # parser accepts is decided by the binary label alone. +CREATE TABLE t2 (b VARBINARY(60)); +INSERT INTO t2 VALUES + (X'7B2261223A22FF227D'), (X'7B22FF61223A317D'), (X'7B2261223A31FF7D'), + (X'7B2261223A22C3227D'), (X'7B2261223A22C3A9227D'); +SELECT HEX(b) AS stored, JSON_VALID(b) AS v FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_EXTRACT(b, '$.a')) AS ext FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_VALUE(b, '$.a')) AS val FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_KEYS(b)) AS ks FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_SET(b, '$.z', 1)) AS mutated FROM t2 ORDER BY stored; +SELECT CHARSET(JSON_EXTRACT(b, '$.a')) AS cs_ext, + CHARSET(JSON_SET(b, '$.z', 1)) AS cs_set FROM t2 LIMIT 1; +DROP TABLE t2; + +--echo # +--echo # 6b. CONVERT and CAST over a value that is already a document. +--echo # CONVERT wraps the value and the wrapper is seen through, so the +--echo # result is still treated as a document; CAST does not. +--echo # +CREATE TABLE tc (j JSON); +INSERT INTO tc VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +SELECT HEX(JSON_ARRAY(CONVERT(j USING latin1))) AS h, + CHARSET(JSON_ARRAY(CONVERT(j USING latin1))) AS cs FROM tc; +SELECT JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tc; +--echo # a mutator takes the same fragment and writes it back out in the +--echo # character set of its document argument, which is where the answer's +--echo # character set comes from +SELECT HEX(JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1))) AS h FROM tc; +SELECT HEX(JSON_ARRAY(CONVERT(j USING binary))) AS h FROM tc; +--echo # CAST produces a plain string, so it is quoted rather than embedded +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR))) AS h FROM tc; +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR CHARACTER SET latin1))) AS h FROM tc; +SELECT HEX(JSON_ARRAY(CONVERT(JSON_OBJECT('a', _utf8mb4 X'C3A9') USING latin1))) AS h; +SELECT HEX(JSON_SET('{}', '$.a', CONVERT(JSON_OBJECT('k','v') USING latin1))) AS h; +DROP TABLE tc; +--echo # the same wrapper over a column whose stored bytes are not a document +CREATE TABLE tp (j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY(CONVERT(j USING latin1)) AS v, + JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tp; +SELECT JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1)) AS v FROM tp; +SELECT JSON_ARRAY(CAST(j AS CHAR)) AS v FROM tp; +DROP TABLE tp; + +--echo # +--echo # 7. The metadata the client is told about, which decides whether the +--echo # bytes are converted on the way out. +--echo # + +--enable_metadata +SELECT JSON_ARRAY(1) AS v; +SELECT JSON_OBJECT('a',1) AS v; +SELECT JSON_SET('{}','$.a',1) AS v; +SELECT JSON_EXTRACT('{"a":1}','$.a') AS v; +SELECT JSON_UNQUOTE('"a"') AS v; +SELECT JSON_QUOTE('a') AS v; +SELECT JSON_KEYS('{"a":1}') AS v; +SELECT JSON_COMPACT('{}') AS v; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +--disable_metadata + +--echo # +--echo # 8. A result delivered in a character set other than the one it was +--echo # computed in. +--echo # + +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAY('a') AS v; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SELECT JSON_SET('{}','$.a','b') AS v; +SET SESSION character_set_results = DEFAULT; + +--echo # The values above are all ASCII, so a conversion on the way out is +--echo # indistinguishable from no conversion. swe7 has no code point for +--echo # the JSON punctuation, so a result that IS converted arrives with +--echo # those characters replaced, and a result that is passed +--echo # unconverted arrives intact. That makes the difference visible. +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('a') AS v; +SELECT JSON_OBJECT('a',1) AS v; +SELECT JSON_SET('{}','$.a','b') AS v; +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +SELECT JSON_KEYS('{"a":1}') AS v; +SELECT JSON_COMPACT('{"a": 1}') AS v; +SELECT JSON_UNQUOTE('"[a]"') AS v; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SET SESSION character_set_results = DEFAULT; +SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_columns.result b/mysql-test/main/func_json_columns.result new file mode 100644 index 0000000000000..25a1aec253f8d --- /dev/null +++ b/mysql-test/main/func_json_columns.result @@ -0,0 +1,936 @@ +# +# Behavioral baseline: JSON values that come out of a table. +# +# A JSON column carries a check constraint that is supposed to keep +# invalid documents out, but the check can be switched off for the +# duration of a statement, after which the column holds bytes that no +# longer satisfy it. This test records what every JSON function does +# with such a column, what the check constraints look like, and how a +# user-created temporary table compares with a base table. +# +SET NAMES utf8mb4; +# +# 1. What declaring a column JSON actually creates. +# +CREATE TABLE t1 (id INT, j JSON); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT column_name, data_type, column_type +FROM information_schema.columns +WHERE table_schema = 'test' AND table_name = 't1' + ORDER BY ordinal_position; +column_name data_type column_type +id int int(11) +j longtext longtext +SELECT constraint_name, check_clause +FROM information_schema.check_constraints +WHERE constraint_schema = 'test' AND table_name = 't1'; +constraint_name check_clause +j json_valid(`j`) +# a column that is only LONGTEXT does not get the constraint +CREATE TABLE t2 (id INT, j LONGTEXT); +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT COUNT(*) AS constraints +FROM information_schema.check_constraints +WHERE constraint_schema = 'test' AND table_name = 't2'; +constraints +0 +# a constraint written out by hand, worded exactly like the automatic one +CREATE TABLE t3 (id INT, j LONGTEXT CHECK (json_valid(j))); +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# a constraint that mentions the column inside a larger expression +CREATE TABLE t4 (id INT, j LONGTEXT CHECK (json_valid(j) AND id > 0)); +SHOW CREATE TABLE t4; +Table Create Table +t4 CREATE TABLE `t4` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL CHECK (json_valid(`j`) and `id` > 0) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# a table level constraint over two columns +CREATE TABLE t5 (id INT, j LONGTEXT, k LONGTEXT, +CONSTRAINT both_valid CHECK (json_valid(j) AND json_valid(k))); +SHOW CREATE TABLE t5; +Table Create Table +t5 CREATE TABLE `t5` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL, + `k` longtext DEFAULT NULL, + CONSTRAINT `both_valid` CHECK (json_valid(`j`) and json_valid(`k`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# +# 2. The constraint doing its job. +# +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'); +INSERT INTO t1 VALUES (2, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t1 VALUES (3, '{"a":1,'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (3, 'not json at all'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1 SET j = '{' WHERE id = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1,"b":[1,2]} +2 {"a": 1, "b": [1, 2]} +# the hand written constraint behaves the same way +INSERT INTO t3 VALUES (1, '{'); +ERROR 23000: CONSTRAINT `t3.j` failed for `test`.`t3` +INSERT INTO t3 VALUES (1, '{"a":1}'); +SELECT id, j FROM t3; +id j +1 {"a":1} +# the constraint shapes that are NOT a bare check of one column. Each +# of these still has to run in full, so what they accept and reject is +# recorded here rather than left to the CREATE statement alone. +# constraint on the column combined with a test of another column +INSERT INTO t4 VALUES (1, '{'); +ERROR 23000: CONSTRAINT `t4.j` failed for `test`.`t4` +INSERT INTO t4 VALUES (0, '{"a":1}'); +ERROR 23000: CONSTRAINT `t4.j` failed for `test`.`t4` +INSERT INTO t4 VALUES (1, '{"a":1}'); +SELECT id, j FROM t4; +id j +1 {"a":1} +# table level constraint over two columns +INSERT INTO t5 VALUES (1, '{', '{"a":1}'); +ERROR 23000: CONSTRAINT `both_valid` failed for `test`.`t5` +INSERT INTO t5 VALUES (1, '{"a":1}', '{'); +ERROR 23000: CONSTRAINT `both_valid` failed for `test`.`t5` +INSERT INTO t5 VALUES (1, '{"a":1}', '{"b":2}'); +SELECT id, j, k FROM t5; +id j k +1 {"a":1} {"b":2} +# an OR, where the other side can let an invalid document through +CREATE TABLE t5b (id INT, j LONGTEXT CHECK (json_valid(j) OR id > 0)); +INSERT INTO t5b VALUES (1, '{'); +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +INSERT INTO t5b VALUES (0, '{'); +ERROR 23000: CONSTRAINT `t5b.j` failed for `test`.`t5b` +SELECT id, j, JSON_VALID(j) AS valid FROM t5b; +id j valid +1 { 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# a constraint attached to one column that tests a different column +CREATE TABLE t5c (a LONGTEXT, b LONGTEXT CHECK (json_valid(a))); +INSERT INTO t5c VALUES ('{"a":1}', '{'); +INSERT INTO t5c VALUES ('{', '{"a":1}'); +ERROR 23000: CONSTRAINT `t5c.b` failed for `test`.`t5c` +SELECT a, b FROM t5c; +a b +{"a":1} { +# writing any constraint of your own on a JSON column replaces the +# automatic one, so invalid documents go in with the checks left on +CREATE TABLE t5e (j JSON CHECK (LENGTH(j) > 0)); +SHOW CREATE TABLE t5e; +Table Create Table +t5e CREATE TABLE `t5e` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (octet_length(`j`) > 0) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t5e VALUES ('{"a":1}'); +INSERT INTO t5e VALUES ('{'); +INSERT INTO t5e VALUES (''); +ERROR 23000: CONSTRAINT `t5e.j` failed for `test`.`t5e` +SELECT j, JSON_VALID(j) AS valid FROM t5e ORDER BY j; +j valid +{ 0 +{"a":1} 1 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS still_valid +FROM t5e ORDER BY j; +v still_valid +["{"] 1 +["{\"a\":1}"] 1 +DROP TABLE t5b, t5c, t5e; +# +# 3. Switching the check off, which is what makes a stored document +# unverifiable afterwards. +# +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,'); +INSERT INTO t1 VALUES (11, 'not json at all'); +INSERT INTO t1 VALUES (12, '{"a":1} trailing'); +INSERT INTO t1 VALUES (13, CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32))); +INSERT INTO t3 VALUES (10, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +# the rows are there and the constraint is back on +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1,"b":[1,2]} +2 {"a": 1, "b": [1, 2]} +10 {"a":1, +11 not json at all +12 {"a":1} trailing +13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT id, JSON_VALID(j) AS valid FROM t1 ORDER BY id; +id valid +1 1 +2 1 +10 0 +11 0 +12 0 +13 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +# a new statement is checked again, the stored rows are not +INSERT INTO t1 VALUES (14, '{'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +# reading the column back does not check anything +SELECT id, j FROM t1 WHERE id >= 10 ORDER BY id; +id j +10 {"a":1, +11 not json at all +12 {"a":1} trailing +13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT id, HEX(j) FROM t1 WHERE id = 10; +id HEX(j) +10 7B2261223A312C +# +# 4. Every function fed the column that no longer satisfies its check. +# +SELECT id, JSON_VALID(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 0 +11 0 +12 0 +13 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT id, JSON_TYPE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_type' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_type' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_type' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_type' at position 32 +SELECT id, JSON_DEPTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_depth' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_depth' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_depth' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +SELECT id, JSON_LENGTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_length' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_length' at position 32 +SELECT id, JSON_KEYS(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 ["a"] +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 1 +SELECT id, JSON_EXTRACT(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_extract' at position 32 +SELECT id, JSON_QUERY(j, '$') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 {"a":1} +13 NULL +SELECT id, JSON_VALUE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 1 +11 NULL +12 1 +13 NULL +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 16 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 32 +SELECT id, JSON_INSERT(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_insert' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 16 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 32 +SELECT id, JSON_REPLACE(j, '$.a', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_replace' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_replace' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_replace' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 32 +SELECT id, JSON_REMOVE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_remove' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 4 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_remove' at position 32 +SELECT id, JSON_ARRAY_APPEND(j, '$', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array_append' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 14 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +SELECT id, JSON_MERGE(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 {"a": 1, "z": 1} +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 32 +SELECT id, JSON_MERGE_PATCH(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 {"a": 1, "z": 1} +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_patch' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_patch' at position 32 +SELECT id, JSON_COMPACT(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_compact' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_compact' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_compact' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_compact' at position 32 +SELECT id, JSON_LOOSE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_loose' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_loose' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_loose' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_loose' at position 32 +SELECT id, JSON_NORMALIZE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_normalize' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_normalize' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_normalize' at position 32 +SELECT id, JSON_EXISTS(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 1 +11 NULL +12 1 +13 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_exists' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_exists' at position 32 +SELECT id, JSON_CONTAINS(j, '1', '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 1 +11 NULL +12 1 +13 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_contains' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains' at position 32 +SELECT id, JSON_SEARCH(j, 'one', '1') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 "$.a" +11 NULL +12 "$.a" +13 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_search' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_search' at position 32 +SELECT id, JSON_EQUALS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_equals' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_equals' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_equals' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_equals' at position 32 +# id 13, an array nested past the scanner limit, is left out here: +# JSON_OVERLAPS crashes the server on such an argument, so the row is +# excluded until that is fixed. Every other function above takes it. +SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v +FROM t1 WHERE id >= 10 AND id <> 13 ORDER BY id; +id v +10 0 +11 0 +12 1 +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_overlaps' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 1 +# +# 5. The column as an embedded argument. A JSON column is embedded as +# a document, so whatever it holds is copied into the result as is. +# +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 [{"a":1,] +11 [not json at all] +12 [{"a":1} trailing] +13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; +id still_valid +10 0 +11 0 +12 0 +13 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 {"k": {"a":1,} +11 {"k": not json at all} +12 {"k": {"a":1} trailing} +13 {"k": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS still_valid +FROM t1 WHERE id >= 10 ORDER BY id; +id still_valid +10 0 +11 0 +12 0 +13 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 15 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 37 +SELECT id, JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +id v +10 [{"a":1,] +11 [not json at all] +12 [{"a":1} trailing] +13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT id, JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +id v +10 {"10":{"a":1,} +11 {"11":not json at all} +12 {"12":{"a":1} trailing} +13 {"13":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} +# the same column read as plain text is quoted instead of embedded +SELECT id, JSON_ARRAY(CONCAT(j, '')) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 ["{\"a\":1,"] +11 ["not json at all"] +12 ["{\"a\":1} trailing"] +13 ["[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]"] +SELECT id, JSON_VALID(JSON_ARRAY(CONCAT(j, ''))) AS still_valid +FROM t1 WHERE id >= 10 ORDER BY id; +id still_valid +10 1 +11 1 +12 1 +13 1 +# a LONGTEXT column holding the same bytes is quoted, not embedded +SELECT id, JSON_ARRAY(j) AS v FROM t3 ORDER BY id; +id v +1 [{"a":1}] +10 [{"a":1,] +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t3 ORDER BY id; +id still_valid +1 1 +10 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +# 5a. The same column reached through a scalar subquery, where the +# value is fetched once and held rather than read per row. +CREATE TABLE ts (id INT, j JSON); +INSERT INTO ts VALUES (1, '{"a": 1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ts VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 1)) AS v; +v +[{"a": 1}] +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 1))) AS ok; +ok +1 +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +v_poisoned +[{"a":1,] +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 2))) AS ok_poisoned; +ok_poisoned +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 1)) AS v; +v +{"k": {"a": 1}} +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +v_poisoned +{"k": {"a":1,} +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 1)) AS v; +v +{"d": {"a": 1}} +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +v_poisoned +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 13 +DROP TABLE ts; +# 5b. The same column reached through a view. A merged view reads the +# table directly; a materialised one copies the value into a +# temporary table first. +CREATE TABLE tv (id INT, j JSON); +INSERT INTO tv VALUES (1, '{"a": 1, "b": [1, 2]}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tv VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +CREATE ALGORITHM=MERGE VIEW v_merge AS SELECT id, j FROM tv; +CREATE ALGORITHM=TEMPTABLE VIEW v_tmp AS SELECT id, j FROM tv; +SHOW CREATE VIEW v_merge; +View Create View character_set_client collation_connection +v_merge CREATE ALGORITHM=MERGE DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v_merge` AS select `tv`.`id` AS `id`,`tv`.`j` AS `j` from `tv` utf8mb4 utf8mb4_general_ci +SHOW CREATE VIEW v_tmp; +View Create View character_set_client collation_connection +v_tmp CREATE ALGORITHM=TEMPTABLE DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v_tmp` AS select `tv`.`id` AS `id`,`tv`.`j` AS `j` from `tv` utf8mb4 utf8mb4_general_ci +SELECT id, j, HEX(j) FROM v_merge ORDER BY id; +id j HEX(j) +1 {"a": 1, "b": [1, 2]} 7B2261223A20312C202262223A205B312C20325D7D +2 {"a":1, 7B2261223A312C +SELECT id, j, HEX(j) FROM v_tmp ORDER BY id; +id j HEX(j) +1 {"a": 1, "b": [1, 2]} 7B2261223A20312C202262223A205B312C20325D7D +2 {"a":1, 7B2261223A312C +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok +FROM v_merge ORDER BY id; +id v ok +1 [{"a": 1, "b": [1, 2]}] 1 +2 [{"a":1,] 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok +FROM v_tmp ORDER BY id; +id v ok +1 [{"a": 1, "b": [1, 2]}] 1 +2 [{"a":1,] 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_merge ORDER BY id; +id v +1 {"a": 1, "b": [1, 2], "z": 1} +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_tmp ORDER BY id; +id v +1 {"a": 1, "b": [1, 2], "z": 1} +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT JSON_ARRAYAGG(j) AS v FROM v_merge WHERE id = 1; +v +[{"a": 1, "b": [1, 2]}] +DROP VIEW v_merge, v_tmp; +DROP TABLE tv; +# +# 6. A valid column, for comparison: the same calls with nothing wrong. +# +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id < 10 ORDER BY id; +id v +1 [{"a":1,"b":[1,2]}] +2 [{"a": 1, "b": [1, 2]}] +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id < 10 ORDER BY id; +id v +1 {"a": 1, "b": [1, 2], "z": 1} +2 {"a": 1, "b": [1, 2], "z": 1} +SELECT id, JSON_EXTRACT(j, '$.b') AS v FROM t1 WHERE id < 10 ORDER BY id; +id v +1 [1, 2] +2 [1, 2] +# a column value keeps the formatting it was stored with +SELECT id, HEX(j) FROM t1 WHERE id < 10 ORDER BY id; +id HEX(j) +1 7B2261223A312C2262223A5B312C325D7D +2 7B2261223A20312C202262223A205B312C20325D7D +# 6a. The same document stored in every column type it can live in. +# Retrieval returns the stored bytes unchanged in all of them; +# only the JSON-typed column is embedded rather than quoted. +CREATE TABLE tr (jj JSON, lt LONGTEXT CHECK (JSON_VALID(lt)), vc VARCHAR(64), +vb VARBINARY(64), bl BLOB); +INSERT INTO tr VALUES ('{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', +'{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', +'{"a": 1, "b":[1, 2]}'); +SELECT HEX(jj) AS h_json, HEX(lt) AS h_longtext, HEX(vc) AS h_varchar, +HEX(vb) AS h_varbinary, HEX(bl) AS h_blob FROM tr; +h_json h_longtext h_varchar h_varbinary h_blob +7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D +SELECT jj, lt, vc, vb, bl FROM tr; +jj lt vc vb bl +{"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} +SELECT JSON_ARRAY(jj) AS a_json, JSON_ARRAY(vc) AS a_varchar, +JSON_ARRAY(vb) AS a_varbinary, JSON_ARRAY(bl) AS a_blob FROM tr; +a_json a_varchar a_varbinary a_blob +[{"a": 1, "b":[1, 2]}] ["{\"a\": 1, \"b\":[1, 2]}"] ["{\"a\": 1, \"b\":[1, 2]}"] ["{\"a\": 1, \"b\":[1, 2]}"] +DROP TABLE tr; +# +# 7. Storing a JSON function result back into a column, which is where +# the check constraint runs again. +# +CREATE TABLE t6 (j JSON); +INSERT INTO t6 SELECT JSON_SET('{"a":1}', '$.b', 2); +INSERT INTO t6 SELECT JSON_ARRAY(1,2); +INSERT INTO t6 SELECT JSON_OBJECT('a',1); +SELECT j FROM t6; +j +{"a": 1, "b": 2} +[1, 2] +{"a": 1} +# a value the constructor built that the check does not accept +CREATE TABLE t7 (j JSON); +INSERT INTO t7 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +ERROR 23000: CONSTRAINT `t7.j` failed for `test`.`t7` +SELECT COUNT(*) AS stored FROM t7; +stored +0 +# the same value stored into a plain LONGTEXT column +CREATE TABLE t8 (j LONGTEXT); +INSERT INTO t8 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +SELECT JSON_VALID(j) AS v, LENGTH(j) AS len FROM t8; +v len +0 65 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +# a value that is truncated on the way into the column +CREATE TABLE t9 (j VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t9 SELECT JSON_SET('{"a":1,"b":2}', '$.a', 9); +Warnings: +Warning 1265 Data truncated for column 'j' at row 1 +SELECT j, JSON_VALID(j) AS still_valid FROM t9; +j still_valid +{"a": 9, " 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode=DEFAULT; +DROP TABLE t6, t7, t8, t9; +# 7a. The same truncating store into a column that IS checked. The +# chop happens first and the constraint then judges the chopped +# bytes, which sometimes still parse and sometimes do not. +CREATE TABLE tn (v VARCHAR(10) CHECK (json_valid(v))); +SET @@sql_mode=''; +# chopped into an unparseable prefix: rejected +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +ERROR 23000: CONSTRAINT `tn.v` failed for `test`.`tn` +# chopped on a boundary that still parses: accepted, with a warning +INSERT INTO tn VALUES (CONCAT('[1,2,3,45]', ',6789')); +Warnings: +Warning 1265 Data truncated for column 'v' at row 1 +INSERT INTO tn VALUES ('{"a":1}'); +SELECT v, JSON_VALID(v) AS valid FROM tn ORDER BY v; +v valid +[1,2,3,45] 1 +{"a":1} 1 +SET @@sql_mode=DEFAULT; +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +ERROR 22001: Data too long for column 'v' at row 1 +SELECT COUNT(*) AS stored FROM tn; +stored +2 +DROP TABLE tn; +# 7b. A store that converts the character set on the way in. The +# constraint judges the converted bytes, not the ones the +# function produced. +# swe7 cannot carry the JSON punctuation at all +CREATE TABLE t11 (j LONGTEXT CHARACTER SET swe7 CHECK (json_valid(j))); +SET @@sql_mode=''; +INSERT INTO t11 VALUES ('{"a":1}'); +ERROR 23000: CONSTRAINT `t11.j` failed for `test`.`t11` +SELECT COUNT(*) AS stored FROM t11; +stored +0 +# the same column without a constraint keeps the damaged bytes +CREATE TABLE t12 (j LONGTEXT CHARACTER SET swe7); +INSERT INTO t12 VALUES ('{"a":1}'); +Warnings: +Warning 1366 Incorrect string value: '{"a":1...' for column `test`.`t12`.`j` at row 1 +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t12; +stored v +3F2261223A313F 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SET @@sql_mode=DEFAULT; +INSERT INTO t11 VALUES ('{"a":1}'); +ERROR 22007: Incorrect string value: '{"a":1...' for column `test`.`t11`.`j` at row 1 +# a conversion that loses nothing still changes the bytes +CREATE TABLE t13 (j VARCHAR(100) CHARACTER SET latin1 CHECK (json_valid(j))); +INSERT INTO t13 VALUES (JSON_OBJECT('a', _utf8mb4 X'C3A9')); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t13; +stored v +7B2261223A2022E9227D 1 +CREATE TABLE tu (j JSON); +INSERT INTO tu VALUES ('{"a":1}'); +INSERT INTO t13 SELECT JSON_SET(j, '$.c', 3) FROM tu; +SELECT HEX(j) FROM t13 ORDER BY j; +HEX(j) +7B2261223A2022E9227D +7B2261223A20312C202263223A20337D +DROP TABLE t11, t12, t13, tu; +# 7c. A table rebuilt by ALTER, which checks every existing row again +# rather than the value of one statement. +CREATE TABLE ta (id INT, j JSON); +INSERT INTO ta VALUES (1, '{"a":1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ta VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +ALTER TABLE ta FORCE, ALGORITHM=COPY; +ERROR 23000: CONSTRAINT `ta.j` failed for `test`.`ta` +ALTER IGNORE TABLE ta FORCE, ALGORITHM=COPY; +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `ta.j` failed for `test`.`ta` +SELECT id, j FROM ta ORDER BY id; +id j +1 {"a":1} +DROP TABLE ta; +# ALTER that converts the character set rewrites the stored bytes +CREATE TABLE tb (j JSON); +INSERT INTO tb VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +ALTER TABLE tb CONVERT TO CHARACTER SET latin1; +SHOW CREATE TABLE tb; +Table Create Table +tb CREATE TABLE `tb` ( + `j` longtext DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT HEX(j) AS stored, JSON_VALID(j) AS valid FROM tb; +stored valid +7B2261223A20312C202262223A2022E9227D 1 +SELECT HEX(JSON_ARRAY(j)) AS h FROM tb; +h +5B7B2261223A20312C202262223A2022E9227D5D +DROP TABLE tb; +# 7d. Generated columns, where the JSON function result is computed +# and stored by the server rather than supplied by the statement. +CREATE TABLE tg (j JSON, +c1 LONGTEXT AS (JSON_SET(j, '$.z', 9)) PERSISTENT, +c2 VARCHAR(20) AS (JSON_EXTRACT(j, '$.a')) PERSISTENT, KEY (c2)); +SHOW CREATE TABLE tg; +Table Create Table +tg CREATE TABLE `tg` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `c1` longtext GENERATED ALWAYS AS (json_set(`j`,'$.z',9)) STORED, + `c2` varchar(20) GENERATED ALWAYS AS (json_extract(`j`,'$.a')) STORED, + KEY `c2` (`c2`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO tg (j) VALUES ('{"a": 1, "b": 2}'); +SELECT j, c1, HEX(c1), c2 FROM tg; +j c1 HEX(c1) c2 +{"a": 1, "b": 2} {"a": 1, "b": 2, "z": 9} 7B2261223A20312C202262223A20322C20227A223A20397D 1 +UPDATE tg SET j = '{"a": 7}'; +SELECT j, c1, HEX(c1), c2 FROM tg; +j c1 HEX(c1) c2 +{"a": 7} {"a": 7, "z": 9} 7B2261223A20372C20227A223A20397D 7 +SELECT c2 FROM tg FORCE INDEX (c2) WHERE c2 = '7'; +c2 +7 +DROP TABLE tg; +# +# 8. A temporary table the user created. It is under user control the +# same way a base table is, including the check switch. +# +CREATE TEMPORARY TABLE tmp1 (id INT, j JSON); +SHOW CREATE TABLE tmp1; +Table Create Table +tmp1 CREATE TEMPORARY TABLE `tmp1` ( + `id` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO tmp1 VALUES (1, '{"a":1}'); +INSERT INTO tmp1 VALUES (2, '{'); +ERROR 23000: CONSTRAINT `tmp1.j` failed for `test`.`tmp1` +SET SESSION check_constraint_checks = OFF; +INSERT INTO tmp1 VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS valid FROM tmp1 ORDER BY id; +id j valid +1 {"a":1} 1 +2 {"a":1, 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT id, JSON_ARRAY(j) AS v FROM tmp1 ORDER BY id; +id v +1 [{"a":1}] +2 [{"a":1,] +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM tmp1 ORDER BY id; +id still_valid +1 1 +2 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM tmp1 ORDER BY id; +id v +1 {"a": 1, "z": 1} +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +DROP TEMPORARY TABLE tmp1; +# +# 9. Values passing through a temporary table the server made for +# itself: grouping, materialised derived tables, union branches and +# window frames. +# +CREATE TABLE t10 (g INT, j JSON, txt VARCHAR(30)); +INSERT INTO t10 VALUES +(1, '{"a":1}', '{"a":1}'), (1, '{"a":2}', '{"a":2}'), +(2, '{"b":1}', '{"b":1}'); +# grouping over a JSON expression +SELECT g, JSON_ARRAYAGG(JSON_SET(j, '$.z', 1)) AS v FROM t10 GROUP BY g ORDER BY g; +g v +1 [{"a": 1, "z": 1},{"a": 2, "z": 1}] +2 [{"b": 1, "z": 1}] +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j ORDER BY j; +v +{"a": 1, "z": 1} +{"a": 2, "z": 1} +{"b": 1, "z": 1} +SELECT g, COUNT(*) AS c FROM t10 GROUP BY g, JSON_SET(j, '$.z', 1) ORDER BY g, c; +g c +1 1 +1 1 +2 1 +# a materialised derived table +SELECT JSON_VALID(v) AS valid, v +FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d +ORDER BY v; +valid v +1 {"a": 1, "z": 1} +1 {"a": 2, "z": 1} +1 {"b": 1, "z": 1} +SELECT JSON_SET(v, '$.y', 2) AS w +FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d +ORDER BY w; +w +{"a": 1, "z": 1, "y": 2} +{"a": 2, "z": 1, "y": 2} +{"b": 1, "z": 1, "y": 2} +SELECT JSON_ARRAY(v) AS w +FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d +ORDER BY w; +w +[{"a": 1, "z": 1}] +[{"a": 2, "z": 1}] +[{"b": 1, "z": 1}] +# union branches, where one branch is a document and the other is text +SELECT v, JSON_VALID(v) AS valid FROM ( +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 +UNION +SELECT txt FROM t10 WHERE g = 2) u ORDER BY v; +v valid +{"a": 1, "z": 1} 1 +{"a": 2, "z": 1} 1 +{"b":1} 1 +SELECT JSON_ARRAY(v) AS w FROM ( +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 +UNION +SELECT txt FROM t10 WHERE g = 2) u ORDER BY w; +w +["{\"a\": 1, \"z\": 1}"] +["{\"a\": 2, \"z\": 1}"] +["{\"b\":1}"] +# a window frame, which materialises its input +SELECT g, JSON_SET(j, '$.z', 1) AS v, +ROW_NUMBER() OVER (PARTITION BY g ORDER BY j) AS rn +FROM t10 ORDER BY g, rn; +g v rn +1 {"a": 1, "z": 1} 1 +1 {"a": 2, "z": 1} 2 +2 {"b": 1, "z": 1} 1 +SELECT g, JSON_ARRAY(JSON_SET(j, '$.z', 1)) AS v, +COUNT(*) OVER (PARTITION BY g) AS c +FROM t10 ORDER BY g, v; +g v c +1 [{"a": 1, "z": 1}] 2 +1 [{"a": 2, "z": 1}] 2 +2 [{"b": 1, "z": 1}] 1 +# an ORDER BY that has to materialise the value first +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 ORDER BY JSON_EXTRACT(j, '$.a'), v; +v +{"b": 1, "z": 1} +{"a": 1, "z": 1} +{"a": 2, "z": 1} +DROP TABLE t1, t2, t3, t4, t5, t10; diff --git a/mysql-test/main/func_json_columns.test b/mysql-test/main/func_json_columns.test new file mode 100644 index 0000000000000..05dee4d531fe0 --- /dev/null +++ b/mysql-test/main/func_json_columns.test @@ -0,0 +1,422 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: JSON values that come out of a table. +--echo # +--echo # A JSON column carries a check constraint that is supposed to keep +--echo # invalid documents out, but the check can be switched off for the +--echo # duration of a statement, after which the column holds bytes that no +--echo # longer satisfy it. This test records what every JSON function does +--echo # with such a column, what the check constraints look like, and how a +--echo # user-created temporary table compares with a base table. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. What declaring a column JSON actually creates. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +SHOW CREATE TABLE t1; +SELECT column_name, data_type, column_type + FROM information_schema.columns + WHERE table_schema = 'test' AND table_name = 't1' + ORDER BY ordinal_position; +SELECT constraint_name, check_clause + FROM information_schema.check_constraints + WHERE constraint_schema = 'test' AND table_name = 't1'; + +--echo # a column that is only LONGTEXT does not get the constraint +CREATE TABLE t2 (id INT, j LONGTEXT); +SHOW CREATE TABLE t2; +SELECT COUNT(*) AS constraints + FROM information_schema.check_constraints + WHERE constraint_schema = 'test' AND table_name = 't2'; + +--echo # a constraint written out by hand, worded exactly like the automatic one +CREATE TABLE t3 (id INT, j LONGTEXT CHECK (json_valid(j))); +SHOW CREATE TABLE t3; +--echo # a constraint that mentions the column inside a larger expression +CREATE TABLE t4 (id INT, j LONGTEXT CHECK (json_valid(j) AND id > 0)); +SHOW CREATE TABLE t4; +--echo # a table level constraint over two columns +CREATE TABLE t5 (id INT, j LONGTEXT, k LONGTEXT, + CONSTRAINT both_valid CHECK (json_valid(j) AND json_valid(k))); +SHOW CREATE TABLE t5; + +--echo # +--echo # 2. The constraint doing its job. +--echo # + +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'); +INSERT INTO t1 VALUES (2, '{"a": 1, "b": [1, 2]}'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, '{"a":1,'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, 'not json at all'); +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = '{' WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +--echo # the hand written constraint behaves the same way +--error ER_CONSTRAINT_FAILED +INSERT INTO t3 VALUES (1, '{'); +INSERT INTO t3 VALUES (1, '{"a":1}'); +SELECT id, j FROM t3; + +--echo # the constraint shapes that are NOT a bare check of one column. Each +--echo # of these still has to run in full, so what they accept and reject is +--echo # recorded here rather than left to the CREATE statement alone. +--echo # constraint on the column combined with a test of another column +--error ER_CONSTRAINT_FAILED +INSERT INTO t4 VALUES (1, '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t4 VALUES (0, '{"a":1}'); +INSERT INTO t4 VALUES (1, '{"a":1}'); +SELECT id, j FROM t4; +--echo # table level constraint over two columns +--error ER_CONSTRAINT_FAILED +INSERT INTO t5 VALUES (1, '{', '{"a":1}'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5 VALUES (1, '{"a":1}', '{'); +INSERT INTO t5 VALUES (1, '{"a":1}', '{"b":2}'); +SELECT id, j, k FROM t5; +--echo # an OR, where the other side can let an invalid document through +CREATE TABLE t5b (id INT, j LONGTEXT CHECK (json_valid(j) OR id > 0)); +INSERT INTO t5b VALUES (1, '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5b VALUES (0, '{'); +SELECT id, j, JSON_VALID(j) AS valid FROM t5b; +--echo # a constraint attached to one column that tests a different column +CREATE TABLE t5c (a LONGTEXT, b LONGTEXT CHECK (json_valid(a))); +INSERT INTO t5c VALUES ('{"a":1}', '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5c VALUES ('{', '{"a":1}'); +SELECT a, b FROM t5c; +--echo # writing any constraint of your own on a JSON column replaces the +--echo # automatic one, so invalid documents go in with the checks left on +CREATE TABLE t5e (j JSON CHECK (LENGTH(j) > 0)); +SHOW CREATE TABLE t5e; +INSERT INTO t5e VALUES ('{"a":1}'); +INSERT INTO t5e VALUES ('{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5e VALUES (''); +SELECT j, JSON_VALID(j) AS valid FROM t5e ORDER BY j; +SELECT JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS still_valid + FROM t5e ORDER BY j; +DROP TABLE t5b, t5c, t5e; + +--echo # +--echo # 3. Switching the check off, which is what makes a stored document +--echo # unverifiable afterwards. +--echo # + +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,'); +INSERT INTO t1 VALUES (11, 'not json at all'); +INSERT INTO t1 VALUES (12, '{"a":1} trailing'); +INSERT INTO t1 VALUES (13, CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32))); +INSERT INTO t3 VALUES (10, '{"a":1,'); +SET SESSION check_constraint_checks = ON; + +--echo # the rows are there and the constraint is back on +SELECT id, j FROM t1 ORDER BY id; +SELECT id, JSON_VALID(j) AS valid FROM t1 ORDER BY id; +--echo # a new statement is checked again, the stored rows are not +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (14, '{'); +--echo # reading the column back does not check anything +SELECT id, j FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, HEX(j) FROM t1 WHERE id = 10; + +--echo # +--echo # 4. Every function fed the column that no longer satisfies its check. +--echo # + +SELECT id, JSON_VALID(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_TYPE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_DEPTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_LENGTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_KEYS(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_EXTRACT(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_QUERY(j, '$') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALUE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_INSERT(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_REPLACE(j, '$.a', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_REMOVE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_ARRAY_APPEND(j, '$', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_MERGE(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_MERGE_PATCH(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_COMPACT(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_LOOSE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_NORMALIZE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_EXISTS(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_CONTAINS(j, '1', '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_SEARCH(j, 'one', '1') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_EQUALS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +--echo # id 13, an array nested past the scanner limit, is left out here: +--echo # JSON_OVERLAPS crashes the server on such an argument, so the row is +--echo # excluded until that is fixed. Every other function above takes it. +SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v + FROM t1 WHERE id >= 10 AND id <> 13 ORDER BY id; + +--echo # +--echo # 5. The column as an embedded argument. A JSON column is embedded as +--echo # a document, so whatever it holds is copied into the result as is. +--echo # + +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS still_valid + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +SELECT id, JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +--echo # the same column read as plain text is quoted instead of embedded +SELECT id, JSON_ARRAY(CONCAT(j, '')) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(CONCAT(j, ''))) AS still_valid + FROM t1 WHERE id >= 10 ORDER BY id; +--echo # a LONGTEXT column holding the same bytes is quoted, not embedded +SELECT id, JSON_ARRAY(j) AS v FROM t3 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t3 ORDER BY id; + +--echo # 5a. The same column reached through a scalar subquery, where the +--echo # value is fetched once and held rather than read per row. +CREATE TABLE ts (id INT, j JSON); +INSERT INTO ts VALUES (1, '{"a": 1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ts VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 1)) AS v; +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 1))) AS ok; +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 2))) AS ok_poisoned; +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 1)) AS v; +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 1)) AS v; +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +DROP TABLE ts; + +--echo # 5b. The same column reached through a view. A merged view reads the +--echo # table directly; a materialised one copies the value into a +--echo # temporary table first. +CREATE TABLE tv (id INT, j JSON); +INSERT INTO tv VALUES (1, '{"a": 1, "b": [1, 2]}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tv VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +CREATE ALGORITHM=MERGE VIEW v_merge AS SELECT id, j FROM tv; +CREATE ALGORITHM=TEMPTABLE VIEW v_tmp AS SELECT id, j FROM tv; +SHOW CREATE VIEW v_merge; +SHOW CREATE VIEW v_tmp; +SELECT id, j, HEX(j) FROM v_merge ORDER BY id; +SELECT id, j, HEX(j) FROM v_tmp ORDER BY id; +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok + FROM v_merge ORDER BY id; +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok + FROM v_tmp ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_merge ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_tmp ORDER BY id; +SELECT JSON_ARRAYAGG(j) AS v FROM v_merge WHERE id = 1; +DROP VIEW v_merge, v_tmp; +DROP TABLE tv; + +--echo # +--echo # 6. A valid column, for comparison: the same calls with nothing wrong. +--echo # + +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id < 10 ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id < 10 ORDER BY id; +SELECT id, JSON_EXTRACT(j, '$.b') AS v FROM t1 WHERE id < 10 ORDER BY id; +--echo # a column value keeps the formatting it was stored with +SELECT id, HEX(j) FROM t1 WHERE id < 10 ORDER BY id; + +--echo # 6a. The same document stored in every column type it can live in. +--echo # Retrieval returns the stored bytes unchanged in all of them; +--echo # only the JSON-typed column is embedded rather than quoted. +CREATE TABLE tr (jj JSON, lt LONGTEXT CHECK (JSON_VALID(lt)), vc VARCHAR(64), + vb VARBINARY(64), bl BLOB); +INSERT INTO tr VALUES ('{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', + '{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', + '{"a": 1, "b":[1, 2]}'); +SELECT HEX(jj) AS h_json, HEX(lt) AS h_longtext, HEX(vc) AS h_varchar, + HEX(vb) AS h_varbinary, HEX(bl) AS h_blob FROM tr; +SELECT jj, lt, vc, vb, bl FROM tr; +SELECT JSON_ARRAY(jj) AS a_json, JSON_ARRAY(vc) AS a_varchar, + JSON_ARRAY(vb) AS a_varbinary, JSON_ARRAY(bl) AS a_blob FROM tr; +DROP TABLE tr; + +--echo # +--echo # 7. Storing a JSON function result back into a column, which is where +--echo # the check constraint runs again. +--echo # + +CREATE TABLE t6 (j JSON); +INSERT INTO t6 SELECT JSON_SET('{"a":1}', '$.b', 2); +INSERT INTO t6 SELECT JSON_ARRAY(1,2); +INSERT INTO t6 SELECT JSON_OBJECT('a',1); +SELECT j FROM t6; +--echo # a value the constructor built that the check does not accept +CREATE TABLE t7 (j JSON); +--error ER_CONSTRAINT_FAILED +INSERT INTO t7 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +SELECT COUNT(*) AS stored FROM t7; +--echo # the same value stored into a plain LONGTEXT column +CREATE TABLE t8 (j LONGTEXT); +INSERT INTO t8 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +SELECT JSON_VALID(j) AS v, LENGTH(j) AS len FROM t8; +--echo # a value that is truncated on the way into the column +CREATE TABLE t9 (j VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t9 SELECT JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT j, JSON_VALID(j) AS still_valid FROM t9; +SET @@sql_mode=DEFAULT; +DROP TABLE t6, t7, t8, t9; + +--echo # 7a. The same truncating store into a column that IS checked. The +--echo # chop happens first and the constraint then judges the chopped +--echo # bytes, which sometimes still parse and sometimes do not. +CREATE TABLE tn (v VARCHAR(10) CHECK (json_valid(v))); +SET @@sql_mode=''; +--echo # chopped into an unparseable prefix: rejected +--error ER_CONSTRAINT_FAILED +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +--echo # chopped on a boundary that still parses: accepted, with a warning +INSERT INTO tn VALUES (CONCAT('[1,2,3,45]', ',6789')); +INSERT INTO tn VALUES ('{"a":1}'); +SELECT v, JSON_VALID(v) AS valid FROM tn ORDER BY v; +SET @@sql_mode=DEFAULT; +--error ER_DATA_TOO_LONG +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT COUNT(*) AS stored FROM tn; +DROP TABLE tn; + +--echo # 7b. A store that converts the character set on the way in. The +--echo # constraint judges the converted bytes, not the ones the +--echo # function produced. +--echo # swe7 cannot carry the JSON punctuation at all +CREATE TABLE t11 (j LONGTEXT CHARACTER SET swe7 CHECK (json_valid(j))); +SET @@sql_mode=''; +--error ER_CONSTRAINT_FAILED +INSERT INTO t11 VALUES ('{"a":1}'); +SELECT COUNT(*) AS stored FROM t11; +--echo # the same column without a constraint keeps the damaged bytes +CREATE TABLE t12 (j LONGTEXT CHARACTER SET swe7); +INSERT INTO t12 VALUES ('{"a":1}'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t12; +SET @@sql_mode=DEFAULT; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +INSERT INTO t11 VALUES ('{"a":1}'); +--echo # a conversion that loses nothing still changes the bytes +CREATE TABLE t13 (j VARCHAR(100) CHARACTER SET latin1 CHECK (json_valid(j))); +INSERT INTO t13 VALUES (JSON_OBJECT('a', _utf8mb4 X'C3A9')); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t13; +CREATE TABLE tu (j JSON); +INSERT INTO tu VALUES ('{"a":1}'); +INSERT INTO t13 SELECT JSON_SET(j, '$.c', 3) FROM tu; +SELECT HEX(j) FROM t13 ORDER BY j; +DROP TABLE t11, t12, t13, tu; + +--echo # 7c. A table rebuilt by ALTER, which checks every existing row again +--echo # rather than the value of one statement. +CREATE TABLE ta (id INT, j JSON); +INSERT INTO ta VALUES (1, '{"a":1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ta VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +--error ER_CONSTRAINT_FAILED +ALTER TABLE ta FORCE, ALGORITHM=COPY; +ALTER IGNORE TABLE ta FORCE, ALGORITHM=COPY; +SELECT id, j FROM ta ORDER BY id; +DROP TABLE ta; +--echo # ALTER that converts the character set rewrites the stored bytes +CREATE TABLE tb (j JSON); +INSERT INTO tb VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +ALTER TABLE tb CONVERT TO CHARACTER SET latin1; +SHOW CREATE TABLE tb; +SELECT HEX(j) AS stored, JSON_VALID(j) AS valid FROM tb; +SELECT HEX(JSON_ARRAY(j)) AS h FROM tb; +DROP TABLE tb; + +--echo # 7d. Generated columns, where the JSON function result is computed +--echo # and stored by the server rather than supplied by the statement. +CREATE TABLE tg (j JSON, + c1 LONGTEXT AS (JSON_SET(j, '$.z', 9)) PERSISTENT, + c2 VARCHAR(20) AS (JSON_EXTRACT(j, '$.a')) PERSISTENT, KEY (c2)); +SHOW CREATE TABLE tg; +INSERT INTO tg (j) VALUES ('{"a": 1, "b": 2}'); +SELECT j, c1, HEX(c1), c2 FROM tg; +UPDATE tg SET j = '{"a": 7}'; +SELECT j, c1, HEX(c1), c2 FROM tg; +SELECT c2 FROM tg FORCE INDEX (c2) WHERE c2 = '7'; +DROP TABLE tg; + +--echo # +--echo # 8. A temporary table the user created. It is under user control the +--echo # same way a base table is, including the check switch. +--echo # + +CREATE TEMPORARY TABLE tmp1 (id INT, j JSON); +SHOW CREATE TABLE tmp1; +INSERT INTO tmp1 VALUES (1, '{"a":1}'); +--error ER_CONSTRAINT_FAILED +INSERT INTO tmp1 VALUES (2, '{'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tmp1 VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS valid FROM tmp1 ORDER BY id; +SELECT id, JSON_ARRAY(j) AS v FROM tmp1 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM tmp1 ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM tmp1 ORDER BY id; +DROP TEMPORARY TABLE tmp1; + +--echo # +--echo # 9. Values passing through a temporary table the server made for +--echo # itself: grouping, materialised derived tables, union branches and +--echo # window frames. +--echo # + +CREATE TABLE t10 (g INT, j JSON, txt VARCHAR(30)); +INSERT INTO t10 VALUES + (1, '{"a":1}', '{"a":1}'), (1, '{"a":2}', '{"a":2}'), + (2, '{"b":1}', '{"b":1}'); + +--echo # grouping over a JSON expression +SELECT g, JSON_ARRAYAGG(JSON_SET(j, '$.z', 1)) AS v FROM t10 GROUP BY g ORDER BY g; +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j ORDER BY j; +SELECT g, COUNT(*) AS c FROM t10 GROUP BY g, JSON_SET(j, '$.z', 1) ORDER BY g, c; + +--echo # a materialised derived table +SELECT JSON_VALID(v) AS valid, v + FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d + ORDER BY v; +SELECT JSON_SET(v, '$.y', 2) AS w + FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d + ORDER BY w; +SELECT JSON_ARRAY(v) AS w + FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d + ORDER BY w; + +--echo # union branches, where one branch is a document and the other is text +SELECT v, JSON_VALID(v) AS valid FROM ( + SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 + UNION + SELECT txt FROM t10 WHERE g = 2) u ORDER BY v; +SELECT JSON_ARRAY(v) AS w FROM ( + SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 + UNION + SELECT txt FROM t10 WHERE g = 2) u ORDER BY w; + +--echo # a window frame, which materialises its input +SELECT g, JSON_SET(j, '$.z', 1) AS v, + ROW_NUMBER() OVER (PARTITION BY g ORDER BY j) AS rn + FROM t10 ORDER BY g, rn; +SELECT g, JSON_ARRAY(JSON_SET(j, '$.z', 1)) AS v, + COUNT(*) OVER (PARTITION BY g) AS c + FROM t10 ORDER BY g, v; + +--echo # an ORDER BY that has to materialise the value first +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 ORDER BY JSON_EXTRACT(j, '$.a'), v; + +DROP TABLE t1, t2, t3, t4, t5, t10; diff --git a/mysql-test/main/func_json_depth.result b/mysql-test/main/func_json_depth.result new file mode 100644 index 0000000000000..0b1124ca1227b --- /dev/null +++ b/mysql-test/main/func_json_depth.result @@ -0,0 +1,378 @@ +# +# Behavioral baseline: nesting depth. +# +# The scanner keeps a fixed stack, so documents nested past its size +# are rejected. This test records the exact depth at which each +# function changes its answer, and what happens when a document that +# is acceptable on its own is spliced into another document deep +# enough that the combination is not. +# +# +# 1. Where the limit sits, for arrays and for objects. +# +SET @d29 = CONCAT(REPEAT('[', 29), '1', REPEAT(']', 29)); +SET @d30 = CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @d31 = CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d32 = CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32)); +SET @d33 = CONCAT(REPEAT('[', 33), '1', REPEAT(']', 33)); +SELECT JSON_VALID(@d29) AS v29, JSON_VALID(@d30) AS v30, JSON_VALID(@d31) AS v31; +v29 v30 v31 +1 1 1 +SELECT JSON_VALID(@d32) AS v32; +v32 +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_VALID(@d33) AS v33; +v33 +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_DEPTH(@d29) AS d29, JSON_DEPTH(@d30) AS d30, JSON_DEPTH(@d31) AS d31; +d29 d30 d31 +30 31 32 +SELECT JSON_DEPTH(@d32) AS d32; +d32 +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +# the same depths built from objects +SET @o30 = CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @o31 = CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SET @o32 = CONCAT(REPEAT('{"a":', 32), '1', REPEAT('}', 32)); +SELECT JSON_VALID(@o30) AS v30, JSON_VALID(@o31) AS v31, JSON_VALID(@o32) AS v32; +v30 v31 v32 +1 1 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 156 +SELECT JSON_DEPTH(@o31) AS d31; +d31 +32 +SELECT JSON_DEPTH(@o32) AS d32; +d32 +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 156 +# alternating containers reach the same limit +SET @m30 = CONCAT(REPEAT('[{"a":', 15), '1', REPEAT('}]', 15)); +SET @m32 = CONCAT(REPEAT('[{"a":', 16), '1', REPEAT('}]', 16)); +SELECT JSON_VALID(@m30) AS v30, JSON_VALID(@m32) AS v32; +v30 v32 +1 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 92 +SELECT JSON_DEPTH(@m30) AS d30; +d30 +31 +# +# 2. Every function at, and one past, the limit. +# +SELECT JSON_VALID(@d31) AS ok, JSON_VALID(@d32) AS over_limit; +ok over_limit +1 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_DEPTH(@d31) AS ok; +ok +32 +SELECT JSON_DEPTH(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +SELECT JSON_TYPE(@d31) AS ok; +ok +ARRAY +SELECT JSON_TYPE(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_type' at position 32 +SELECT JSON_LENGTH(@d31) AS ok; +ok +1 +SELECT JSON_LENGTH(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_length' at position 32 +SELECT JSON_COMPACT(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_compact' at position 32 +SELECT JSON_LOOSE(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_loose' at position 32 +SELECT JSON_DETAILED(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_detailed' at position 32 +SELECT JSON_NORMALIZE(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_normalize' at position 32 +SELECT JSON_EXTRACT(@d32, '$') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_extract' at position 32 +SELECT JSON_QUERY(@d32, '$') AS over_limit; +over_limit +NULL +SELECT JSON_VALUE(@d32, '$[0]') AS over_limit; +over_limit +NULL +SELECT JSON_KEYS(@o32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_keys' at position 160 +SELECT JSON_SET(@d32, '$[0]', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 32 +SELECT JSON_INSERT(@d32, '$.a', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 32 +SELECT JSON_REPLACE(@d32, '$[0]', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 32 +SELECT JSON_REMOVE(@d32, '$[0]') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_remove' at position 32 +SELECT JSON_ARRAY_APPEND(@d32, '$', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +SELECT JSON_ARRAY_INSERT(@d32, '$[0]', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_insert' at position 35 +SELECT JSON_MERGE(@d32, '[1]') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 32 +SELECT JSON_MERGE('[1]', @d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_merge_preserve' at position 32 +SELECT JSON_MERGE_PATCH(@d32, '{"a":1}') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_patch' at position 32 +SELECT JSON_MERGE_PATCH('{"a":1}', @d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_merge_patch' at position 32 +SELECT JSON_CONTAINS(@d32, '1') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains' at position 32 +SELECT JSON_CONTAINS_PATH(@d32, 'one', '$') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains_path' at position 32 +SELECT JSON_OVERLAPS(@d32, '[1]') AS over_limit; +over_limit +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 33 +SELECT JSON_EQUALS(@d32, @d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_equals' at position 32 +SELECT JSON_SEARCH(@d32, 'one', '1') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_search' at position 32 +SELECT JSON_EXISTS(@d32, '$') AS over_limit; +over_limit +1 +# +# 3. Path depth is limited independently of document depth. +# +SET @p31 = CONCAT('$', REPEAT('[0]', 31)); +SET @p32 = CONCAT('$', REPEAT('[0]', 32)); +SET @p33 = CONCAT('$', REPEAT('[0]', 33)); +SELECT JSON_EXTRACT(@d31, @p31) AS p31; +p31 +1 +SELECT JSON_EXTRACT(@d31, @p32) AS p32; +p32 +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_extract' at position 96 +SELECT JSON_EXTRACT(@d31, @p33) AS p33; +p33 +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_extract' at position 96 +SELECT JSON_EXISTS(@d31, @p32) AS p32; +p32 +NULL +SELECT JSON_SET('[1]', @p33, 2) AS p33; +p33 +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_set' at position 96 +# +# 4. Composed depth. A document that is acceptable on its own can be +# placed inside another document, and the sum is what the result +# has to carry. These record what happens today when the sum goes +# past the limit. +# +# a JSON-typed argument is embedded as a document, not as a string +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d29))) AS d30; +d30 +31 +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d30))) AS d31; +d31 +32 +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS d32; +d32 +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT(@d31))) AS still_valid; +still_valid +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT LENGTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS produced_bytes; +produced_bytes +65 +# the same through an object constructor +SELECT JSON_DEPTH(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS d32; +d32 +32 +SELECT JSON_VALID(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS still_valid; +still_valid +1 +# two levels of constructor +SELECT JSON_VALID(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d30)))) AS still_valid; +still_valid +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d29)))) AS d31; +d31 +32 +# a mutator splicing a deep value into a shallow document +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d29))) AS d30; +d30 +31 +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d30))) AS d31; +d31 +32 +SELECT JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 43 +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31))) AS still_valid; +still_valid +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 43 +SELECT JSON_INSERT('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 43 +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 35 +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_insert' at position 32 +# a mutator splicing a deep value into an already deep document +SET @d15 = CONCAT(REPEAT('[', 15), '1', REPEAT(']', 15)); +SET @d16 = CONCAT(REPEAT('[', 16), '1', REPEAT(']', 16)); +SET @p15 = CONCAT('$', REPEAT('[0]', 15)); +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS composed; +composed +31 +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS still_valid; +still_valid +1 +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS composed; +composed +32 +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS still_valid; +still_valid +1 +# merging two deep documents +SELECT JSON_DEPTH(JSON_MERGE(@d30, @d30)) AS merged; +merged +31 +SELECT JSON_DEPTH(JSON_MERGE(JSON_ARRAY(JSON_COMPACT(@d29)), '[1]')) AS merged; +merged +31 +# merging two objects, which nest rather than concatenate, so the +# composed depth is the sum and can cross the limit +SELECT JSON_DEPTH(JSON_MERGE(@o30, @o30)) AS composed_at_limit; +composed_at_limit +32 +SELECT JSON_VALID(JSON_MERGE(@o30, @o30)) AS still_valid; +still_valid +1 +SELECT JSON_MERGE(@o31, @o31) AS composed_over_limit; +composed_over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 156 +SELECT JSON_VALID(JSON_MERGE(@o31, @o31)) AS still_valid; +still_valid +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 156 +# this one stays within the limit and succeeds +SELECT JSON_MERGE_PATCH(CONCAT('{"a":', @d30, '}'), '{"b":1}') AS at_limit_ok; +at_limit_ok +{"a": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], "b": 1} +# a deep value spliced as a plain string is escaped, not embedded +SELECT JSON_DEPTH(JSON_ARRAY(CONCAT(@d31, ''))) AS as_string; +as_string +2 +SELECT JSON_VALID(JSON_ARRAY(CONCAT(@d31, ''))) AS still_valid; +still_valid +1 +# +# 5. Depth reached through repeated mutation, one level at a time. +# +SELECT JSON_DEPTH(JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), +'$.a[0]', JSON_ARRAY()), +'$.a[0][0]', JSON_ARRAY())) AS d; +d +4 +SELECT JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), +'$.a[0]', JSON_ARRAY()), +'$.a[0][0]', JSON_ARRAY()) AS v; +v +{"a": [[[]]]} diff --git a/mysql-test/main/func_json_depth.test b/mysql-test/main/func_json_depth.test new file mode 100644 index 0000000000000..9af7f0cebbb3c --- /dev/null +++ b/mysql-test/main/func_json_depth.test @@ -0,0 +1,155 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: nesting depth. +--echo # +--echo # The scanner keeps a fixed stack, so documents nested past its size +--echo # are rejected. This test records the exact depth at which each +--echo # function changes its answer, and what happens when a document that +--echo # is acceptable on its own is spliced into another document deep +--echo # enough that the combination is not. +--echo # + +--echo # +--echo # 1. Where the limit sits, for arrays and for objects. +--echo # + +SET @d29 = CONCAT(REPEAT('[', 29), '1', REPEAT(']', 29)); +SET @d30 = CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @d31 = CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d32 = CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32)); +SET @d33 = CONCAT(REPEAT('[', 33), '1', REPEAT(']', 33)); + +SELECT JSON_VALID(@d29) AS v29, JSON_VALID(@d30) AS v30, JSON_VALID(@d31) AS v31; +SELECT JSON_VALID(@d32) AS v32; +SELECT JSON_VALID(@d33) AS v33; +SELECT JSON_DEPTH(@d29) AS d29, JSON_DEPTH(@d30) AS d30, JSON_DEPTH(@d31) AS d31; +SELECT JSON_DEPTH(@d32) AS d32; + +--echo # the same depths built from objects +SET @o30 = CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @o31 = CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SET @o32 = CONCAT(REPEAT('{"a":', 32), '1', REPEAT('}', 32)); +SELECT JSON_VALID(@o30) AS v30, JSON_VALID(@o31) AS v31, JSON_VALID(@o32) AS v32; +SELECT JSON_DEPTH(@o31) AS d31; +SELECT JSON_DEPTH(@o32) AS d32; + +--echo # alternating containers reach the same limit +SET @m30 = CONCAT(REPEAT('[{"a":', 15), '1', REPEAT('}]', 15)); +SET @m32 = CONCAT(REPEAT('[{"a":', 16), '1', REPEAT('}]', 16)); +SELECT JSON_VALID(@m30) AS v30, JSON_VALID(@m32) AS v32; +SELECT JSON_DEPTH(@m30) AS d30; + +--echo # +--echo # 2. Every function at, and one past, the limit. +--echo # + +SELECT JSON_VALID(@d31) AS ok, JSON_VALID(@d32) AS over_limit; +SELECT JSON_DEPTH(@d31) AS ok; +SELECT JSON_DEPTH(@d32) AS over_limit; +SELECT JSON_TYPE(@d31) AS ok; +SELECT JSON_TYPE(@d32) AS over_limit; +SELECT JSON_LENGTH(@d31) AS ok; +SELECT JSON_LENGTH(@d32) AS over_limit; +SELECT JSON_COMPACT(@d32) AS over_limit; +SELECT JSON_LOOSE(@d32) AS over_limit; +SELECT JSON_DETAILED(@d32) AS over_limit; +SELECT JSON_NORMALIZE(@d32) AS over_limit; +SELECT JSON_EXTRACT(@d32, '$') AS over_limit; +SELECT JSON_QUERY(@d32, '$') AS over_limit; +SELECT JSON_VALUE(@d32, '$[0]') AS over_limit; +SELECT JSON_KEYS(@o32) AS over_limit; +SELECT JSON_SET(@d32, '$[0]', 2) AS over_limit; +SELECT JSON_INSERT(@d32, '$.a', 2) AS over_limit; +SELECT JSON_REPLACE(@d32, '$[0]', 2) AS over_limit; +SELECT JSON_REMOVE(@d32, '$[0]') AS over_limit; +SELECT JSON_ARRAY_APPEND(@d32, '$', 2) AS over_limit; +SELECT JSON_ARRAY_INSERT(@d32, '$[0]', 2) AS over_limit; +SELECT JSON_MERGE(@d32, '[1]') AS over_limit; +SELECT JSON_MERGE('[1]', @d32) AS over_limit; +SELECT JSON_MERGE_PATCH(@d32, '{"a":1}') AS over_limit; +SELECT JSON_MERGE_PATCH('{"a":1}', @d32) AS over_limit; +SELECT JSON_CONTAINS(@d32, '1') AS over_limit; +SELECT JSON_CONTAINS_PATH(@d32, 'one', '$') AS over_limit; +SELECT JSON_OVERLAPS(@d32, '[1]') AS over_limit; +SELECT JSON_EQUALS(@d32, @d32) AS over_limit; +SELECT JSON_SEARCH(@d32, 'one', '1') AS over_limit; +SELECT JSON_EXISTS(@d32, '$') AS over_limit; + +--echo # +--echo # 3. Path depth is limited independently of document depth. +--echo # + +SET @p31 = CONCAT('$', REPEAT('[0]', 31)); +SET @p32 = CONCAT('$', REPEAT('[0]', 32)); +SET @p33 = CONCAT('$', REPEAT('[0]', 33)); +SELECT JSON_EXTRACT(@d31, @p31) AS p31; +SELECT JSON_EXTRACT(@d31, @p32) AS p32; +SELECT JSON_EXTRACT(@d31, @p33) AS p33; +SELECT JSON_EXISTS(@d31, @p32) AS p32; +SELECT JSON_SET('[1]', @p33, 2) AS p33; + +--echo # +--echo # 4. Composed depth. A document that is acceptable on its own can be +--echo # placed inside another document, and the sum is what the result +--echo # has to carry. These record what happens today when the sum goes +--echo # past the limit. +--echo # + +--echo # a JSON-typed argument is embedded as a document, not as a string +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d29))) AS d30; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d30))) AS d31; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS d32; +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT(@d31))) AS still_valid; +SELECT LENGTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS produced_bytes; +--echo # the same through an object constructor +SELECT JSON_DEPTH(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS d32; +SELECT JSON_VALID(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS still_valid; +--echo # two levels of constructor +SELECT JSON_VALID(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d30)))) AS still_valid; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d29)))) AS d31; + +--echo # a mutator splicing a deep value into a shallow document +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d29))) AS d30; +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d30))) AS d31; +SELECT JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31))) AS still_valid; +SELECT JSON_INSERT('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_COMPACT(@d31)) AS over_limit; +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_COMPACT(@d31)) AS over_limit; + +--echo # a mutator splicing a deep value into an already deep document +SET @d15 = CONCAT(REPEAT('[', 15), '1', REPEAT(']', 15)); +SET @d16 = CONCAT(REPEAT('[', 16), '1', REPEAT(']', 16)); +SET @p15 = CONCAT('$', REPEAT('[0]', 15)); +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS composed; +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS still_valid; +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS composed; +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS still_valid; + +--echo # merging two deep documents +SELECT JSON_DEPTH(JSON_MERGE(@d30, @d30)) AS merged; +SELECT JSON_DEPTH(JSON_MERGE(JSON_ARRAY(JSON_COMPACT(@d29)), '[1]')) AS merged; +--echo # merging two objects, which nest rather than concatenate, so the +--echo # composed depth is the sum and can cross the limit +SELECT JSON_DEPTH(JSON_MERGE(@o30, @o30)) AS composed_at_limit; +SELECT JSON_VALID(JSON_MERGE(@o30, @o30)) AS still_valid; +SELECT JSON_MERGE(@o31, @o31) AS composed_over_limit; +SELECT JSON_VALID(JSON_MERGE(@o31, @o31)) AS still_valid; +--echo # this one stays within the limit and succeeds +SELECT JSON_MERGE_PATCH(CONCAT('{"a":', @d30, '}'), '{"b":1}') AS at_limit_ok; + +--echo # a deep value spliced as a plain string is escaped, not embedded +SELECT JSON_DEPTH(JSON_ARRAY(CONCAT(@d31, ''))) AS as_string; +SELECT JSON_VALID(JSON_ARRAY(CONCAT(@d31, ''))) AS still_valid; + +--echo # +--echo # 5. Depth reached through repeated mutation, one level at a time. +--echo # + +SELECT JSON_DEPTH(JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), + '$.a[0]', JSON_ARRAY()), + '$.a[0][0]', JSON_ARRAY())) AS d; +SELECT JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), + '$.a[0]', JSON_ARRAY()), + '$.a[0][0]', JSON_ARRAY()) AS v; diff --git a/mysql-test/main/func_json_format.result b/mysql-test/main/func_json_format.result new file mode 100644 index 0000000000000..ed0d9da85f3d1 --- /dev/null +++ b/mysql-test/main/func_json_format.result @@ -0,0 +1,508 @@ +# +# Behavioral baseline: exact output formatting of the JSON functions. +# +# Every JSON producer is exercised with input in several formattings +# (compact, already-spaced, whitespace-heavy) so that the exact bytes +# each function emits are recorded. HEX() is used next to the plain +# output wherever whitespace decides the answer, because leading and +# trailing space is easy to lose in a result file. +# +# +# 1. Mutating functions. These reformat their document argument. +# +# --- JSON_SET --- +SELECT JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2], "c": {"d": "x"}} +SELECT HEX(JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D2C202263223A207B2264223A202278227D7D +SELECT JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2], "c": {"d": "x"}} +SELECT HEX(JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D2C202263223A207B2264223A202278227D7D +SELECT JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +SELECT HEX(JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D7D +# a path that matches nothing still reformats the document +SELECT JSON_SET('{"a":1,"b":[1,2]}', '$.zzz.deeper', 2) AS v; +v +{"a": 1, "b": [1, 2]} +# scalar document +SELECT JSON_SET('1', '$', 2) AS v; +v +2 +SELECT JSON_SET('"str"', '$.a', 2) AS v; +v +"str" +# --- JSON_INSERT --- +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2], "new": 2} +SELECT HEX(JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2)) AS h; +h +7B2261223A20312C202262223A205B312C20325D2C20226E6577223A20327D +SELECT JSON_INSERT('{"a": 1, "b": [1, 2]}', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2], "new": 2} +SELECT JSON_INSERT('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2], "new": 2} +# existing key is not replaced, document is still reformatted +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.a', 9) AS v; +v +{"a": 1, "b": [1, 2]} +# --- JSON_REPLACE --- +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +SELECT HEX(JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D7D +SELECT JSON_REPLACE('{"a": 1, "b": [1, 2]}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +SELECT JSON_REPLACE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +# missing key is not inserted, document is still reformatted +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2]} +# --- JSON_REMOVE --- +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a') AS v; +v +{"b": [1, 2], "c": {"d": "x"}} +SELECT HEX(JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a')) AS h; +h +7B2262223A205B312C20325D2C202263223A207B2264223A202278227D7D +SELECT JSON_REMOVE('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a') AS v; +v +{"b": [1, 2], "c": {"d": "x"}} +SELECT JSON_REMOVE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a') AS v; +v +{"b": [1, 2]} +# removal at the cut boundaries: first, middle and last member +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.b') AS v; +v +{"a": 1, "c": {"d": "x"}} +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.c') AS v; +v +{"a": 1, "b": [1, 2]} +SELECT JSON_REMOVE('[1,2,3]', '$[0]') AS v; +v +[2, 3] +SELECT JSON_REMOVE('[1,2,3]', '$[1]') AS v; +v +[1, 3] +SELECT JSON_REMOVE('[1,2,3]', '$[2]') AS v; +v +[1, 2] +# removing the only member +SELECT JSON_REMOVE('{"a":1}', '$.a') AS v, HEX(JSON_REMOVE('{"a":1}', '$.a')) AS h; +v h +{} 7B7D +SELECT JSON_REMOVE('[1]', '$[0]') AS v, HEX(JSON_REMOVE('[1]', '$[0]')) AS h; +v h +[] 5B5D +# --- JSON_ARRAY_APPEND --- +SELECT JSON_ARRAY_APPEND('[1,2]', '$', 3) AS v; +v +[1, 2, 3] +SELECT HEX(JSON_ARRAY_APPEND('[1,2]', '$', 3)) AS h; +h +5B312C20322C20335D +SELECT JSON_ARRAY_APPEND('[1, 2]', '$', 3) AS v; +v +[1, 2, 3] +SELECT JSON_ARRAY_APPEND('{"a":1,"b":[1,2]}', '$.b', 3) AS v; +v +{"a": 1, "b": [1, 2, 3]} +# appending to a scalar wraps it into an array +SELECT JSON_ARRAY_APPEND('{"a":1}', '$.a', 2) AS v; +v +{"a": [1, 2]} +# --- JSON_ARRAY_INSERT --- +SELECT JSON_ARRAY_INSERT('[1,2]', '$[0]', 0) AS v; +v +[0, 1, 2] +SELECT HEX(JSON_ARRAY_INSERT('[1,2]', '$[0]', 0)) AS h; +h +5B302C20312C20325D +SELECT JSON_ARRAY_INSERT('[1, 2]', '$[1]', 0) AS v; +v +[1, 0, 2] +SELECT JSON_ARRAY_INSERT('[1,2]', '$[9]', 0) AS v; +v +[1, 2, 0] +SELECT JSON_ARRAY_INSERT('{"a":1,"b":[1,2]}', '$.b[0]', 0) AS v; +v +{"a": 1, "b": [0, 1, 2]} +# --- JSON_MERGE / JSON_MERGE_PRESERVE --- +SELECT JSON_MERGE('[1,2]', '[3,4]') AS v, HEX(JSON_MERGE('[1,2]', '[3,4]')) AS h; +v h +[1, 2, 3, 4] 5B312C20322C20332C20345D +SELECT JSON_MERGE('{"a":1}', '{"b":2}') AS v; +v +{"a": 1, "b": 2} +SELECT JSON_MERGE('{"a":1}', '{"a":2}') AS v; +v +{"a": [1, 2]} +SELECT JSON_MERGE_PRESERVE('{"a":1}', '{"a":2}') AS v; +v +{"a": [1, 2]} +SELECT JSON_MERGE('1', '2') AS v; +v +[1, 2] +# three-way merge +SELECT JSON_MERGE('[1]', '[2]', '[3]') AS v; +v +[1, 2, 3] +# --- JSON_MERGE_PATCH --- +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS v; +v +{"a": 1, "b": 2} +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '{"b":2}')) AS h; +h +7B2261223A20312C202262223A20327D +SELECT JSON_MERGE_PATCH('{"a":1}', '{"a":null}') AS v; +v +{} +SELECT JSON_MERGE_PATCH('{"a":{"b":1,"c":2}}', '{"a":{"b":9}}') AS v; +v +{"a": {"b": 9, "c": 2}} +# a non-object patch is adopted wholesale +SELECT JSON_MERGE_PATCH('{"a":1}', '[1, 2]') AS v; +v +[1, 2] +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '[1, 2]')) AS h; +h +5B312C20325D +SELECT JSON_MERGE_PATCH('{"a":1}', '"str"') AS v; +v +"str" +# the last argument decides when it is not an object +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}', '[9]') AS v; +v +[9] +# +# 2. Constructors. These copy their arguments and do NOT reformat. +# +# plain strings are quoted, JSON-typed arguments are embedded raw +SELECT JSON_ARRAY(1, 'str', NULL, TRUE) AS v; +v +[1, "str", null, true] +SELECT JSON_ARRAY() AS v, HEX(JSON_ARRAY()) AS h; +v h +[] 5B5D +SELECT JSON_OBJECT() AS v, HEX(JSON_OBJECT()) AS h; +v h +{} 7B7D +SELECT JSON_OBJECT('a', 1, 'b', 'str') AS v; +v +{"a": 1, "b": "str"} +SELECT HEX(JSON_OBJECT('a', 1, 'b', 'str')) AS h; +h +7B2261223A20312C202262223A2022737472227D +# embedded document arguments keep their own formatting +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1,"b": 2}')) AS v; +v +[{"a": 1,"b": 2}] +SELECT JSON_ARRAY(JSON_DETAILED('{"a":1}')) AS v; +v +[{"a":1}] +SELECT JSON_ARRAY(JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +v +[{"a":1,"b": 2}] +SELECT JSON_OBJECT('k', JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +v +{"k": {"a":1,"b": 2}} +# a nested constructor call +SELECT JSON_ARRAY(JSON_ARRAY(1,2), JSON_OBJECT('a',1)) AS v; +v +[[1, 2], {"a": 1}] +SELECT JSON_OBJECT('a', JSON_ARRAY(1,2)) AS v; +v +{"a": [1, 2]} +# a mutator result embedded in a constructor +SELECT JSON_ARRAY(JSON_SET('{"a":1,"b":2}', '$.a', 9)) AS v; +v +[{"a": 9, "b": 2}] +# +# 3. JSON_EXTRACT: formatting and validation depend on the evaluation +# context. In string context the document is scanned to the end and +# the result is reformatted; in numeric context the scan stops at the +# first match and nothing is reformatted. +# +SELECT JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a') AS v; +v +{"b": 1, "c": 2} +SELECT HEX(JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a')) AS h; +h +7B2262223A20312C202263223A20327D +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS v; +v +{"b": 1, "c": 2} +# several paths produce an array wrapper +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS v; +v +[1, 2] +SELECT HEX(JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b')) AS h; +h +5B312C20325D +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.*') AS v; +v +[1, 2] +# string context rejects trailing garbage after the document +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +# numeric context does not reach the trailing garbage +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') + 0 AS v; +v +1 +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') * 1.0 AS v; +v +1 +SELECT CAST(JSON_EXTRACT('{"a":1} trailing', '$.a') AS SIGNED) AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +# the same asymmetry with a structurally broken tail +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') + 0 AS v; +v +1 +# numeric context on a non-scalar match +SELECT JSON_EXTRACT('{"a":[1,2]}', '$.a') + 0 AS v; +v +0 +# +# 4. JSON_QUERY and JSON_VALUE return a slice of the input and keep the +# input formatting. +# +SELECT JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a') AS v; +v +{"b":1, "c":2} +SELECT HEX(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a')) AS h; +h +7B2262223A312C2020202263223A327D +SELECT JSON_QUERY('{"a":[1, 2]}', '$.a') AS v; +v +[1, 2] +# JSON_QUERY only returns objects and arrays +SELECT JSON_QUERY('{"a":1}', '$.a') AS v; +v +NULL +SELECT JSON_VALUE('{"a":1}', '$.a') AS v; +v +1 +SELECT JSON_VALUE('{"a":"s t"}', '$.a') AS v, HEX(JSON_VALUE('{"a":"s t"}', '$.a')) AS h; +v h +s t 73202074 +# JSON_VALUE unquotes, so escapes are resolved +SELECT JSON_VALUE('{"a":"q\\"q"}', '$.a') AS v; +v +q"q +SELECT JSON_VALUE('{"a":[1,2]}', '$.a') AS v; +v +NULL +# a slice taken past trailing garbage +SELECT JSON_QUERY('{"a":{"b":1}} trailing', '$.a') AS v; +v +{"b":1} +SELECT JSON_VALUE('{"a":1} trailing', '$.a') AS v; +v +1 +# +# 5. The explicit formatters. These stay after the mutator epilogues +# are gone, so their exact bytes matter. +# +SELECT JSON_COMPACT('{"a": 1, "b": [1, 2]}') AS v; +v +{"a":1,"b":[1,2]} +SELECT HEX(JSON_COMPACT('{"a": 1, "b": [1, 2]}')) AS h; +h +7B2261223A312C2262223A5B312C325D7D +SELECT JSON_LOOSE('{"a":1,"b":[1,2]}') AS v; +v +{"a": 1, "b": [1, 2]} +SELECT HEX(JSON_LOOSE('{"a":1,"b":[1,2]}')) AS h; +h +7B2261223A20312C202262223A205B312C20325D7D +SELECT JSON_DETAILED('{"a":1,"b":[1,2]}') AS v; +v +{ + "a": 1, + "b": + [ + 1, + 2 + ] +} +SELECT HEX(JSON_DETAILED('{"a":1,"b":[1,2]}')) AS h; +h +7B0A202020202261223A20312C0A202020202262223A200A202020205B0A2020202020202020312C0A2020202020202020320A202020205D0A7D +SELECT JSON_PRETTY('{"a":1,"b":[1,2]}') AS v; +v +{ + "a": 1, + "b": + [ + 1, + 2 + ] +} +SELECT HEX(JSON_PRETTY('{"a":1,"b":[1,2]}')) AS h; +h +7B0A202020202261223A20312C0A202020202262223A200A202020205B0A2020202020202020312C0A2020202020202020320A202020205D0A7D +# JSON_DETAILED with an explicit tab size +SELECT HEX(JSON_DETAILED('{"a":1}', 2)) AS h; +h +7B0A20202261223A20310A7D +SELECT HEX(JSON_DETAILED('{"a":1}', 0)) AS h; +h +7B0A2261223A20310A7D +# empty containers +SELECT HEX(JSON_LOOSE('{}')) AS h, HEX(JSON_LOOSE('[]')) AS h2; +h h2 +7B7D 5B5D +SELECT HEX(JSON_DETAILED('{}')) AS h, HEX(JSON_DETAILED('[]')) AS h2; +h h2 +7B0A7D 5B5D +# the formatters are a pass-through in JSON context +SELECT JSON_ARRAY(JSON_LOOSE('{"a":1}')) AS v; +v +[{"a":1}] +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1}')) AS v; +v +[{"a": 1}] +# +# 6. The remaining functions, for completeness of the surface. +# +SELECT JSON_KEYS('{"a":1,"b":2}') AS v, HEX(JSON_KEYS('{"a":1,"b":2}')) AS h; +v h +["a", "b"] 5B2261222C202262225D +SELECT JSON_KEYS('{"a":{"b":1}}', '$.a') AS v; +v +["b"] +SELECT JSON_KEYS('{}') AS v; +v +[] +SELECT JSON_QUOTE('a"b') AS v, HEX(JSON_QUOTE('a"b')) AS h; +v h +"a\"b" 22615C226222 +SELECT JSON_UNQUOTE('"a\\"b"') AS v; +v +a"b +SELECT JSON_UNQUOTE('not quoted') AS v; +v +not quoted +SELECT JSON_NORMALIZE('{"b":1,"a":2}') AS v; +v +{"a":2.0E0,"b":1.0E0} +SELECT HEX(JSON_NORMALIZE('{"b":1,"a":2}')) AS h; +h +7B2261223A322E3045302C2262223A312E3045307D +SELECT JSON_TYPE('{}') AS o, JSON_TYPE('[]') AS a, JSON_TYPE('1') AS i, +JSON_TYPE('1.5') AS d, JSON_TYPE('"s"') AS s, JSON_TYPE('true') AS b, +JSON_TYPE('null') AS n; +o a i d s b n +OBJECT ARRAY INTEGER DOUBLE STRING BOOLEAN NULL +SELECT JSON_LENGTH('{"a":1,"b":2}') AS v, JSON_LENGTH('[1,2,3]') AS a; +v a +2 3 +SELECT JSON_DEPTH('1') AS s, JSON_DEPTH('[]') AS e, JSON_DEPTH('[[1]]') AS n; +s e n +1 1 3 +SELECT JSON_VALID('{"a":1}') AS ok, JSON_VALID('{') AS bad, JSON_VALID(NULL) AS n; +ok bad n +1 0 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_EXISTS('{"a":1}', '$.a') AS y, JSON_EXISTS('{"a":1}', '$.b') AS n; +y n +1 0 +SELECT JSON_CONTAINS('{"a":1,"b":2}', '1', '$.a') AS y; +y +1 +SELECT JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a', '$.z') AS y; +y +1 +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS y; +y +1 +SELECT JSON_EQUALS('{"a":1,"b":2}', '{"b":2,"a":1}') AS y; +y +1 +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS v; +v +"$.a" +SELECT JSON_SEARCH('["x","x"]', 'all', 'x') AS v; +v +["$[0]", "$[1]"] +SELECT HEX(JSON_SEARCH('["x","x"]', 'all', 'x')) AS h; +h +5B22245B305D222C2022245B315D225D +# +# 7. Chained calls. A JSON function feeding another JSON function is +# the shape the reparse work targets, so the current bytes of every +# combination are recorded here. +# +SELECT JSON_SET(JSON_SET('{"a":1,"b":2}', '$.a', 9), '$.b', 8) AS v; +v +{"a": 9, "b": 8} +SELECT JSON_INSERT(JSON_REMOVE('{"a":1,"b":2}', '$.a'), '$.c', 3) AS v; +v +{"b": 2, "c": 3} +SELECT JSON_MERGE(JSON_ARRAY(1,2), JSON_ARRAY(3)) AS v; +v +[1, 2, 3] +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a',1), JSON_OBJECT('b',2)) AS v; +v +{"a": 1, "b": 2} +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', 2) AS v; +v +[1, 2] +SELECT JSON_EXTRACT(JSON_SET('{"a":1}', '$.b', 2), '$') AS v; +v +{"a": 1, "b": 2} +SELECT JSON_QUERY(JSON_SET('{"a":{"b":1}}', '$.a.c', 2), '$.a') AS v; +v +{"b": 1, "c": 2} +SELECT JSON_COMPACT(JSON_SET('{"a":1}', '$.b', 2)) AS v; +v +{"a":1,"b":2} +SELECT JSON_LOOSE(JSON_COMPACT('{"a": 1}')) AS v; +v +{"a": 1} +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', 2)) AS v; +v +1 +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_ARRAY(1))) AS v; +v +3 +# a mutator over an extracted fragment +SELECT JSON_SET(JSON_EXTRACT('{"a":{"b":1}}', '$.a'), '$.c', 2) AS v; +v +{"b": 1, "c": 2} +# a mutator over a query slice that kept odd input spacing +SELECT JSON_SET(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a'), '$.d', 3) AS v; +v +{"b": 1, "c": 2, "d": 3} +# a constructor fed by a mutator fed by a constructor +SELECT JSON_OBJECT('k', JSON_SET(JSON_OBJECT('a', 1), '$.b', 2)) AS v; +v +{"k": {"a": 1, "b": 2}} diff --git a/mysql-test/main/func_json_format.test b/mysql-test/main/func_json_format.test new file mode 100644 index 0000000000000..b8b3fb5562ddc --- /dev/null +++ b/mysql-test/main/func_json_format.test @@ -0,0 +1,235 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: exact output formatting of the JSON functions. +--echo # +--echo # Every JSON producer is exercised with input in several formattings +--echo # (compact, already-spaced, whitespace-heavy) so that the exact bytes +--echo # each function emits are recorded. HEX() is used next to the plain +--echo # output wherever whitespace decides the answer, because leading and +--echo # trailing space is easy to lose in a result file. +--echo # + +--echo # +--echo # 1. Mutating functions. These reformat their document argument. +--echo # + +--echo # --- JSON_SET --- +SELECT JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2) AS v; +SELECT HEX(JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2)) AS h; +SELECT JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2) AS v; +SELECT HEX(JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2)) AS h; +SELECT JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +SELECT HEX(JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2)) AS h; +--echo # a path that matches nothing still reformats the document +SELECT JSON_SET('{"a":1,"b":[1,2]}', '$.zzz.deeper', 2) AS v; +--echo # scalar document +SELECT JSON_SET('1', '$', 2) AS v; +SELECT JSON_SET('"str"', '$.a', 2) AS v; + +--echo # --- JSON_INSERT --- +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2) AS v; +SELECT HEX(JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2)) AS h; +SELECT JSON_INSERT('{"a": 1, "b": [1, 2]}', '$.new', 2) AS v; +SELECT JSON_INSERT('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.new', 2) AS v; +--echo # existing key is not replaced, document is still reformatted +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.a', 9) AS v; + +--echo # --- JSON_REPLACE --- +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2) AS v; +SELECT HEX(JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2)) AS h; +SELECT JSON_REPLACE('{"a": 1, "b": [1, 2]}', '$.a', 2) AS v; +SELECT JSON_REPLACE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +--echo # missing key is not inserted, document is still reformatted +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.new', 2) AS v; + +--echo # --- JSON_REMOVE --- +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a') AS v; +SELECT HEX(JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a')) AS h; +SELECT JSON_REMOVE('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a') AS v; +SELECT JSON_REMOVE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a') AS v; +--echo # removal at the cut boundaries: first, middle and last member +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.b') AS v; +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.c') AS v; +SELECT JSON_REMOVE('[1,2,3]', '$[0]') AS v; +SELECT JSON_REMOVE('[1,2,3]', '$[1]') AS v; +SELECT JSON_REMOVE('[1,2,3]', '$[2]') AS v; +--echo # removing the only member +SELECT JSON_REMOVE('{"a":1}', '$.a') AS v, HEX(JSON_REMOVE('{"a":1}', '$.a')) AS h; +SELECT JSON_REMOVE('[1]', '$[0]') AS v, HEX(JSON_REMOVE('[1]', '$[0]')) AS h; + +--echo # --- JSON_ARRAY_APPEND --- +SELECT JSON_ARRAY_APPEND('[1,2]', '$', 3) AS v; +SELECT HEX(JSON_ARRAY_APPEND('[1,2]', '$', 3)) AS h; +SELECT JSON_ARRAY_APPEND('[1, 2]', '$', 3) AS v; +SELECT JSON_ARRAY_APPEND('{"a":1,"b":[1,2]}', '$.b', 3) AS v; +--echo # appending to a scalar wraps it into an array +SELECT JSON_ARRAY_APPEND('{"a":1}', '$.a', 2) AS v; + +--echo # --- JSON_ARRAY_INSERT --- +SELECT JSON_ARRAY_INSERT('[1,2]', '$[0]', 0) AS v; +SELECT HEX(JSON_ARRAY_INSERT('[1,2]', '$[0]', 0)) AS h; +SELECT JSON_ARRAY_INSERT('[1, 2]', '$[1]', 0) AS v; +SELECT JSON_ARRAY_INSERT('[1,2]', '$[9]', 0) AS v; +SELECT JSON_ARRAY_INSERT('{"a":1,"b":[1,2]}', '$.b[0]', 0) AS v; + +--echo # --- JSON_MERGE / JSON_MERGE_PRESERVE --- +SELECT JSON_MERGE('[1,2]', '[3,4]') AS v, HEX(JSON_MERGE('[1,2]', '[3,4]')) AS h; +SELECT JSON_MERGE('{"a":1}', '{"b":2}') AS v; +SELECT JSON_MERGE('{"a":1}', '{"a":2}') AS v; +SELECT JSON_MERGE_PRESERVE('{"a":1}', '{"a":2}') AS v; +SELECT JSON_MERGE('1', '2') AS v; +--echo # three-way merge +SELECT JSON_MERGE('[1]', '[2]', '[3]') AS v; + +--echo # --- JSON_MERGE_PATCH --- +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS v; +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '{"b":2}')) AS h; +SELECT JSON_MERGE_PATCH('{"a":1}', '{"a":null}') AS v; +SELECT JSON_MERGE_PATCH('{"a":{"b":1,"c":2}}', '{"a":{"b":9}}') AS v; +--echo # a non-object patch is adopted wholesale +SELECT JSON_MERGE_PATCH('{"a":1}', '[1, 2]') AS v; +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '[1, 2]')) AS h; +SELECT JSON_MERGE_PATCH('{"a":1}', '"str"') AS v; +--echo # the last argument decides when it is not an object +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}', '[9]') AS v; + +--echo # +--echo # 2. Constructors. These copy their arguments and do NOT reformat. +--echo # + +--echo # plain strings are quoted, JSON-typed arguments are embedded raw +SELECT JSON_ARRAY(1, 'str', NULL, TRUE) AS v; +SELECT JSON_ARRAY() AS v, HEX(JSON_ARRAY()) AS h; +SELECT JSON_OBJECT() AS v, HEX(JSON_OBJECT()) AS h; +SELECT JSON_OBJECT('a', 1, 'b', 'str') AS v; +SELECT HEX(JSON_OBJECT('a', 1, 'b', 'str')) AS h; +--echo # embedded document arguments keep their own formatting +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1,"b": 2}')) AS v; +SELECT JSON_ARRAY(JSON_DETAILED('{"a":1}')) AS v; +SELECT JSON_ARRAY(JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +SELECT JSON_OBJECT('k', JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +--echo # a nested constructor call +SELECT JSON_ARRAY(JSON_ARRAY(1,2), JSON_OBJECT('a',1)) AS v; +SELECT JSON_OBJECT('a', JSON_ARRAY(1,2)) AS v; +--echo # a mutator result embedded in a constructor +SELECT JSON_ARRAY(JSON_SET('{"a":1,"b":2}', '$.a', 9)) AS v; + +--echo # +--echo # 3. JSON_EXTRACT: formatting and validation depend on the evaluation +--echo # context. In string context the document is scanned to the end and +--echo # the result is reformatted; in numeric context the scan stops at the +--echo # first match and nothing is reformatted. +--echo # + +SELECT JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a') AS v; +SELECT HEX(JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a')) AS h; +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS v; +--echo # several paths produce an array wrapper +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS v; +SELECT HEX(JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b')) AS h; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.*') AS v; +--echo # string context rejects trailing garbage after the document +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') AS v; +--echo # numeric context does not reach the trailing garbage +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') + 0 AS v; +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') * 1.0 AS v; +SELECT CAST(JSON_EXTRACT('{"a":1} trailing', '$.a') AS SIGNED) AS v; +--echo # the same asymmetry with a structurally broken tail +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') AS v; +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') + 0 AS v; +--echo # numeric context on a non-scalar match +SELECT JSON_EXTRACT('{"a":[1,2]}', '$.a') + 0 AS v; + +--echo # +--echo # 4. JSON_QUERY and JSON_VALUE return a slice of the input and keep the +--echo # input formatting. +--echo # + +SELECT JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a') AS v; +SELECT HEX(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a')) AS h; +SELECT JSON_QUERY('{"a":[1, 2]}', '$.a') AS v; +--echo # JSON_QUERY only returns objects and arrays +SELECT JSON_QUERY('{"a":1}', '$.a') AS v; +SELECT JSON_VALUE('{"a":1}', '$.a') AS v; +SELECT JSON_VALUE('{"a":"s t"}', '$.a') AS v, HEX(JSON_VALUE('{"a":"s t"}', '$.a')) AS h; +--echo # JSON_VALUE unquotes, so escapes are resolved +SELECT JSON_VALUE('{"a":"q\\"q"}', '$.a') AS v; +SELECT JSON_VALUE('{"a":[1,2]}', '$.a') AS v; +--echo # a slice taken past trailing garbage +SELECT JSON_QUERY('{"a":{"b":1}} trailing', '$.a') AS v; +SELECT JSON_VALUE('{"a":1} trailing', '$.a') AS v; + +--echo # +--echo # 5. The explicit formatters. These stay after the mutator epilogues +--echo # are gone, so their exact bytes matter. +--echo # + +SELECT JSON_COMPACT('{"a": 1, "b": [1, 2]}') AS v; +SELECT HEX(JSON_COMPACT('{"a": 1, "b": [1, 2]}')) AS h; +SELECT JSON_LOOSE('{"a":1,"b":[1,2]}') AS v; +SELECT HEX(JSON_LOOSE('{"a":1,"b":[1,2]}')) AS h; +SELECT JSON_DETAILED('{"a":1,"b":[1,2]}') AS v; +SELECT HEX(JSON_DETAILED('{"a":1,"b":[1,2]}')) AS h; +SELECT JSON_PRETTY('{"a":1,"b":[1,2]}') AS v; +SELECT HEX(JSON_PRETTY('{"a":1,"b":[1,2]}')) AS h; +--echo # JSON_DETAILED with an explicit tab size +SELECT HEX(JSON_DETAILED('{"a":1}', 2)) AS h; +SELECT HEX(JSON_DETAILED('{"a":1}', 0)) AS h; +--echo # empty containers +SELECT HEX(JSON_LOOSE('{}')) AS h, HEX(JSON_LOOSE('[]')) AS h2; +SELECT HEX(JSON_DETAILED('{}')) AS h, HEX(JSON_DETAILED('[]')) AS h2; +--echo # the formatters are a pass-through in JSON context +SELECT JSON_ARRAY(JSON_LOOSE('{"a":1}')) AS v; +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1}')) AS v; + +--echo # +--echo # 6. The remaining functions, for completeness of the surface. +--echo # + +SELECT JSON_KEYS('{"a":1,"b":2}') AS v, HEX(JSON_KEYS('{"a":1,"b":2}')) AS h; +SELECT JSON_KEYS('{"a":{"b":1}}', '$.a') AS v; +SELECT JSON_KEYS('{}') AS v; +SELECT JSON_QUOTE('a"b') AS v, HEX(JSON_QUOTE('a"b')) AS h; +SELECT JSON_UNQUOTE('"a\\"b"') AS v; +SELECT JSON_UNQUOTE('not quoted') AS v; +SELECT JSON_NORMALIZE('{"b":1,"a":2}') AS v; +SELECT HEX(JSON_NORMALIZE('{"b":1,"a":2}')) AS h; +SELECT JSON_TYPE('{}') AS o, JSON_TYPE('[]') AS a, JSON_TYPE('1') AS i, + JSON_TYPE('1.5') AS d, JSON_TYPE('"s"') AS s, JSON_TYPE('true') AS b, + JSON_TYPE('null') AS n; +SELECT JSON_LENGTH('{"a":1,"b":2}') AS v, JSON_LENGTH('[1,2,3]') AS a; +SELECT JSON_DEPTH('1') AS s, JSON_DEPTH('[]') AS e, JSON_DEPTH('[[1]]') AS n; +SELECT JSON_VALID('{"a":1}') AS ok, JSON_VALID('{') AS bad, JSON_VALID(NULL) AS n; +SELECT JSON_EXISTS('{"a":1}', '$.a') AS y, JSON_EXISTS('{"a":1}', '$.b') AS n; +SELECT JSON_CONTAINS('{"a":1,"b":2}', '1', '$.a') AS y; +SELECT JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a', '$.z') AS y; +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS y; +SELECT JSON_EQUALS('{"a":1,"b":2}', '{"b":2,"a":1}') AS y; +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS v; +SELECT JSON_SEARCH('["x","x"]', 'all', 'x') AS v; +SELECT HEX(JSON_SEARCH('["x","x"]', 'all', 'x')) AS h; + +--echo # +--echo # 7. Chained calls. A JSON function feeding another JSON function is +--echo # the shape the reparse work targets, so the current bytes of every +--echo # combination are recorded here. +--echo # + +SELECT JSON_SET(JSON_SET('{"a":1,"b":2}', '$.a', 9), '$.b', 8) AS v; +SELECT JSON_INSERT(JSON_REMOVE('{"a":1,"b":2}', '$.a'), '$.c', 3) AS v; +SELECT JSON_MERGE(JSON_ARRAY(1,2), JSON_ARRAY(3)) AS v; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a',1), JSON_OBJECT('b',2)) AS v; +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', 2) AS v; +SELECT JSON_EXTRACT(JSON_SET('{"a":1}', '$.b', 2), '$') AS v; +SELECT JSON_QUERY(JSON_SET('{"a":{"b":1}}', '$.a.c', 2), '$.a') AS v; +SELECT JSON_COMPACT(JSON_SET('{"a":1}', '$.b', 2)) AS v; +SELECT JSON_LOOSE(JSON_COMPACT('{"a": 1}')) AS v; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', 2)) AS v; +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_ARRAY(1))) AS v; +--echo # a mutator over an extracted fragment +SELECT JSON_SET(JSON_EXTRACT('{"a":{"b":1}}', '$.a'), '$.c', 2) AS v; +--echo # a mutator over a query slice that kept odd input spacing +SELECT JSON_SET(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a'), '$.d', 3) AS v; +--echo # a constructor fed by a mutator fed by a constructor +SELECT JSON_OBJECT('k', JSON_SET(JSON_OBJECT('a', 1), '$.b', 2)) AS v; diff --git a/mysql-test/main/func_json_invalid.result b/mysql-test/main/func_json_invalid.result new file mode 100644 index 0000000000000..82c1fdffe6869 --- /dev/null +++ b/mysql-test/main/func_json_invalid.result @@ -0,0 +1,692 @@ +# +# Behavioral baseline: how the JSON functions attest to input that is +# not valid JSON, is NULL, is not a string at all, or contains bytes +# that JSON cannot carry unescaped. +# +# For every case the recorded observable is the whole trichotomy: +# the value (or SQL NULL), and the warning or error with its code. +# Results are shown through HEX() whenever the answer contains bytes +# that are not printable ASCII. +# +# +# 1. Documents that are not valid JSON. +# +# --- truncated at every structural position --- +SELECT JSON_VALID('') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a"') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a":') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a":1') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a":1,') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('[') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('[1') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('[1,') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('"unterminated') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# --- structurally wrong --- +SELECT JSON_VALID('}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID(']') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID('{]') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('[}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('{a:1}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('{"a" 1}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 +SELECT JSON_VALID('{"a":1,,"b":2}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_VALID('{"a":1}{"b":2}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_VALID('{"a":1} trailing') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT JSON_VALID('nul') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('TRUE') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID('01') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('1.') AS v; +v +1 +SELECT JSON_VALID('.1') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID('1e') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('+1') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# --- bad escapes inside strings --- +SELECT JSON_VALID('{"a":"\\q"}') AS v; +v +1 +SELECT JSON_VALID('{"a":"\\u12"}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +SELECT JSON_VALID('{"a":"\\uZZZZ"}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_VALID('{"a":"trailing backslash\\"}') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# 2. The same invalid documents fed to each function. This records +# which functions answer NULL with a warning and which raise an +# error, and with which code. +# +SELECT JSON_SET('{"a":1,', '$.a', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT JSON_INSERT('{"a":1,', '$.b', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_insert' +SELECT JSON_REPLACE('{"a":1,', '$.a', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_replace' +SELECT JSON_REMOVE('{"a":1,', '$.a') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_remove' +SELECT JSON_ARRAY_APPEND('[1,', '$', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array_append' +SELECT JSON_ARRAY_INSERT('[1,', '$[0]', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array_insert' +SELECT JSON_MERGE('[1,', '[2]') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +SELECT JSON_MERGE('[1]', '[2,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE_PATCH('{"a":1,', '{"b":2}') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_patch' +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +SELECT JSON_EXTRACT('{"a":1,', '$.a') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_QUERY('{"a":1,', '$.a') AS v; +v +NULL +SELECT JSON_VALUE('{"a":1,', '$.a') AS v; +v +1 +SELECT JSON_KEYS('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +SELECT JSON_LENGTH('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_length' +SELECT JSON_DEPTH('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_depth' +SELECT JSON_TYPE('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_type' +SELECT JSON_EXISTS('{"a":1,', '$.a') AS v; +v +1 +SELECT JSON_CONTAINS('{"a":1,', '1', '$.a') AS v; +v +1 +SELECT JSON_CONTAINS_PATH('{"a":1,', 'one', '$.a') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_OVERLAPS('[1,', '[1]') AS v; +v +1 +SELECT JSON_EQUALS('{"a":1,', '{"a":1}') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_equals' +SELECT JSON_SEARCH('{"a":"x",', 'one', 'x') AS v; +v +"$.a" +SELECT JSON_NORMALIZE('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize' +SELECT JSON_COMPACT('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_compact' +SELECT JSON_LOOSE('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_loose' +SELECT JSON_DETAILED('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_detailed' +SELECT JSON_UNQUOTE('"unterminated') AS v; +v +"unterminated +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_unquote' +SELECT JSON_QUOTE('{"a":1,') AS v; +v +"{\"a\":1," +# +# 3. Invalid path arguments. +# +SELECT JSON_EXTRACT('{"a":1}', '') AS v; +v +NULL +Warnings: +Warning 4041 Unexpected end of JSON path in argument 2 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":1}', 'a') AS v; +v +NULL +Warnings: +Warning 4042 Syntax error in JSON path in argument 2 to function 'json_extract' at position 1 +SELECT JSON_EXTRACT('{"a":1}', '$..a') AS v; +v +NULL +Warnings: +Warning 4042 Syntax error in JSON path in argument 2 to function 'json_extract' at position 3 +SELECT JSON_EXTRACT('{"a":1}', '$.') AS v; +v +NULL +Warnings: +Warning 4041 Unexpected end of JSON path in argument 2 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":1}', '$[') AS v; +v +NULL +Warnings: +Warning 4041 Unexpected end of JSON path in argument 2 to function 'json_extract' +SELECT JSON_SET('{"a":1}', '$.*', 2) AS v; +v +NULL +Warnings: +Warning 4044 Wildcards or range in JSON path not allowed in argument 2 to function 'json_set' +SELECT JSON_SET('{"a":1}', '$**.a', 2) AS v; +v +NULL +Warnings: +Warning 4044 Wildcards or range in JSON path not allowed in argument 2 to function 'json_set' +# a path containing a quote, which the key comparison must not accept +SELECT JSON_EXTRACT('{"a\\"b":1}', '$.a"b') AS v; +v +NULL +SELECT JSON_SET('{"a":1}', '$.a"b', 2) AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 11 +SELECT JSON_INSERT('{"a":1}', '$.a"b', 2) AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 11 +# +# 4. NULL arguments in every position. +# +SELECT JSON_SET(NULL, '$.a', 1) AS v; +v +NULL +SELECT JSON_SET('{"a":1}', NULL, 1) AS v; +v +NULL +SELECT JSON_SET('{"a":1}', '$.a', NULL) AS v; +v +{"a": null} +SELECT JSON_INSERT(NULL, '$.a', 1) AS v; +v +NULL +SELECT JSON_REMOVE(NULL, '$.a') AS v; +v +NULL +SELECT JSON_REMOVE('{"a":1}', NULL) AS v; +v +NULL +SELECT JSON_MERGE(NULL, '[1]') AS v; +v +NULL +SELECT JSON_MERGE('[1]', NULL) AS v; +v +NULL +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS v; +v +NULL +SELECT JSON_MERGE_PATCH('{"a":1}', NULL) AS v; +v +NULL +SELECT JSON_EXTRACT(NULL, '$.a') AS v; +v +NULL +SELECT JSON_EXTRACT('{"a":1}', NULL) AS v; +v +NULL +SELECT JSON_ARRAY(NULL) AS v; +v +[null] +SELECT JSON_OBJECT('a', NULL) AS v; +v +{"a": null} +SELECT JSON_OBJECT(NULL, 1) AS v; +v +{"": 1} +SELECT JSON_KEYS(NULL) AS v; +v +NULL +SELECT JSON_LENGTH(NULL) AS v; +v +NULL +SELECT JSON_DEPTH(NULL) AS v; +v +NULL +SELECT JSON_TYPE(NULL) AS v; +v +NULL +SELECT JSON_QUOTE(NULL) AS v; +v +NULL +SELECT JSON_UNQUOTE(NULL) AS v; +v +NULL +SELECT JSON_VALID(NULL) AS v; +v +NULL +SELECT JSON_COMPACT(NULL) AS v; +v +NULL +# a JSON null inside a document is not an SQL NULL +SELECT JSON_EXTRACT('{"a":null}', '$.a') AS v, JSON_TYPE(JSON_EXTRACT('{"a":null}', '$.a')) AS t; +v t +null NULL +SELECT JSON_VALUE('{"a":null}', '$.a') AS v; +v +NULL +SELECT JSON_VALUE('{"a":null}', '$.a') IS NULL AS is_null; +is_null +1 +# +# 5. Arguments that are not strings. The document argument is taken +# through its string value, so the conversion is part of the result. +# +SELECT JSON_VALID(1) AS v, JSON_VALID(1.5) AS d, JSON_VALID(TRUE) AS b; +v d b +1 1 1 +SELECT JSON_TYPE(1) AS v, JSON_TYPE(1.5) AS d; +v d +INTEGER DOUBLE +SELECT JSON_EXTRACT(1, '$') AS v; +v +1 +SELECT JSON_SET(1, '$', 2) AS v; +v +2 +SELECT JSON_DEPTH(1) AS v; +v +1 +SELECT JSON_VALID(DATE'2020-01-01') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 +SELECT JSON_EXTRACT(DATE'2020-01-01', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 5 +# numeric and temporal values as inserted values, not documents +SELECT JSON_ARRAY(1, 1.5, -0.0, 1e300, DATE'2020-01-01', TIME'10:20:30') AS v; +v +[1, 1.5, 0.0, 1e300, "2020-01-01", "10:20:30"] +SELECT JSON_OBJECT('i', 1, 'd', 1.5, 'dt', DATE'2020-01-01') AS v; +v +{"i": 1, "d": 1.5, "dt": "2020-01-01"} +SELECT JSON_SET('{}', '$.i', 1, '$.d', 1.5, '$.dt', DATE'2020-01-01') AS v; +v +{"i": 1, "d": 1.5, "dt": "2020-01-01"} +# binary values +SELECT JSON_ARRAY(X'41') AS v, HEX(JSON_ARRAY(X'41')) AS h; +v h +["A"] 5B2241225D +SELECT JSON_ARRAY(X'00') AS v, HEX(JSON_ARRAY(X'00')) AS h; +v h +["\u0000"] 5B225C7530303030225D +SELECT JSON_VALID(X'7B7D') AS v; +v +1 +# +# 6. Hostile strings: control characters and bytes that JSON requires +# to be escaped. These pin what the constructors emit today for +# values that cannot be carried literally. +# +# --- control characters through the constructors --- +SELECT HEX(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS h; +h +5B225C7530303031225D +SELECT HEX(JSON_ARRAY(CHAR(9 USING utf8mb4))) AS h; +h +5B225C74225D +SELECT HEX(JSON_ARRAY(CHAR(10 USING utf8mb4))) AS h; +h +5B225C6E225D +SELECT HEX(JSON_ARRAY(CHAR(13 USING utf8mb4))) AS h; +h +5B225C72225D +SELECT HEX(JSON_ARRAY(CHAR(27 USING utf8mb4))) AS h; +h +5B225C7530303142225D +SELECT HEX(JSON_ARRAY(CHAR(31 USING utf8mb4))) AS h; +h +5B225C7530303146225D +SELECT HEX(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS h; +h +7B226B223A20225C7530303031227D +# a control character in the KEY position +SELECT HEX(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS h; +h +7B226B5C7530303031223A20317D +# is the constructor output accepted back as a document? +SELECT JSON_VALID(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS v; +v +1 +SELECT JSON_VALID(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS v; +v +1 +SELECT JSON_VALID(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS v; +v +1 +# the same values through the mutators +SELECT HEX(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS h; +h +7B2261223A20225C7530303031227D +SELECT JSON_VALID(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS v; +v +1 +SELECT HEX(JSON_ARRAY_APPEND('[]', '$', CHAR(1 USING utf8mb4))) AS h; +h +5B225C7530303031225D +# and through JSON_QUOTE, which escapes for exactly this purpose +SELECT HEX(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS h; +h +225C753030303122 +SELECT HEX(JSON_QUOTE(CHAR(9 USING utf8mb4))) AS h; +h +225C7422 +SELECT JSON_VALID(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS v; +v +1 +# --- quote and backslash edges --- +SELECT JSON_ARRAY('a"b') AS v, HEX(JSON_ARRAY('a"b')) AS h; +v h +["a\"b"] 5B22615C2262225D +SELECT JSON_ARRAY('a\\b') AS v, HEX(JSON_ARRAY('a\\b')) AS h; +v h +["a\\b"] 5B22615C5C62225D +SELECT JSON_ARRAY('a\\"b') AS v; +v +["a\\\"b"] +SELECT JSON_ARRAY('"') AS v; +v +["\""] +SELECT JSON_ARRAY('\\') AS v; +v +["\\"] +SELECT JSON_OBJECT('a"b', 1) AS v; +v +{"a\"b": 1} +SELECT JSON_OBJECT('a\\b', 1) AS v; +v +{"a\\b": 1} +SELECT JSON_VALID(JSON_ARRAY('a"b')) AS q, JSON_VALID(JSON_ARRAY('a\\b')) AS bs; +q bs +1 1 +SELECT JSON_VALID(JSON_OBJECT('a"b', 1)) AS q, JSON_VALID(JSON_OBJECT('a\\b', 1)) AS bs; +q bs +1 1 +SELECT JSON_QUOTE('a"b') AS v, JSON_QUOTE('a\\b') AS v2; +v v2 +"a\"b" "a\\b" +SELECT JSON_UNQUOTE('"a\\\\b"') AS v; +v +a\b +SELECT JSON_UNQUOTE(JSON_QUOTE('a"b')) AS v; +v +a"b +# a value that already looks like a quoted JSON string +SELECT JSON_ARRAY('"quoted"') AS v; +v +["\"quoted\""] +SELECT JSON_ARRAY('{"a":1}') AS v; +v +["{\"a\":1}"] +SELECT JSON_VALID(JSON_ARRAY('{"a":1}')) AS v; +v +1 +# --- byte sequences that are not valid in the connection charset --- +SET NAMES utf8mb4; +SELECT HEX(JSON_ARRAY(_utf8mb4 X'41')) AS h; +h +5B2241225D +SELECT HEX(JSON_ARRAY(_utf8mb4 X'FF')) AS h; +ERROR HY000: Invalid utf8mb4 character string: 'FF' +SELECT HEX(JSON_ARRAY(_utf8mb4 X'C3')) AS h; +ERROR HY000: Invalid utf8mb4 character string: 'C3' +# the same bytes carried in as binary and converted +SELECT HEX(JSON_ARRAY(CONVERT(X'FF' USING utf8mb4))) AS h; +h +5B223F225D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF' +SELECT HEX(JSON_ARRAY(CONVERT(X'C3' USING utf8mb4))) AS h; +h +5B223F225D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xC3' +SELECT HEX(JSON_ARRAY(CONVERT(X'C328' USING utf8mb4))) AS h; +h +5B223F28225D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xC3(' +SELECT HEX(JSON_ARRAY(CONVERT(X'EDA080' USING utf8mb4))) AS h; +h +5B22EDA080225D +SELECT HEX(JSON_QUOTE(CONVERT(X'FF' USING utf8mb4))) AS h; +h +223F22 +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF' +# invalid bytes inside a document argument +SELECT JSON_VALID(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4)) AS v; +v +1 +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF"}' +SELECT HEX(JSON_SET(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.b', 1)) AS h; +h +7B2261223A20223F222C202262223A20317D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF"}' +SELECT HEX(JSON_EXTRACT(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.a')) AS h; +h +223F22 +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF"}' +# an escape sequence naming a code point that utf8mb4 cannot encode +SELECT HEX(JSON_UNQUOTE('"\\ud800"')) AS h; +h +225C756438303022 +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_unquote' at position 7 +SELECT HEX(JSON_UNQUOTE('"\\ud800\\udc00"')) AS h; +h +F0908080 +SELECT JSON_VALID('"\\ud800"') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +# +# 7. Strict and non-strict sql_mode. The document argument never +# depends on sql_mode, but stores of the result do; both are +# recorded so that a later change of the store rule is visible. +# +CREATE TABLE t1 (a VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +Warnings: +Warning 1265 Data truncated for column 'a' at row 1 +SELECT a, JSON_VALID(a) AS still_valid FROM t1; +a still_valid +{"a": 9, " 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +DELETE FROM t1; +SET @@sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +ERROR 22001: Data too long for column 'a' at row 1 +SELECT COUNT(*) AS rows_stored FROM t1; +rows_stored +0 +SET @@sql_mode=DEFAULT; +DROP TABLE t1; +# +# 8. Errors raised before a document can be built at all, so that +# no partially built result is ever returned as a value. +# +SELECT JSON_OBJECT('a') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_OBJECT' +SELECT JSON_SET('{}') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_SET' +SELECT JSON_SET('{}', '$.a') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_SET' +SELECT JSON_REMOVE('{}') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_REMOVE' +SELECT JSON_KEYS() AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_KEYS' +SELECT JSON_MERGE('[1]') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_MERGE' diff --git a/mysql-test/main/func_json_invalid.test b/mysql-test/main/func_json_invalid.test new file mode 100644 index 0000000000000..1c224a1a89ee0 --- /dev/null +++ b/mysql-test/main/func_json_invalid.test @@ -0,0 +1,262 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: how the JSON functions attest to input that is +--echo # not valid JSON, is NULL, is not a string at all, or contains bytes +--echo # that JSON cannot carry unescaped. +--echo # +--echo # For every case the recorded observable is the whole trichotomy: +--echo # the value (or SQL NULL), and the warning or error with its code. +--echo # Results are shown through HEX() whenever the answer contains bytes +--echo # that are not printable ASCII. +--echo # + +--echo # +--echo # 1. Documents that are not valid JSON. +--echo # + +--echo # --- truncated at every structural position --- +SELECT JSON_VALID('') AS v; +SELECT JSON_VALID('{') AS v; +SELECT JSON_VALID('{"a"') AS v; +SELECT JSON_VALID('{"a":') AS v; +SELECT JSON_VALID('{"a":1') AS v; +SELECT JSON_VALID('{"a":1,') AS v; +SELECT JSON_VALID('[') AS v; +SELECT JSON_VALID('[1') AS v; +SELECT JSON_VALID('[1,') AS v; +SELECT JSON_VALID('"unterminated') AS v; + +--echo # --- structurally wrong --- +SELECT JSON_VALID('}') AS v; +SELECT JSON_VALID(']') AS v; +SELECT JSON_VALID('{]') AS v; +SELECT JSON_VALID('[}') AS v; +SELECT JSON_VALID('{a:1}') AS v; +SELECT JSON_VALID('{"a" 1}') AS v; +SELECT JSON_VALID('{"a":1,,"b":2}') AS v; +SELECT JSON_VALID('{"a":1}{"b":2}') AS v; +SELECT JSON_VALID('{"a":1} trailing') AS v; +SELECT JSON_VALID('nul') AS v; +SELECT JSON_VALID('TRUE') AS v; +SELECT JSON_VALID('01') AS v; +SELECT JSON_VALID('1.') AS v; +SELECT JSON_VALID('.1') AS v; +SELECT JSON_VALID('1e') AS v; +SELECT JSON_VALID('+1') AS v; + +--echo # --- bad escapes inside strings --- +SELECT JSON_VALID('{"a":"\\q"}') AS v; +SELECT JSON_VALID('{"a":"\\u12"}') AS v; +SELECT JSON_VALID('{"a":"\\uZZZZ"}') AS v; +SELECT JSON_VALID('{"a":"trailing backslash\\"}') AS v; + +--echo # +--echo # 2. The same invalid documents fed to each function. This records +--echo # which functions answer NULL with a warning and which raise an +--echo # error, and with which code. +--echo # + +SELECT JSON_SET('{"a":1,', '$.a', 2) AS v; +SELECT JSON_INSERT('{"a":1,', '$.b', 2) AS v; +SELECT JSON_REPLACE('{"a":1,', '$.a', 2) AS v; +SELECT JSON_REMOVE('{"a":1,', '$.a') AS v; +SELECT JSON_ARRAY_APPEND('[1,', '$', 2) AS v; +SELECT JSON_ARRAY_INSERT('[1,', '$[0]', 2) AS v; +SELECT JSON_MERGE('[1,', '[2]') AS v; +SELECT JSON_MERGE('[1]', '[2,') AS v; +SELECT JSON_MERGE_PATCH('{"a":1,', '{"b":2}') AS v; +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2,') AS v; +SELECT JSON_EXTRACT('{"a":1,', '$.a') AS v; +SELECT JSON_QUERY('{"a":1,', '$.a') AS v; +SELECT JSON_VALUE('{"a":1,', '$.a') AS v; +SELECT JSON_KEYS('{"a":1,') AS v; +SELECT JSON_LENGTH('{"a":1,') AS v; +SELECT JSON_DEPTH('{"a":1,') AS v; +SELECT JSON_TYPE('{"a":1,') AS v; +SELECT JSON_EXISTS('{"a":1,', '$.a') AS v; +SELECT JSON_CONTAINS('{"a":1,', '1', '$.a') AS v; +SELECT JSON_CONTAINS_PATH('{"a":1,', 'one', '$.a') AS v; +SELECT JSON_OVERLAPS('[1,', '[1]') AS v; +SELECT JSON_EQUALS('{"a":1,', '{"a":1}') AS v; +SELECT JSON_SEARCH('{"a":"x",', 'one', 'x') AS v; +SELECT JSON_NORMALIZE('{"a":1,') AS v; +SELECT JSON_COMPACT('{"a":1,') AS v; +SELECT JSON_LOOSE('{"a":1,') AS v; +SELECT JSON_DETAILED('{"a":1,') AS v; +SELECT JSON_UNQUOTE('"unterminated') AS v; +SELECT JSON_QUOTE('{"a":1,') AS v; + +--echo # +--echo # 3. Invalid path arguments. +--echo # + +SELECT JSON_EXTRACT('{"a":1}', '') AS v; +SELECT JSON_EXTRACT('{"a":1}', 'a') AS v; +SELECT JSON_EXTRACT('{"a":1}', '$..a') AS v; +SELECT JSON_EXTRACT('{"a":1}', '$.') AS v; +SELECT JSON_EXTRACT('{"a":1}', '$[') AS v; +SELECT JSON_SET('{"a":1}', '$.*', 2) AS v; +SELECT JSON_SET('{"a":1}', '$**.a', 2) AS v; +--echo # a path containing a quote, which the key comparison must not accept +SELECT JSON_EXTRACT('{"a\\"b":1}', '$.a"b') AS v; +SELECT JSON_SET('{"a":1}', '$.a"b', 2) AS v; +SELECT JSON_INSERT('{"a":1}', '$.a"b', 2) AS v; + +--echo # +--echo # 4. NULL arguments in every position. +--echo # + +SELECT JSON_SET(NULL, '$.a', 1) AS v; +SELECT JSON_SET('{"a":1}', NULL, 1) AS v; +SELECT JSON_SET('{"a":1}', '$.a', NULL) AS v; +SELECT JSON_INSERT(NULL, '$.a', 1) AS v; +SELECT JSON_REMOVE(NULL, '$.a') AS v; +SELECT JSON_REMOVE('{"a":1}', NULL) AS v; +SELECT JSON_MERGE(NULL, '[1]') AS v; +SELECT JSON_MERGE('[1]', NULL) AS v; +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS v; +SELECT JSON_MERGE_PATCH('{"a":1}', NULL) AS v; +SELECT JSON_EXTRACT(NULL, '$.a') AS v; +SELECT JSON_EXTRACT('{"a":1}', NULL) AS v; +SELECT JSON_ARRAY(NULL) AS v; +SELECT JSON_OBJECT('a', NULL) AS v; +SELECT JSON_OBJECT(NULL, 1) AS v; +SELECT JSON_KEYS(NULL) AS v; +SELECT JSON_LENGTH(NULL) AS v; +SELECT JSON_DEPTH(NULL) AS v; +SELECT JSON_TYPE(NULL) AS v; +SELECT JSON_QUOTE(NULL) AS v; +SELECT JSON_UNQUOTE(NULL) AS v; +SELECT JSON_VALID(NULL) AS v; +SELECT JSON_COMPACT(NULL) AS v; +--echo # a JSON null inside a document is not an SQL NULL +SELECT JSON_EXTRACT('{"a":null}', '$.a') AS v, JSON_TYPE(JSON_EXTRACT('{"a":null}', '$.a')) AS t; +SELECT JSON_VALUE('{"a":null}', '$.a') AS v; +SELECT JSON_VALUE('{"a":null}', '$.a') IS NULL AS is_null; + +--echo # +--echo # 5. Arguments that are not strings. The document argument is taken +--echo # through its string value, so the conversion is part of the result. +--echo # + +SELECT JSON_VALID(1) AS v, JSON_VALID(1.5) AS d, JSON_VALID(TRUE) AS b; +SELECT JSON_TYPE(1) AS v, JSON_TYPE(1.5) AS d; +SELECT JSON_EXTRACT(1, '$') AS v; +SELECT JSON_SET(1, '$', 2) AS v; +SELECT JSON_DEPTH(1) AS v; +SELECT JSON_VALID(DATE'2020-01-01') AS v; +SELECT JSON_EXTRACT(DATE'2020-01-01', '$') AS v; +--echo # numeric and temporal values as inserted values, not documents +SELECT JSON_ARRAY(1, 1.5, -0.0, 1e300, DATE'2020-01-01', TIME'10:20:30') AS v; +SELECT JSON_OBJECT('i', 1, 'd', 1.5, 'dt', DATE'2020-01-01') AS v; +SELECT JSON_SET('{}', '$.i', 1, '$.d', 1.5, '$.dt', DATE'2020-01-01') AS v; +--echo # binary values +SELECT JSON_ARRAY(X'41') AS v, HEX(JSON_ARRAY(X'41')) AS h; +SELECT JSON_ARRAY(X'00') AS v, HEX(JSON_ARRAY(X'00')) AS h; +SELECT JSON_VALID(X'7B7D') AS v; + +--echo # +--echo # 6. Hostile strings: control characters and bytes that JSON requires +--echo # to be escaped. These pin what the constructors emit today for +--echo # values that cannot be carried literally. +--echo # + +--echo # --- control characters through the constructors --- +SELECT HEX(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(9 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(10 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(13 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(27 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(31 USING utf8mb4))) AS h; +SELECT HEX(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS h; +--echo # a control character in the KEY position +SELECT HEX(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS h; +--echo # is the constructor output accepted back as a document? +SELECT JSON_VALID(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS v; +SELECT JSON_VALID(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS v; +SELECT JSON_VALID(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS v; +--echo # the same values through the mutators +SELECT HEX(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS h; +SELECT JSON_VALID(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS v; +SELECT HEX(JSON_ARRAY_APPEND('[]', '$', CHAR(1 USING utf8mb4))) AS h; +--echo # and through JSON_QUOTE, which escapes for exactly this purpose +SELECT HEX(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS h; +SELECT HEX(JSON_QUOTE(CHAR(9 USING utf8mb4))) AS h; +SELECT JSON_VALID(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS v; + +--echo # --- quote and backslash edges --- +SELECT JSON_ARRAY('a"b') AS v, HEX(JSON_ARRAY('a"b')) AS h; +SELECT JSON_ARRAY('a\\b') AS v, HEX(JSON_ARRAY('a\\b')) AS h; +SELECT JSON_ARRAY('a\\"b') AS v; +SELECT JSON_ARRAY('"') AS v; +SELECT JSON_ARRAY('\\') AS v; +SELECT JSON_OBJECT('a"b', 1) AS v; +SELECT JSON_OBJECT('a\\b', 1) AS v; +SELECT JSON_VALID(JSON_ARRAY('a"b')) AS q, JSON_VALID(JSON_ARRAY('a\\b')) AS bs; +SELECT JSON_VALID(JSON_OBJECT('a"b', 1)) AS q, JSON_VALID(JSON_OBJECT('a\\b', 1)) AS bs; +SELECT JSON_QUOTE('a"b') AS v, JSON_QUOTE('a\\b') AS v2; +SELECT JSON_UNQUOTE('"a\\\\b"') AS v; +SELECT JSON_UNQUOTE(JSON_QUOTE('a"b')) AS v; +--echo # a value that already looks like a quoted JSON string +SELECT JSON_ARRAY('"quoted"') AS v; +SELECT JSON_ARRAY('{"a":1}') AS v; +SELECT JSON_VALID(JSON_ARRAY('{"a":1}')) AS v; + +--echo # --- byte sequences that are not valid in the connection charset --- +SET NAMES utf8mb4; +SELECT HEX(JSON_ARRAY(_utf8mb4 X'41')) AS h; +--error ER_INVALID_CHARACTER_STRING +SELECT HEX(JSON_ARRAY(_utf8mb4 X'FF')) AS h; +--error ER_INVALID_CHARACTER_STRING +SELECT HEX(JSON_ARRAY(_utf8mb4 X'C3')) AS h; +--echo # the same bytes carried in as binary and converted +SELECT HEX(JSON_ARRAY(CONVERT(X'FF' USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CONVERT(X'C3' USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CONVERT(X'C328' USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CONVERT(X'EDA080' USING utf8mb4))) AS h; +SELECT HEX(JSON_QUOTE(CONVERT(X'FF' USING utf8mb4))) AS h; +--echo # invalid bytes inside a document argument +SELECT JSON_VALID(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4)) AS v; +SELECT HEX(JSON_SET(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.b', 1)) AS h; +SELECT HEX(JSON_EXTRACT(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.a')) AS h; +--echo # an escape sequence naming a code point that utf8mb4 cannot encode +SELECT HEX(JSON_UNQUOTE('"\\ud800"')) AS h; +SELECT HEX(JSON_UNQUOTE('"\\ud800\\udc00"')) AS h; +SELECT JSON_VALID('"\\ud800"') AS v; + +--echo # +--echo # 7. Strict and non-strict sql_mode. The document argument never +--echo # depends on sql_mode, but stores of the result do; both are +--echo # recorded so that a later change of the store rule is visible. +--echo # + +CREATE TABLE t1 (a VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT a, JSON_VALID(a) AS still_valid FROM t1; +DELETE FROM t1; +SET @@sql_mode='STRICT_ALL_TABLES'; +--error ER_DATA_TOO_LONG +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT COUNT(*) AS rows_stored FROM t1; +SET @@sql_mode=DEFAULT; +DROP TABLE t1; + +--echo # +--echo # 8. Errors raised before a document can be built at all, so that +--echo # no partially built result is ever returned as a value. +--echo # + +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_OBJECT('a') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_SET('{}') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_SET('{}', '$.a') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_REMOVE('{}') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_KEYS() AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_MERGE('[1]') AS v; diff --git a/mysql-test/main/func_json_mislabel.result b/mysql-test/main/func_json_mislabel.result new file mode 100644 index 0000000000000..838e18e5de54e --- /dev/null +++ b/mysql-test/main/func_json_mislabel.result @@ -0,0 +1,86 @@ +# +# Behavioral baseline: the same stored bytes read under a different +# character set label. +# +# Going through a binary column and back out again relabels a column +# without touching its contents, so the identical bytes are handed to +# the JSON parser as a different character set. This is the only way +# to put a document in front of the parser that is not valid in the +# character set it is claimed to be in - converting it would repair or +# reject it first. What each label makes of the same bytes is +# recorded here. +# +SET NAMES utf8mb4; +# +# 1. A gbk character whose second byte is the one JSON uses to escape. +# Read as gbk it is one letter inside a string; read as anything +# that treats bytes singly it starts an escape sequence and runs +# off the end of the document. +# +CREATE TABLE t1 (j VARCHAR(60) CHARACTER SET gbk); +INSERT INTO t1 VALUES (X'7B2261223A22815C227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_gbk FROM t1; +stored v_gbk +7B2261223A22815C227D 1 +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs +FROM t1; +ext cs +22815C22 gbk +ALTER TABLE t1 MODIFY j VARBINARY(60); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t1; +stored v_binary +7B2261223A22815C227D 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +ALTER TABLE t1 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_latin1 FROM t1; +stored v_latin1 +7B2261223A22815C227D 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t1; +ext +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +# +# 2. Bytes that are one character in utf8mb4 and two in latin1. The +# document stays parseable either way, but the value it yields is +# not the same. +# +CREATE TABLE t2 (j VARCHAR(60) CHARACTER SET utf8mb4); +INSERT INTO t2 VALUES (X'7B2261223A22C3A9227D'); +SELECT HEX(j) AS stored, HEX(JSON_EXTRACT(j, '$.a')) AS ext_utf8mb4, +CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_utf8mb4 FROM t2; +stored ext_utf8mb4 chars_utf8mb4 +7B2261223A22C3A9227D 22C3A922 1 +ALTER TABLE t2 MODIFY j VARBINARY(60); +ALTER TABLE t2 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored_unchanged, JSON_VALID(j) AS v FROM t2; +stored_unchanged v +7B2261223A22C3A9227D 1 +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs, +CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_latin1 FROM t2; +ext cs chars_latin1 +22C3A922 latin1 2 +# +# 3. Relabelling in the other direction, to a character set that +# cannot hold the bytes. This one does not go through quietly. +# +CREATE TABLE t3 (j VARBINARY(60)); +INSERT INTO t3 VALUES (X'7B2261223A22FF227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t3; +stored v_binary +7B2261223A22FF227D 1 +ALTER TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +ERROR 22007: Incorrect string value: '\xFF"}' for column `test`.`t3`.`j` at row 1 +ALTER IGNORE TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +Warnings: +Warning 1366 Incorrect string value: '\xFF"}' for column `test`.`t3`.`j` at row 1 +SELECT HEX(j) AS repaired, JSON_VALID(j) AS v FROM t3; +repaired v +7B2261223A223F227D 1 +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t3; +ext +223F22 +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/func_json_mislabel.test b/mysql-test/main/func_json_mislabel.test new file mode 100644 index 0000000000000..3913ee2dfea98 --- /dev/null +++ b/mysql-test/main/func_json_mislabel.test @@ -0,0 +1,67 @@ +--source include/have_gbk.inc +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: the same stored bytes read under a different +--echo # character set label. +--echo # +--echo # Going through a binary column and back out again relabels a column +--echo # without touching its contents, so the identical bytes are handed to +--echo # the JSON parser as a different character set. This is the only way +--echo # to put a document in front of the parser that is not valid in the +--echo # character set it is claimed to be in - converting it would repair or +--echo # reject it first. What each label makes of the same bytes is +--echo # recorded here. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A gbk character whose second byte is the one JSON uses to escape. +--echo # Read as gbk it is one letter inside a string; read as anything +--echo # that treats bytes singly it starts an escape sequence and runs +--echo # off the end of the document. +--echo # + +CREATE TABLE t1 (j VARCHAR(60) CHARACTER SET gbk); +INSERT INTO t1 VALUES (X'7B2261223A22815C227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_gbk FROM t1; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs + FROM t1; +ALTER TABLE t1 MODIFY j VARBINARY(60); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t1; +ALTER TABLE t1 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_latin1 FROM t1; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t1; + +--echo # +--echo # 2. Bytes that are one character in utf8mb4 and two in latin1. The +--echo # document stays parseable either way, but the value it yields is +--echo # not the same. +--echo # + +CREATE TABLE t2 (j VARCHAR(60) CHARACTER SET utf8mb4); +INSERT INTO t2 VALUES (X'7B2261223A22C3A9227D'); +SELECT HEX(j) AS stored, HEX(JSON_EXTRACT(j, '$.a')) AS ext_utf8mb4, + CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_utf8mb4 FROM t2; +ALTER TABLE t2 MODIFY j VARBINARY(60); +ALTER TABLE t2 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored_unchanged, JSON_VALID(j) AS v FROM t2; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs, + CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_latin1 FROM t2; + +--echo # +--echo # 3. Relabelling in the other direction, to a character set that +--echo # cannot hold the bytes. This one does not go through quietly. +--echo # + +CREATE TABLE t3 (j VARBINARY(60)); +INSERT INTO t3 VALUES (X'7B2261223A22FF227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t3; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +ALTER TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +ALTER IGNORE TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +SELECT HEX(j) AS repaired, JSON_VALID(j) AS v FROM t3; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t3; + +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/func_json_sp.result b/mysql-test/main/func_json_sp.result new file mode 100644 index 0000000000000..e03e282e517a8 --- /dev/null +++ b/mysql-test/main/func_json_sp.result @@ -0,0 +1,457 @@ +# +# Behavioral baseline: JSON values held in stored program variables. +# +# A variable declared JSON is not the same thing as a column declared +# JSON: it carries no check constraint and, as the first section +# shows, JSON functions do not treat it as a document. This test +# records that difference, what the declared type does to a value on +# assignment, and what happens when a variable is both the source and +# the target of the same statement. +# +SET NAMES utf8mb4; +# +# 1. A variable declared JSON compared with a column declared JSON. +# A JSON column is embedded into a constructor as a document; the +# variable is quoted like any other string. +# +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'); +CREATE PROCEDURE p_decl() +BEGIN +DECLARE v JSON DEFAULT '{"a":1}'; +DECLARE t LONGTEXT DEFAULT '{"a":1}'; +SELECT v AS var_json; +SELECT JSON_ARRAY(v) AS from_json_var; +SELECT JSON_ARRAY(t) AS from_text_var; +SELECT JSON_ARRAY(j) AS from_json_column FROM t1; +SELECT JSON_OBJECT('k', v) AS obj_from_json_var; +SELECT JSON_OBJECT('k', j) AS obj_from_json_column FROM t1; +SELECT JSON_VALID(JSON_ARRAY(v)) AS var_still_valid; +SELECT JSON_VALID(JSON_ARRAY(j)) AS col_still_valid FROM t1; +SELECT JSON_SET(v, '$.b', 2) AS set_over_var; +SELECT JSON_EXTRACT(v, '$.a') AS extract_from_var; +SELECT JSON_TYPE(v) AS type_of_var; +END $$ +CALL p_decl(); +var_json +{"a":1} +from_json_var +["{\"a\":1}"] +from_text_var +["{\"a\":1}"] +from_json_column +[{"a":1}] +obj_from_json_var +{"k": "{\"a\":1}"} +obj_from_json_column +{"k": {"a":1}} +var_still_valid +1 +col_still_valid +1 +set_over_var +{"a": 1, "b": 2} +extract_from_var +1 +type_of_var +OBJECT +DROP PROCEDURE p_decl; +# the declared type as the server records it +CREATE FUNCTION f_ret_json() RETURNS JSON +BEGIN +RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +SELECT f_ret_json() AS ret; +ret +{"a": 1, "b": 2} +SELECT JSON_ARRAY(f_ret_json()) AS embedded; +embedded +["{\"a\": 1, \"b\": 2}"] +SELECT JSON_VALID(JSON_ARRAY(f_ret_json())) AS still_valid; +still_valid +1 +SELECT JSON_TYPE(f_ret_json()) AS t; +t +OBJECT +DROP FUNCTION f_ret_json; +# a function declared to return LONGTEXT, for comparison +CREATE FUNCTION f_ret_text() RETURNS LONGTEXT +BEGIN +RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +SELECT f_ret_text() AS ret; +ret +{"a": 1, "b": 2} +SELECT JSON_ARRAY(f_ret_text()) AS embedded; +embedded +["{\"a\": 1, \"b\": 2}"] +DROP FUNCTION f_ret_text; +# +# 2. What the declared type does to a value on assignment. +# +# a value that fits, and the same value converted to other types +CREATE PROCEDURE p_types() +BEGIN +DECLARE longer VARCHAR(100); +DECLARE num INT; +DECLARE bin VARBINARY(100); +SET longer = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT longer AS intact, JSON_VALID(longer) AS still_valid; +SET num = JSON_EXTRACT('{"a":42}', '$.a'); +SELECT num AS as_number; +SET bin = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT HEX(bin) AS as_binary, JSON_VALID(bin) AS still_valid; +END $$ +CALL p_types(); +intact still_valid +{"a": 9, "b": 2} 1 +as_number +42 +as_binary still_valid +7B2261223A20392C202262223A20327D 1 +DROP PROCEDURE p_types; +# a value too long for the variable. A stored program runs under the +# sql_mode it was CREATED with, so setting the mode around the CALL +# decides nothing and each mode needs a procedure of its own. In +# strict mode the assignment is refused; in non-strict mode it +# truncates silently - no error and no warning - leaving a value that +# is no longer a document. +SET @@sql_mode='STRICT_ALL_TABLES'; +CREATE PROCEDURE p_cut_strict() +BEGIN +DECLARE v VARCHAR(8); +SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +SET @@sql_mode=''; +CREATE PROCEDURE p_cut_lax() +BEGIN +DECLARE v VARCHAR(8); +SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +SET @@sql_mode=DEFAULT; +# the one created in strict mode, called under each mode in turn +SET @@sql_mode=''; +CALL p_cut_strict(); +ERROR 22001: Data too long for column 'v' at row 0 +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_cut_strict(); +ERROR 22001: Data too long for column 'v' at row 0 +SET @@sql_mode=DEFAULT; +# the one created in non-strict mode, the same way +SET @@sql_mode=''; +CALL p_cut_lax(); +cut h still_valid +{"a": 9, 7B2261223A20392C 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_cut_lax(); +cut h still_valid +{"a": 9, 7B2261223A20392C 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_cut_strict; +DROP PROCEDURE p_cut_lax; +# the same value into a column of the same width, for contrast +CREATE TABLE t_narrow (v VARCHAR(8)); +SET @@sql_mode=''; +INSERT INTO t_narrow VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +Warnings: +Warning 1265 Data truncated for column 'v' at row 1 +SELECT v, HEX(v) AS h, JSON_VALID(v) AS still_valid FROM t_narrow; +v h still_valid +{"a": 9, 7B2261223A20392C 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode=DEFAULT; +DROP TABLE t_narrow; +# an assignment that loses characters rather than length: the value +# fits, but the variable's character set cannot carry it. +CREATE PROCEDURE p_lossy_latin1() +BEGIN +DECLARE v VARCHAR(100) CHARACTER SET latin1; +SET v = JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4)); +SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +CREATE PROCEDURE p_lossy_swe7() +BEGIN +DECLARE w VARCHAR(100) CHARACTER SET swe7; +SET w = JSON_SET('{"a":1}', '$.b', 2); +SELECT HEX(w) AS as_swe7, JSON_VALID(w) AS still_valid; +END $$ +CREATE PROCEDURE p_lossless_latin1() +BEGIN +DECLARE v VARCHAR(100) CHARACTER SET latin1; +SET v = JSON_SET('{"a":1}', '$.b', 2); +SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +SET @@sql_mode=''; +CALL p_lossy_latin1(); +ERROR 22007: Incorrect string value: '\xE4\xBD\xA0"}' for column ``.``.`v` at row 0 +CALL p_lossy_swe7(); +ERROR 22007: Incorrect string value: '{"a": ...' for column ``.``.`w` at row 0 +CALL p_lossless_latin1(); +as_latin1 still_valid +7B2261223A20312C202262223A20327D 1 +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_lossy_latin1(); +ERROR 22007: Incorrect string value: '\xE4\xBD\xA0"}' for column ``.``.`v` at row 0 +CALL p_lossy_swe7(); +ERROR 22007: Incorrect string value: '{"a": ...' for column ``.``.`w` at row 0 +CALL p_lossless_latin1(); +as_latin1 still_valid +7B2261223A20312C202262223A20327D 1 +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_lossy_latin1; +DROP PROCEDURE p_lossy_swe7; +DROP PROCEDURE p_lossless_latin1; +# the same lossy value into a column, which is where non-strict mode +# keeps the value instead of refusing it +CREATE TABLE t_lossy (v VARCHAR(100) CHARACTER SET latin1); +SET @@sql_mode=''; +INSERT INTO t_lossy +VALUES (JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4))); +Warnings: +Warning 1366 Incorrect string value: '\xE4\xBD\xA0"}' for column `test`.`t_lossy`.`v` at row 1 +SELECT HEX(v) AS stored, JSON_VALID(v) AS still_valid FROM t_lossy; +stored still_valid +7B2261223A20312C202262223A20223F227D 1 +SET @@sql_mode=DEFAULT; +DROP TABLE t_lossy; +# +# 3. A variable that is both the source and the target. +# +CREATE PROCEDURE p_self() +BEGIN +DECLARE v JSON DEFAULT '{"a":1}'; +SET v = JSON_SET(v, '$.b', 2); +SELECT v AS after_one; +SET v = JSON_SET(v, '$.c', 3); +SELECT v AS after_two; +SET v = JSON_INSERT(v, '$.d', 4); +SELECT v AS after_insert; +SET v = JSON_REMOVE(v, '$.a'); +SELECT v AS after_remove; +SET v = JSON_MERGE(v, '{"e":5}'); +SELECT v AS after_merge; +SET v = JSON_ARRAY(v); +SELECT v AS after_wrap; +SET v = JSON_EXTRACT(v, '$[0]'); +SELECT v AS after_extract; +SELECT JSON_VALID(v) AS still_valid; +END $$ +CALL p_self(); +after_one +{"a": 1, "b": 2} +after_two +{"a": 1, "b": 2, "c": 3} +after_insert +{"a": 1, "b": 2, "c": 3, "d": 4} +after_remove +{"b": 2, "c": 3, "d": 4} +after_merge +{"b": 2, "c": 3, "d": 4, "e": 5} +after_wrap +["{\"b\": 2, \"c\": 3, \"d\": 4, \"e\": 5}"] +after_extract +"{\"b\": 2, \"c\": 3, \"d\": 4, \"e\": 5}" +still_valid +1 +DROP PROCEDURE p_self; +# a long document repeatedly rewritten in place, so the variable's +# storage has to grow and be reused +CREATE PROCEDURE p_grow() +BEGIN +DECLARE v JSON DEFAULT '{"a":1}'; +DECLARE i INT DEFAULT 0; +WHILE i < 20 DO +SET v = JSON_SET(v, CONCAT('$.k', i), REPEAT('x', 50)); +SET i = i + 1; +END WHILE; +SELECT LENGTH(v) AS len, JSON_VALID(v) AS still_valid, JSON_LENGTH(v) AS members; +SELECT JSON_EXTRACT(v, '$.k0') AS first_added; +SELECT JSON_EXTRACT(v, '$.k19') AS last_added; +END $$ +CALL p_grow(); +len still_valid members +1218 1 21 +first_added +"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +last_added +"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +DROP PROCEDURE p_grow; +# +# 4. Variables passed in and out of procedures. +# +CREATE PROCEDURE p_out(IN a JSON, OUT b JSON, INOUT c JSON) +BEGIN +SET b = JSON_SET(a, '$.from_in', 1); +SET c = JSON_SET(c, '$.from_inout', 1); +SELECT JSON_ARRAY(a) AS in_embedded, JSON_ARRAY(c) AS inout_embedded; +END $$ +SET @in = '{"a":1}'; +SET @out = NULL; +SET @inout = '{"c":1}'; +CALL p_out(@in, @out, @inout); +in_embedded inout_embedded +["{\"a\":1}"] ["{\"c\": 1, \"from_inout\": 1}"] +SELECT @out AS out_value, @inout AS inout_value; +out_value inout_value +{"a": 1, "from_in": 1} {"c": 1, "from_inout": 1} +SELECT JSON_VALID(@out) AS out_valid, JSON_VALID(@inout) AS inout_valid; +out_valid inout_valid +1 1 +SELECT JSON_ARRAY(@out) AS user_var_embedded; +user_var_embedded +["{\"a\": 1, \"from_in\": 1}"] +SELECT JSON_VALID(JSON_ARRAY(@out)) AS still_valid; +still_valid +1 +DROP PROCEDURE p_out; +# a variable holding a document that is not valid +SET @bad = '{"a":1,'; +SELECT JSON_VALID(@bad) AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_ARRAY(@bad) AS embedded; +embedded +["{\"a\":1,"] +SELECT JSON_VALID(JSON_ARRAY(@bad)) AS still_valid; +still_valid +1 +SELECT JSON_SET(@bad, '$.b', 1) AS mutated; +mutated +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +# +# 5. Variables inside a ROW, and a variable fed from a cursor. +# +CREATE PROCEDURE p_row() +BEGIN +DECLARE r ROW (a JSON, b LONGTEXT); +SET r.a = JSON_SET('{"a":1}', '$.b', 2); +SET r.b = JSON_SET('{"a":1}', '$.b', 2); +SELECT r.a AS row_json, r.b AS row_text; +SELECT JSON_ARRAY(r.a) AS from_row_json, JSON_ARRAY(r.b) AS from_row_text; +SELECT JSON_VALID(r.a) AS a_valid, JSON_VALID(r.b) AS b_valid; +SET r.a = JSON_SET(r.a, '$.c', 3); +SELECT r.a AS after_self_assign; +END $$ +CALL p_row(); +row_json row_text +{"a": 1, "b": 2} {"a": 1, "b": 2} +from_row_json from_row_text +[{"a": 1, "b": 2}] ["{\"a\": 1, \"b\": 2}"] +a_valid b_valid +1 1 +after_self_assign +{"a": 1, "b": 2, "c": 3} +DROP PROCEDURE p_row; +CREATE PROCEDURE p_cursor() +BEGIN +DECLARE done INT DEFAULT 0; +DECLARE v JSON; +DECLARE cur CURSOR FOR SELECT j FROM t1; +DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1; +OPEN cur; +FETCH cur INTO v; +CLOSE cur; +SELECT v AS fetched; +SELECT JSON_ARRAY(v) AS embedded; +SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; +SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +CALL p_cursor(); +fetched +{"a":1} +embedded +["{\"a\":1}"] +still_valid +1 +mutated +{"a": 1, "b": 2} +DROP PROCEDURE p_cursor; +# a cursor over a column that no longer satisfies its check +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +CREATE PROCEDURE p_cursor_bad() +BEGIN +DECLARE v JSON; +DECLARE cur CURSOR FOR SELECT j FROM t1 WHERE j = '{"a":1,'; +OPEN cur; +FETCH cur INTO v; +CLOSE cur; +SELECT v AS fetched, JSON_VALID(v) AS valid; +SELECT JSON_ARRAY(v) AS embedded; +SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +CALL p_cursor_bad(); +fetched valid +{"a":1, 0 +embedded +["{\"a\":1,"] +mutated +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +DROP PROCEDURE p_cursor_bad; +# +# 6. A variable read several times in one call, including through a +# stored function that reads it again while the outer call is still +# evaluating. No SQL-visible way to REASSIGN the variable from +# inside an argument has been identified; if one is found, the +# aliasing shape belongs here too. +# +CREATE FUNCTION f_side(x JSON) RETURNS INT +BEGIN +RETURN JSON_LENGTH(x); +END $$ +CREATE PROCEDURE p_multi() +BEGIN +DECLARE v JSON DEFAULT '{"a":1,"b":2}'; +SELECT JSON_SET(v, '$.c', f_side(v)) AS both_uses; +SELECT JSON_MERGE(v, v) AS twice; +SELECT JSON_ARRAY(v, v) AS twice_embedded; +SELECT JSON_SET(v, '$.x', 1, '$.y', JSON_EXTRACT(v, '$.a')) AS mixed; +END $$ +CALL p_multi(); +both_uses +{"a": 1, "b": 2, "c": 2} +twice +{"a": [1, 1], "b": [2, 2]} +twice_embedded +["{\"a\":1,\"b\":2}", "{\"a\":1,\"b\":2}"] +mixed +{"a": 1, "b": 2, "x": 1, "y": 1} +DROP PROCEDURE p_multi; +DROP FUNCTION f_side; +# +# 7. A trigger, where the row values are columns rather than variables. +# +CREATE TABLE t2 (j JSON, note LONGTEXT); +CREATE TRIGGER tr BEFORE INSERT ON t2 FOR EACH ROW +BEGIN +SET NEW.note = JSON_ARRAY(NEW.j); +END $$ +INSERT INTO t2 (j) VALUES ('{"a":1}'); +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2; +j note note_valid +{"a":1} [{"a":1}] 1 +SET SESSION check_constraint_checks = OFF; +INSERT INTO t2 (j) VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2 ORDER BY j; +j note note_valid +{"a":1, [{"a":1,] 0 +{"a":1} [{"a":1}] 1 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +DROP TABLE t2; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp.test b/mysql-test/main/func_json_sp.test new file mode 100644 index 0000000000000..5bc533e198cda --- /dev/null +++ b/mysql-test/main/func_json_sp.test @@ -0,0 +1,371 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: JSON values held in stored program variables. +--echo # +--echo # A variable declared JSON is not the same thing as a column declared +--echo # JSON: it carries no check constraint and, as the first section +--echo # shows, JSON functions do not treat it as a document. This test +--echo # records that difference, what the declared type does to a value on +--echo # assignment, and what happens when a variable is both the source and +--echo # the target of the same statement. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A variable declared JSON compared with a column declared JSON. +--echo # A JSON column is embedded into a constructor as a document; the +--echo # variable is quoted like any other string. +--echo # + +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'); + +--delimiter $$ +CREATE PROCEDURE p_decl() +BEGIN + DECLARE v JSON DEFAULT '{"a":1}'; + DECLARE t LONGTEXT DEFAULT '{"a":1}'; + SELECT v AS var_json; + SELECT JSON_ARRAY(v) AS from_json_var; + SELECT JSON_ARRAY(t) AS from_text_var; + SELECT JSON_ARRAY(j) AS from_json_column FROM t1; + SELECT JSON_OBJECT('k', v) AS obj_from_json_var; + SELECT JSON_OBJECT('k', j) AS obj_from_json_column FROM t1; + SELECT JSON_VALID(JSON_ARRAY(v)) AS var_still_valid; + SELECT JSON_VALID(JSON_ARRAY(j)) AS col_still_valid FROM t1; + SELECT JSON_SET(v, '$.b', 2) AS set_over_var; + SELECT JSON_EXTRACT(v, '$.a') AS extract_from_var; + SELECT JSON_TYPE(v) AS type_of_var; +END $$ +--delimiter ; +CALL p_decl(); +DROP PROCEDURE p_decl; + +--echo # the declared type as the server records it +--delimiter $$ +CREATE FUNCTION f_ret_json() RETURNS JSON +BEGIN + RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +--delimiter ; +SELECT f_ret_json() AS ret; +SELECT JSON_ARRAY(f_ret_json()) AS embedded; +SELECT JSON_VALID(JSON_ARRAY(f_ret_json())) AS still_valid; +SELECT JSON_TYPE(f_ret_json()) AS t; +DROP FUNCTION f_ret_json; + +--echo # a function declared to return LONGTEXT, for comparison +--delimiter $$ +CREATE FUNCTION f_ret_text() RETURNS LONGTEXT +BEGIN + RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +--delimiter ; +SELECT f_ret_text() AS ret; +SELECT JSON_ARRAY(f_ret_text()) AS embedded; +DROP FUNCTION f_ret_text; + +--echo # +--echo # 2. What the declared type does to a value on assignment. +--echo # + +--echo # a value that fits, and the same value converted to other types +--delimiter $$ +CREATE PROCEDURE p_types() +BEGIN + DECLARE longer VARCHAR(100); + DECLARE num INT; + DECLARE bin VARBINARY(100); + SET longer = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT longer AS intact, JSON_VALID(longer) AS still_valid; + SET num = JSON_EXTRACT('{"a":42}', '$.a'); + SELECT num AS as_number; + SET bin = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT HEX(bin) AS as_binary, JSON_VALID(bin) AS still_valid; +END $$ +--delimiter ; +CALL p_types(); +DROP PROCEDURE p_types; + +--echo # a value too long for the variable. A stored program runs under the +--echo # sql_mode it was CREATED with, so setting the mode around the CALL +--echo # decides nothing and each mode needs a procedure of its own. In +--echo # strict mode the assignment is refused; in non-strict mode it +--echo # truncates silently - no error and no warning - leaving a value that +--echo # is no longer a document. +SET @@sql_mode='STRICT_ALL_TABLES'; +--delimiter $$ +CREATE PROCEDURE p_cut_strict() +BEGIN + DECLARE v VARCHAR(8); + SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +SET @@sql_mode=''; +--delimiter $$ +CREATE PROCEDURE p_cut_lax() +BEGIN + DECLARE v VARCHAR(8); + SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +SET @@sql_mode=DEFAULT; +--echo # the one created in strict mode, called under each mode in turn +SET @@sql_mode=''; +--error ER_DATA_TOO_LONG +CALL p_cut_strict(); +SET @@sql_mode='STRICT_ALL_TABLES'; +--error ER_DATA_TOO_LONG +CALL p_cut_strict(); +SET @@sql_mode=DEFAULT; +--echo # the one created in non-strict mode, the same way +SET @@sql_mode=''; +CALL p_cut_lax(); +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_cut_lax(); +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_cut_strict; +DROP PROCEDURE p_cut_lax; +--echo # the same value into a column of the same width, for contrast +CREATE TABLE t_narrow (v VARCHAR(8)); +SET @@sql_mode=''; +INSERT INTO t_narrow VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT v, HEX(v) AS h, JSON_VALID(v) AS still_valid FROM t_narrow; +SET @@sql_mode=DEFAULT; +DROP TABLE t_narrow; + +--echo # an assignment that loses characters rather than length: the value +--echo # fits, but the variable's character set cannot carry it. +--delimiter $$ +CREATE PROCEDURE p_lossy_latin1() +BEGIN + DECLARE v VARCHAR(100) CHARACTER SET latin1; + SET v = JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4)); + SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +CREATE PROCEDURE p_lossy_swe7() +BEGIN + DECLARE w VARCHAR(100) CHARACTER SET swe7; + SET w = JSON_SET('{"a":1}', '$.b', 2); + SELECT HEX(w) AS as_swe7, JSON_VALID(w) AS still_valid; +END $$ +CREATE PROCEDURE p_lossless_latin1() +BEGIN + DECLARE v VARCHAR(100) CHARACTER SET latin1; + SET v = JSON_SET('{"a":1}', '$.b', 2); + SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +SET @@sql_mode=''; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_latin1(); +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_swe7(); +CALL p_lossless_latin1(); +SET @@sql_mode='STRICT_ALL_TABLES'; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_latin1(); +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_swe7(); +CALL p_lossless_latin1(); +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_lossy_latin1; +DROP PROCEDURE p_lossy_swe7; +DROP PROCEDURE p_lossless_latin1; +--echo # the same lossy value into a column, which is where non-strict mode +--echo # keeps the value instead of refusing it +CREATE TABLE t_lossy (v VARCHAR(100) CHARACTER SET latin1); +SET @@sql_mode=''; +INSERT INTO t_lossy + VALUES (JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4))); +SELECT HEX(v) AS stored, JSON_VALID(v) AS still_valid FROM t_lossy; +SET @@sql_mode=DEFAULT; +DROP TABLE t_lossy; + +--echo # +--echo # 3. A variable that is both the source and the target. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_self() +BEGIN + DECLARE v JSON DEFAULT '{"a":1}'; + SET v = JSON_SET(v, '$.b', 2); + SELECT v AS after_one; + SET v = JSON_SET(v, '$.c', 3); + SELECT v AS after_two; + SET v = JSON_INSERT(v, '$.d', 4); + SELECT v AS after_insert; + SET v = JSON_REMOVE(v, '$.a'); + SELECT v AS after_remove; + SET v = JSON_MERGE(v, '{"e":5}'); + SELECT v AS after_merge; + SET v = JSON_ARRAY(v); + SELECT v AS after_wrap; + SET v = JSON_EXTRACT(v, '$[0]'); + SELECT v AS after_extract; + SELECT JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +CALL p_self(); +DROP PROCEDURE p_self; + +--echo # a long document repeatedly rewritten in place, so the variable's +--echo # storage has to grow and be reused +--delimiter $$ +CREATE PROCEDURE p_grow() +BEGIN + DECLARE v JSON DEFAULT '{"a":1}'; + DECLARE i INT DEFAULT 0; + WHILE i < 20 DO + SET v = JSON_SET(v, CONCAT('$.k', i), REPEAT('x', 50)); + SET i = i + 1; + END WHILE; + SELECT LENGTH(v) AS len, JSON_VALID(v) AS still_valid, JSON_LENGTH(v) AS members; + SELECT JSON_EXTRACT(v, '$.k0') AS first_added; + SELECT JSON_EXTRACT(v, '$.k19') AS last_added; +END $$ +--delimiter ; +CALL p_grow(); +DROP PROCEDURE p_grow; + +--echo # +--echo # 4. Variables passed in and out of procedures. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_out(IN a JSON, OUT b JSON, INOUT c JSON) +BEGIN + SET b = JSON_SET(a, '$.from_in', 1); + SET c = JSON_SET(c, '$.from_inout', 1); + SELECT JSON_ARRAY(a) AS in_embedded, JSON_ARRAY(c) AS inout_embedded; +END $$ +--delimiter ; +SET @in = '{"a":1}'; +SET @out = NULL; +SET @inout = '{"c":1}'; +CALL p_out(@in, @out, @inout); +SELECT @out AS out_value, @inout AS inout_value; +SELECT JSON_VALID(@out) AS out_valid, JSON_VALID(@inout) AS inout_valid; +SELECT JSON_ARRAY(@out) AS user_var_embedded; +SELECT JSON_VALID(JSON_ARRAY(@out)) AS still_valid; +DROP PROCEDURE p_out; + +--echo # a variable holding a document that is not valid +SET @bad = '{"a":1,'; +SELECT JSON_VALID(@bad) AS v; +SELECT JSON_ARRAY(@bad) AS embedded; +SELECT JSON_VALID(JSON_ARRAY(@bad)) AS still_valid; +SELECT JSON_SET(@bad, '$.b', 1) AS mutated; + +--echo # +--echo # 5. Variables inside a ROW, and a variable fed from a cursor. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_row() +BEGIN + DECLARE r ROW (a JSON, b LONGTEXT); + SET r.a = JSON_SET('{"a":1}', '$.b', 2); + SET r.b = JSON_SET('{"a":1}', '$.b', 2); + SELECT r.a AS row_json, r.b AS row_text; + SELECT JSON_ARRAY(r.a) AS from_row_json, JSON_ARRAY(r.b) AS from_row_text; + SELECT JSON_VALID(r.a) AS a_valid, JSON_VALID(r.b) AS b_valid; + SET r.a = JSON_SET(r.a, '$.c', 3); + SELECT r.a AS after_self_assign; +END $$ +--delimiter ; +CALL p_row(); +DROP PROCEDURE p_row; + +--delimiter $$ +CREATE PROCEDURE p_cursor() +BEGIN + DECLARE done INT DEFAULT 0; + DECLARE v JSON; + DECLARE cur CURSOR FOR SELECT j FROM t1; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1; + OPEN cur; + FETCH cur INTO v; + CLOSE cur; + SELECT v AS fetched; + SELECT JSON_ARRAY(v) AS embedded; + SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; + SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +--delimiter ; +CALL p_cursor(); +DROP PROCEDURE p_cursor; + +--echo # a cursor over a column that no longer satisfies its check +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +--delimiter $$ +CREATE PROCEDURE p_cursor_bad() +BEGIN + DECLARE v JSON; + DECLARE cur CURSOR FOR SELECT j FROM t1 WHERE j = '{"a":1,'; + OPEN cur; + FETCH cur INTO v; + CLOSE cur; + SELECT v AS fetched, JSON_VALID(v) AS valid; + SELECT JSON_ARRAY(v) AS embedded; + SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +--delimiter ; +CALL p_cursor_bad(); +DROP PROCEDURE p_cursor_bad; + +--echo # +--echo # 6. A variable read several times in one call, including through a +--echo # stored function that reads it again while the outer call is still +--echo # evaluating. No SQL-visible way to REASSIGN the variable from +--echo # inside an argument has been identified; if one is found, the +--echo # aliasing shape belongs here too. +--echo # + +--delimiter $$ +CREATE FUNCTION f_side(x JSON) RETURNS INT +BEGIN + RETURN JSON_LENGTH(x); +END $$ +--delimiter ; +--delimiter $$ +CREATE PROCEDURE p_multi() +BEGIN + DECLARE v JSON DEFAULT '{"a":1,"b":2}'; + SELECT JSON_SET(v, '$.c', f_side(v)) AS both_uses; + SELECT JSON_MERGE(v, v) AS twice; + SELECT JSON_ARRAY(v, v) AS twice_embedded; + SELECT JSON_SET(v, '$.x', 1, '$.y', JSON_EXTRACT(v, '$.a')) AS mixed; +END $$ +--delimiter ; +CALL p_multi(); +DROP PROCEDURE p_multi; +DROP FUNCTION f_side; + +--echo # +--echo # 7. A trigger, where the row values are columns rather than variables. +--echo # + +CREATE TABLE t2 (j JSON, note LONGTEXT); +--delimiter $$ +CREATE TRIGGER tr BEFORE INSERT ON t2 FOR EACH ROW +BEGIN + SET NEW.note = JSON_ARRAY(NEW.j); +END $$ +--delimiter ; +INSERT INTO t2 (j) VALUES ('{"a":1}'); +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2; +SET SESSION check_constraint_checks = OFF; +INSERT INTO t2 (j) VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2 ORDER BY j; +DROP TABLE t2; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_table.result b/mysql-test/main/func_json_table.result new file mode 100644 index 0000000000000..5cec1551fd48d --- /dev/null +++ b/mysql-test/main/func_json_table.result @@ -0,0 +1,87 @@ +# +# Behavioral baseline: JSON_TABLE. +# +# JSON_TABLE parses its document itself and presents the pieces as +# columns of a table that only exists for the duration of the query. +# This test records what those columns are treated as when they are +# fed back into JSON functions, what happens when the document comes +# from a column whose stored bytes are not valid, and what a DEFAULT +# clause is allowed to put into a column declared JSON. +# +SET NAMES utf8mb4; +# +# 1. A column declared JSON, consumed by JSON functions. Compare the +# embedding decision here with the one a JSON table column gets in +# func_json_columns. +# +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('[{"a": {"x": 1}}, {"a":{"y":2}}]'); +# the fragment keeps the spacing it had in the input document +SELECT jt.frag AS v, HEX(jt.frag) AS h +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +v h +{"x": 1} 7B2278223A2020317D +{"y":2} 7B2279223A327D +# fed to a constructor it is quoted, not embedded +SELECT JSON_ARRAY(jt.frag) AS embedded, JSON_VALID(JSON_ARRAY(jt.frag)) AS ok +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +embedded ok +["{\"x\": 1}"] 1 +["{\"y\":2}"] 1 +SELECT JSON_OBJECT('k', jt.frag) AS v +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +v +{"k": "{\"x\": 1}"} +{"k": "{\"y\":2}"} +# but a mutator and JSON_TYPE both take it as a document +SELECT JSON_SET(jt.frag, '$.z', 3) AS v, JSON_TYPE(jt.frag) AS t +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +v t +{"x": 1, "z": 3} OBJECT +{"y": 2, "z": 3} OBJECT +# a column declared VARCHAR over the same path +SELECT JSON_ARRAY(jt.txt) AS quoted +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (txt VARCHAR(50) PATH '$.a' NULL ON ERROR)) jt; +quoted +[null] +[null] +DROP TABLE t1; +# +# 2. The document coming from a column whose stored bytes no longer +# satisfy its check constraint. +# +CREATE TABLE tp (id INT PRIMARY KEY, j JSON); +INSERT INTO tp VALUES (1, '[{"a":1},{"a":2}]'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES (2, '[{"a":1},'), (3, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))); +SET SESSION check_constraint_checks = ON; +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +WHERE id = 1; +id a +1 1 +1 2 +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +WHERE id = 2; +ERROR HY000: Unexpected end of JSON text in argument 1 to function 'JSON_TABLE' +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +WHERE id = 3; +ERROR HY000: Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'JSON_TABLE' at position 32 +# one bad row aborts the whole statement, it is not skipped +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +ORDER BY id; +ERROR HY000: Unexpected end of JSON text in argument 1 to function 'JSON_TABLE' +DROP TABLE tp; +# +# 3. A DEFAULT clause supplying a value for a column declared JSON. +# The default is stored as given, without being checked. +# +SELECT jt.frag AS v, JSON_ARRAY(jt.frag) AS embedded, +JSON_VALID(JSON_ARRAY(jt.frag)) AS still_valid +FROM JSON_TABLE('[{"b":1}]', '$[*]' + COLUMNS (frag JSON PATH '$.a' DEFAULT '{"broken":' ON EMPTY)) jt; +v embedded still_valid +{"broken": ["{\"broken\":"] 1 +SELECT * FROM JSON_TABLE('[{"a":1},{"a":2}]', '$' + COLUMNS (frag JSON PATH '$[*].a' DEFAULT 'not json' ON ERROR)) jt; +frag +not json diff --git a/mysql-test/main/func_json_table.test b/mysql-test/main/func_json_table.test new file mode 100644 index 0000000000000..54ddf135ffb3f --- /dev/null +++ b/mysql-test/main/func_json_table.test @@ -0,0 +1,74 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: JSON_TABLE. +--echo # +--echo # JSON_TABLE parses its document itself and presents the pieces as +--echo # columns of a table that only exists for the duration of the query. +--echo # This test records what those columns are treated as when they are +--echo # fed back into JSON functions, what happens when the document comes +--echo # from a column whose stored bytes are not valid, and what a DEFAULT +--echo # clause is allowed to put into a column declared JSON. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A column declared JSON, consumed by JSON functions. Compare the +--echo # embedding decision here with the one a JSON table column gets in +--echo # func_json_columns. +--echo # + +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('[{"a": {"x": 1}}, {"a":{"y":2}}]'); +--echo # the fragment keeps the spacing it had in the input document +SELECT jt.frag AS v, HEX(jt.frag) AS h + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +--echo # fed to a constructor it is quoted, not embedded +SELECT JSON_ARRAY(jt.frag) AS embedded, JSON_VALID(JSON_ARRAY(jt.frag)) AS ok + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +SELECT JSON_OBJECT('k', jt.frag) AS v + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +--echo # but a mutator and JSON_TYPE both take it as a document +SELECT JSON_SET(jt.frag, '$.z', 3) AS v, JSON_TYPE(jt.frag) AS t + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +--echo # a column declared VARCHAR over the same path +SELECT JSON_ARRAY(jt.txt) AS quoted + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (txt VARCHAR(50) PATH '$.a' NULL ON ERROR)) jt; +DROP TABLE t1; + +--echo # +--echo # 2. The document coming from a column whose stored bytes no longer +--echo # satisfy its check constraint. +--echo # + +CREATE TABLE tp (id INT PRIMARY KEY, j JSON); +INSERT INTO tp VALUES (1, '[{"a":1},{"a":2}]'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES (2, '[{"a":1},'), (3, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))); +SET SESSION check_constraint_checks = ON; +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + WHERE id = 1; +--error ER_JSON_EOS +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + WHERE id = 2; +--error ER_JSON_DEPTH +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + WHERE id = 3; +--echo # one bad row aborts the whole statement, it is not skipped +--error ER_JSON_EOS +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + ORDER BY id; +DROP TABLE tp; + +--echo # +--echo # 3. A DEFAULT clause supplying a value for a column declared JSON. +--echo # The default is stored as given, without being checked. +--echo # + +SELECT jt.frag AS v, JSON_ARRAY(jt.frag) AS embedded, + JSON_VALID(JSON_ARRAY(jt.frag)) AS still_valid + FROM JSON_TABLE('[{"b":1}]', '$[*]' + COLUMNS (frag JSON PATH '$.a' DEFAULT '{"broken":' ON EMPTY)) jt; +SELECT * FROM JSON_TABLE('[{"a":1},{"a":2}]', '$' + COLUMNS (frag JSON PATH '$[*].a' DEFAULT 'not json' ON ERROR)) jt; From 3a720658871aea41631bd53e4e85e7380c90a33d Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 3 Aug 2026 00:54:26 -0400 Subject: [PATCH 02/30] Say what was written into a document, and never change the answer The JSON functions splice one document into another without reading the result back, so bytes that are not a document, or that mean something else in the character set they land in, pass through in silence. This makes each of those a note, without altering a single answer the server already gives. Every diagnostic added here is a **note**, not a warning. A warning is escalated to an error inside a statement running under strict mode, so a warning would stop statements that finish today. `json_valid()` already reports at this level for the same condition. **What is said now that was not said before** - `JSON_ARRAY`, `JSON_OBJECT`, the aggregates and the mutators scan what they splice, in the character set the result will be read in, and say which argument did not parse. The bytes are still spliced, exactly as they were. - Composed nesting is charged to the argument that caused it: a document that is inside the limit on its own but over it once wrapped is now named, rather than being noticed by whoever reads the result. - A value that cannot be written into a document at all - a character with no JSON formatting - is named where it was previously dropped without a word, and so is a key with the same character in it. The two go through different routines and only one of them used to say anything, so a document could be given up on over a key while the value beside it was named. - `JSON_QUOTE` says it too. It is one of five callers of `st_append_escaped()` and the only one folding that call into a `||` chain, which cannot tell a character it may not write from a buffer that would not grow: on a cp1250 row `JSON_ARRAY(c)` answers NULL naming argument 1 while `JSON_QUOTE(c)` answered NULL with nothing at all. The chain is split so that the two are told apart and the note reported. The answer does not move; only the silence does. **What was wrong, and is now right** - A path key was spliced in the character set of the *path*, into a document in a different one. An ASCII key was enough to make a ucs2 or utf16 document unreadable, so `JSON_SET` and `JSON_INSERT` could not add any key at all to one. The key is converted when the document has room for it, and goes in as it arrived when it has not. Whether the result reads back is then decided where it always was, on the way out. - The same mislabelling made `JSON_REMOVE` match nothing, and so remove nothing, in those documents. - A value spliced by a mutator kept its own character set, because the mutators take the character set of the document and aggregate nothing. A utf8mb4 value landing in a latin1 document arrived as twice as many characters as it was, and latin1 admits every byte, so nothing caught it. A conversion that loses nothing is now made; one that would lose something is not made at all, and the original bytes stay. - `JSON_UNQUOTE` declares `utf8mb4_bin` before it has read anything, then returned the argument unchanged in whatever character set it arrived in. Everything downstream read those bytes as utf8mb4. They are converted now, except where converting them would change them, and the conversion is made from the binary character set itself, whose `mb_wc` returns the byte. That is what makes the two ways out of the function name a byte the same character: `json_unescape()` on the string way out maps each byte to the character of that number, and latin1 - which in MariaDB is cp1252 - answers something else at 27 of the 32 positions in 0x80-0x9F. Read through latin1, `X'228022'` came back `C280` and `X'5B2280225D'` came back carrying the euro sign. - `json_nice()` ignored eight of its own writes, and `JSON_OBJECTAGG` ignored four, either of which lets a function report success over a half written document. **Nothing here returns NULL where it did not before, and no result changes bytes except where one of the conversions above is made.** That conversion is done with `copy_and_convert()` and not with `String::copy()`, which for a binary source does not convert at all: `needs_conversion()` answers false whenever the length divides the destination's `mbminlen`, and utf8mb4's is 1, so the bytes would go across untouched and be relabelled - 0x80 would leave as a lone continuation byte, which is not a character of the set the result says it is in. `st_append_escaped()` now returns which of the two ways it failed, since a buffer that will not grow has already stopped the statement while a character that cannot be written has not, and the two were previously indistinguishable to callers. It also reported success when it failed to reserve room for an empty value. It says so in its return type, as do the two other functions that write escaped text into a document, rather than in an `int` that happens to carry the same values; and the one place that answered with a bare success-or-failure of an append names which of the two it means, that append having failed for the one reason the type has a name for. The parameter block over `append_json_value()` says what the three parameters threaded through it do: the depth is consulted before the type is looked at rather than only for a value typed JSON, the function name and argument number reach both arms - the non-JSON one names the function in the bad-character note - and every function that edits a document passes the depth the path reached, taken off the scanner, never 0. `my_realloc()` gains a debug keyword of its own for failing a buffer that is being grown. The existing keyword fails every allocation including the first, which cannot reach the code that appends to a buffer already held - and that is where the unchecked writes are. Failing a buffer as it grows still cannot reach a write into a buffer that was sized before it was written, and every write that puts a value into a document is one of those - the room is asked for up front, at the widest the answer could come to. Each of those writes therefore names a keyword of its own, so that a test can have that one write report no room while everything else in the statement goes through. Where the debug support is off a keyword is a constant the compiler folds away, and the condition is the one it always was. Four new test files: `func_json_embed` for what is spliced and what is said about it, `func_json_pathkey` and `func_json_unquote` for the two character set faults, and `func_json_oom` for the invariant that a function either errors or returns something that parses, never a half written document. `func_json_embed` section 10 puts the cp1250 fixture through `JSON_QUOTE` beside the constructor that always named the argument, and `func_json_unquote` section 6 reads 0x80 and 0x9F both ways out, plus 0x81, one of the five positions where the two readings agree anyway. `func_json_oom` carries both instruments. Its first sections fail the buffer as it grows; the ones after them fail one write at a time - the quotes round a value, the room the escaping asks for, the quote that opens a key, the two quotes `JSON_QUOTE` is made of, both ways by which `JSON_UNQUOTE` returns an argument, a value converted into the character set of the document it is joining, and a path key converted the same way. The recorded baselines that move do so by gaining notes, except where one of the conversions above is made. `func_json_charset` gains values where a mutator could not add any key to a ucs2 document at all, and moves one where a constructor spliced a ucs2 fragment with its bytes relabelled rather than converted; `json_no_table` moves the one line where `JSON_UNQUOTE` is handed the bytes of a geometry value. No other recorded value moves. --- mysql-test/main/func_json.result | 2 + mysql-test/main/func_json_agg_limits.result | 1 + mysql-test/main/func_json_charset.result | 33 +- mysql-test/main/func_json_columns.result | 46 ++ mysql-test/main/func_json_depth.result | 5 + mysql-test/main/func_json_embed.result | 670 ++++++++++++++++++ mysql-test/main/func_json_embed.test | 336 +++++++++ mysql-test/main/func_json_oom.result | 309 ++++++++ mysql-test/main/func_json_oom.test | 275 ++++++++ mysql-test/main/func_json_pathkey.result | 202 ++++++ mysql-test/main/func_json_pathkey.test | 153 ++++ mysql-test/main/func_json_unquote.result | 159 +++++ mysql-test/main/func_json_unquote.test | 120 ++++ mysql-test/suite/json/r/json_no_table.result | 2 +- mysys/my_malloc.c | 21 +- sql/item_jsonfunc.cc | 698 ++++++++++++++++--- sql/item_jsonfunc.h | 52 +- 17 files changed, 2967 insertions(+), 117 deletions(-) create mode 100644 mysql-test/main/func_json_embed.result create mode 100644 mysql-test/main/func_json_embed.test create mode 100644 mysql-test/main/func_json_oom.result create mode 100644 mysql-test/main/func_json_oom.test create mode 100644 mysql-test/main/func_json_pathkey.result create mode 100644 mysql-test/main/func_json_pathkey.test create mode 100644 mysql-test/main/func_json_unquote.result create mode 100644 mysql-test/main/func_json_unquote.test diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index a5b21ad20294c..448e5c9043db8 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -780,6 +780,8 @@ DROP TABLE t1; select json_array(1,user(),compress(5.140264e+307)); json_array(1,user(),compress(5.140264e+307)) NULL +Warnings: +Note 4035 Broken JSON string in argument 3 to function 'json_array' at position 0 # # MDEV-16869 String functions don't respect character set of JSON_VALUE. # diff --git a/mysql-test/main/func_json_agg_limits.result b/mysql-test/main/func_json_agg_limits.result index 6197735f35fb5..196247ab7a4b8 100644 --- a/mysql-test/main/func_json_agg_limits.result +++ b/mysql-test/main/func_json_agg_limits.result @@ -295,6 +295,7 @@ v [["aaaaaaaaaa","]] Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_DEPTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; v NULL diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result index 43575c1c933a8..717ce0694f483 100644 --- a/mysql-test/main/func_json_charset.result +++ b/mysql-test/main/func_json_charset.result @@ -196,9 +196,7 @@ cs ucs2 SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS setv; setv -NULL -Warnings: -Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 26 +007B002200610022003A00200031002C0020002200620022003A00200032007D SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS cs; cs ucs2 @@ -246,9 +244,7 @@ ucs2 SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', CONVERT('x' USING utf8mb4))) AS v; v -NULL -Warnings: -Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 30 +007B002200610022003A00200031002C0020002200620022003A0020002200780022007D # a utf8mb4 document with a ucs2 value spliced into it SELECT CHARSET(JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2))) AS cs; cs @@ -276,27 +272,19 @@ NULL SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', JSON_COMPACT('{"x":2}'))) AS typed_into_ucs2; typed_into_ucs2 -NULL -Warnings: -Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 38 +007B002200610022003A00200031002C0020002200620022003A0020007B002200780022003A00200032007D007D SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', JSON_COMPACT('{"x":2}'))) AS ok; ok -NULL -Warnings: -Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 38 +1 SELECT HEX(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS latin1_typed; latin1_typed -NULL -Warnings: -Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 18 +7B2261223A20312C202262223A207B2278223A2022C3A9227D7D SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS ok; ok -NULL -Warnings: -Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 18 +1 # the constructors take the same fragment without complaint SELECT HEX(JSON_ARRAY( JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS embedded; @@ -311,7 +299,7 @@ cs latin1 SELECT HEX(JSON_OBJECT('k', JSON_COMPACT(CONVERT('{"x":2}' USING ucs2)))) AS obj_embedded; obj_embedded -007B0022006B0022003A00200000007B0000002200000078000000220000003A000000320000007D007D +007B0022006B0022003A0020007B002200780022003A0032007D007D # # 6. Binary arguments, which are not text and are copied through. # @@ -435,9 +423,7 @@ ok # character set comes from SELECT HEX(JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1))) AS h FROM tc; h -NULL -Warnings: -Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 27 +7B2278223A20312C202279223A207B2261223A20312C202262223A2022C3A9227D7D SELECT HEX(JSON_ARRAY(CONVERT(j USING binary))) AS h FROM tc; h 5B7B2261223A20312C202262223A2022C3A9227D5D @@ -465,11 +451,14 @@ JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tp; v ok [{"a":1,] 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 SELECT JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1)) AS v FROM tp; v NULL Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 20 SELECT JSON_ARRAY(CAST(j AS CHAR)) AS v FROM tp; v diff --git a/mysql-test/main/func_json_columns.result b/mysql-test/main/func_json_columns.result index 25a1aec253f8d..d114ff93f00fe 100644 --- a/mysql-test/main/func_json_columns.result +++ b/mysql-test/main/func_json_columns.result @@ -451,6 +451,11 @@ id v 11 [not json at all] 12 [{"a":1} trailing] 13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; id still_valid 10 0 @@ -458,9 +463,13 @@ id still_valid 12 0 13 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v @@ -468,6 +477,11 @@ id v 11 {"k": not json at all} 12 {"k": {"a":1} trailing} 13 {"k": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; id still_valid @@ -476,9 +490,13 @@ id still_valid 12 0 13 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 15 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 37 SELECT id, JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; id v @@ -486,12 +504,22 @@ id v 11 [not json at all] 12 [{"a":1} trailing] 13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 32 SELECT id, JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; id v 10 {"10":{"a":1,} 11 {"11":not json at all} 12 {"12":{"a":1} trailing} 13 {"13":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 32 # the same column read as plain text is quoted instead of embedded SELECT id, JSON_ARRAY(CONCAT(j, '')) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v @@ -511,11 +539,14 @@ SELECT id, JSON_ARRAY(j) AS v FROM t3 ORDER BY id; id v 1 [{"a":1}] 10 [{"a":1,] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t3 ORDER BY id; id still_valid 1 1 10 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 # 5a. The same column reached through a scalar subquery, where the # value is fetched once and held rather than read per row. @@ -533,10 +564,13 @@ ok SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 2)) AS v_poisoned; v_poisoned [{"a":1,] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 2))) AS ok_poisoned; ok_poisoned 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 1)) AS v; v @@ -544,6 +578,8 @@ v SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; v_poisoned {"k": {"a":1,} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 1)) AS v; v {"d": {"a": 1}} @@ -551,6 +587,7 @@ SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; v_poisoned NULL Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 13 DROP TABLE ts; # 5b. The same column reached through a view. A merged view reads the @@ -583,6 +620,8 @@ id v ok 1 [{"a": 1, "b": [1, 2]}] 1 2 [{"a":1,] 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok FROM v_tmp ORDER BY id; @@ -590,6 +629,8 @@ id v ok 1 [{"a": 1, "b": [1, 2]}] 1 2 [{"a":1,] 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_merge ORDER BY id; id v @@ -671,6 +712,8 @@ stored # the same value stored into a plain LONGTEXT column CREATE TABLE t8 (j LONGTEXT); INSERT INTO t8 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 SELECT JSON_VALID(j) AS v, LENGTH(j) AS len FROM t8; v len 0 65 @@ -836,11 +879,14 @@ SELECT id, JSON_ARRAY(j) AS v FROM tmp1 ORDER BY id; id v 1 [{"a":1}] 2 [{"a":1,] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM tmp1 ORDER BY id; id still_valid 1 1 2 0 Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 SELECT id, JSON_SET(j, '$.z', 1) AS v FROM tmp1 ORDER BY id; id v diff --git a/mysql-test/main/func_json_depth.result b/mysql-test/main/func_json_depth.result index 0b1124ca1227b..82a90ead1280b 100644 --- a/mysql-test/main/func_json_depth.result +++ b/mysql-test/main/func_json_depth.result @@ -254,15 +254,19 @@ SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS d32; d32 NULL Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT(@d31))) AS still_valid; still_valid 0 Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 SELECT LENGTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS produced_bytes; produced_bytes 65 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 # the same through an object constructor SELECT JSON_DEPTH(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS d32; d32 @@ -275,6 +279,7 @@ SELECT JSON_VALID(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d30)))) AS still_valid; still_valid 0 Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d29)))) AS d31; d31 diff --git a/mysql-test/main/func_json_embed.result b/mysql-test/main/func_json_embed.result new file mode 100644 index 0000000000000..f9d3d88931428 --- /dev/null +++ b/mysql-test/main/func_json_embed.result @@ -0,0 +1,670 @@ +# +# Splicing a value that is typed as JSON into a document. +# +# A value whose type is JSON is spliced into the result as it +# stands rather than quoted as a string. This is not only about +# objects and arrays: a JSON number, string or boolean is spliced +# the same way, which is what keeps a stored "a string" from coming +# back out with its quotes escaped. +# +# Being typed as JSON and being JSON are different things: the type +# comes from a check constraint, and a constraint can be switched +# off for a statement, leaving bytes behind that it would have +# rejected. Nothing between the column and the result reads those +# bytes. +# +# The constructors are the interesting half, because they have no +# closing pass over what they built. Whatever they splice is what +# the caller gets. +# +# A value that does not parse is spliced all the same, byte for +# byte, and a note is raised saying which argument it was. Reading +# the result back is still where a caller finds out that what came +# out is not a document; the note is what says so at the time. +# +# Non ASCII characters below are written as the bytes that encode +# them, so this file needs no character set of its own: +# C3 A9 is U+00E9, one byte in latin1 (E9) +# E6 BC A2 is U+6F22, which latin1 has no room for +# +SET NAMES utf8mb4; +# +# 1. Which values are spliced as documents and which are quoted. +# +CREATE TABLE t1 (id INT, j JSON, txt LONGTEXT); +INSERT INTO t1 VALUES (1, '{"a": 1}', '{"a": 1}'); +SELECT JSON_ARRAY(j) AS typed, JSON_ARRAY(txt) AS untyped FROM t1; +typed untyped +[{"a": 1}] ["{\"a\": 1}"] +# a function that returns JSON is typed the same way +SELECT JSON_ARRAY(JSON_COMPACT('[1, 2]')) AS from_function; +from_function +[[1, 2]] +SELECT JSON_ARRAY(CONCAT(j, '')) AS through_concat FROM t1; +through_concat +["{\"a\": 1}"] +# +# 2. A column holding bytes its constraint would have rejected. +# +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,', NULL); +INSERT INTO t1 VALUES (11, 'not json at all', NULL); +INSERT INTO t1 VALUES (12, '{"a":1} and then some', NULL); +SET SESSION check_constraint_checks = ON; +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 [{"a":1,] +11 [not json at all] +12 [{"a":1} and then some] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document +FROM t1 WHERE id >= 10 ORDER BY id; +id built_a_document +10 0 +11 0 +12 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 {"k": {"a":1,} +11 {"k": not json at all} +12 {"k": {"a":1} and then some} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS built_a_document +FROM t1 WHERE id >= 10 ORDER BY id; +id built_a_document +10 0 +11 0 +12 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 15 +# more than one element, so the value is spliced in the middle of a +# document rather than as the whole of it +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 [1, {"a":1,, "z"] +11 [1, not json at all, "z"] +12 [1, {"a":1} and then some, "z"] +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 9 +SELECT id, JSON_OBJECT('before', 1, 'k', j, 'after', 2) AS v +FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 {"before": 1, "k": {"a":1,, "after": 2} +11 {"before": 1, "k": not json at all, "after": 2} +12 {"before": 1, "k": {"a":1} and then some, "after": 2} +Warnings: +Note 4037 Unexpected end of JSON text in argument 4 to function 'json_object' +Note 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 1 +Note 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 9 +# the aggregates, which splice a value per row +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10; +v +[{"a":1,,not json at all,{"a":1} and then some] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS built_a_document +FROM t1 WHERE id >= 10; +built_a_document +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10; +v +{"10":{"a":1,, "11":not json at all, "12":{"a":1} and then some} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +SELECT JSON_VALID(JSON_OBJECTAGG(id, j)) AS built_a_document +FROM t1 WHERE id >= 10; +built_a_document +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 +# a group mixing a good row with a bad one +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id = 1 OR id = 10; +v +[{"a": 1},{"a":1,] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +# the mutators splice the same way, but read back what they built +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v +FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 21 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_set' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 14 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_set' at position 9 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 22 +SELECT id, JSON_ARRAY_APPEND('[0]', '$', j) AS v +FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_array_append' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 12 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_array_append' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 5 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_array_append' at position 9 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 13 +# +# 3. Bytes that are not a document but do not look wrong afterwards. +# +# An empty value leaves nothing behind at all, so the result parses +# and reports one element fewer than it was given. The document +# itself gives no sign of this, which is what makes the note the +# only way to tell it apart from an array that really was short. +# +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (20, '', NULL); +INSERT INTO t1 VALUES (21, ' ', NULL); +INSERT INTO t1 VALUES (22, '// nothing here', NULL); +SET SESSION check_constraint_checks = ON; +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS looks_fine, +JSON_LENGTH(JSON_ARRAY(j)) AS elements +FROM t1 WHERE id >= 20 ORDER BY id; +id v looks_fine elements +20 [] 1 0 +21 [ ] 1 0 +22 [// nothing here] 0 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 2 +SELECT id, JSON_ARRAY(1, j, 2) AS v, JSON_VALID(JSON_ARRAY(1, j, 2)) AS looks_fine, +JSON_LENGTH(JSON_ARRAY(1, j, 2)) AS elements +FROM t1 WHERE id >= 20 ORDER BY id; +id v looks_fine elements +20 [1, , 2] 0 NULL +21 [1, , 2] 0 NULL +22 [1, // nothing here, 2] 0 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 5 +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 8 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 5 +SELECT id, JSON_OBJECT('k', j) AS v, JSON_VALID(JSON_OBJECT('k', j)) AS looks_fine +FROM t1 WHERE id >= 20 ORDER BY id; +id v looks_fine +20 {"k": } 0 +21 {"k": } 0 +22 {"k": // nothing here} 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +SELECT JSON_ARRAYAGG(j) AS v, JSON_VALID(JSON_ARRAYAGG(j)) AS looks_fine +FROM t1 WHERE id >= 20; +v looks_fine +[, ,// nothing here] 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +# +# 4. Nesting that only goes over the limit once the value is wrapped. +# +# The scanner takes 31 nested structures. A document of exactly 31 +# is a document; put it inside one more and it is not. +# +CREATE TABLE t2 (id INT, j JSON); +INSERT INTO t2 VALUES (31, CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))); +INSERT INTO t2 VALUES (30, CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30))); +SELECT id, JSON_VALID(j) AS on_its_own, JSON_DEPTH(j) AS deep FROM t2 ORDER BY id; +id on_its_own deep +30 1 31 +31 1 32 +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS wrapped_once, +LENGTH(JSON_ARRAY(j)) AS bytes FROM t2 ORDER BY id; +id wrapped_once bytes +30 1 63 +31 0 65 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS wrapped_once FROM t2 ORDER BY id; +id wrapped_once +30 1 +31 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 63 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 37 +# without a table in the way +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT( +CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))))) AS wrapped_once; +wrapped_once +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +# a mutator splices at the depth the path reaches, and reads back +SELECT id, JSON_VALID(JSON_SET('{}', '$.k', j)) AS spliced FROM t2 ORDER BY id; +id spliced +30 1 +31 NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 36 +# +# 5. A document written in a different character set from the one it +# is being spliced into. +# +# The constructors agree a character set across their arguments +# first, so the value arrives already converted. The mutators take +# the character set of the document they are editing and convert +# nothing, so the value arrives as it was written. +# +CREATE TABLE t3 (id INT, j JSON); +INSERT INTO t3 VALUES (1, _utf8mb4 0x5B22C3A9225D); +INSERT INTO t3 VALUES (2, _utf8mb4 0x5B22E6BCA2225D); +SELECT id, CHARSET(j) AS column_cs, HEX(j) AS bytes FROM t3 ORDER BY id; +id column_cs bytes +1 utf8mb4 5B22C3A9225D +2 utf8mb4 5B22E6BCA2225D +# spliced by a constructor whose other argument is latin1 +SELECT id, CHARSET(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS result_cs, +HEX(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS bytes +FROM t3 ORDER BY id; +id result_cs bytes +1 utf8mb4 5B2278222C205B22C3A9225D5D +2 utf8mb4 5B2278222C205B22E6BCA2225D5D +# spliced by a mutator into a latin1 document +SELECT id, CHARSET(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) +AS result_cs, +HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) AS bytes +FROM t3 ORDER BY id; +id result_cs bytes +1 latin1 7B2261223A20312C202262223A205B22E9225D7D +2 latin1 7B2261223A20312C202262223A205B22E6BCA2225D7D +SELECT id, HEX(JSON_ARRAY_APPEND(CONVERT(_utf8mb4'[1]' USING latin1), '$', j)) +AS bytes +FROM t3 ORDER BY id; +id bytes +1 5B312C205B22E9225D5D +2 5B312C205B22E6BCA2225D5D +# read the spliced value back out of the document it landed in +SELECT id, HEX(JSON_EXTRACT( +JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j), +'$.b')) AS read_back +FROM t3 ORDER BY id; +id read_back +1 5B22E9225D +2 5B22E6BCA2225D +# +# 6. Values that are what they say they are, which must not move. +# +# Note the scalars among them. A column holding a JSON string is +# spliced with one set of quotes, not two, which is the difference +# between being spliced as JSON and being quoted as a string. +# +CREATE TABLE t4 (id INT, j JSON); +INSERT INTO t4 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t4 VALUES (2, '[1, 2, 3]'); +INSERT INTO t4 VALUES (3, '"a string"'); +INSERT INTO t4 VALUES (4, '42'); +INSERT INTO t4 VALUES (5, 'null'); +INSERT INTO t4 VALUES (6, 'true'); +INSERT INTO t4 VALUES (7, '{"compact":[1,2],"no":"spaces"}'); +SELECT id, JSON_ARRAY(j) AS v FROM t4 ORDER BY id; +id v +1 [{"a": 1, "b": [1, 2]}] +2 [[1, 2, 3]] +3 ["a string"] +4 [42] +5 [null] +6 [true] +7 [{"compact":[1,2],"no":"spaces"}] +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 ORDER BY id; +id v +1 {"k": {"a": 1, "b": [1, 2]}} +2 {"k": [1, 2, 3]} +3 {"k": "a string"} +4 {"k": 42} +5 {"k": null} +6 {"k": true} +7 {"k": {"compact":[1,2],"no":"spaces"}} +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t4 ORDER BY id; +id v +1 [1, {"a": 1, "b": [1, 2]}, "z"] +2 [1, [1, 2, 3], "z"] +3 [1, "a string", "z"] +4 [1, 42, "z"] +5 [1, null, "z"] +6 [1, true, "z"] +7 [1, {"compact":[1,2],"no":"spaces"}, "z"] +SELECT JSON_ARRAYAGG(j) AS v FROM t4; +v +[{"a": 1, "b": [1, 2]},[1, 2, 3],"a string",42,null,true,{"compact":[1,2],"no":"spaces"}] +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4; +v +{"1":{"a": 1, "b": [1, 2]}, "2":[1, 2, 3], "3":"a string", "4":42, "5":null, "6":true, "7":{"compact":[1,2],"no":"spaces"}} +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v FROM t4 ORDER BY id; +id v +1 {"x": 0, "k": {"a": 1, "b": [1, 2]}} +2 {"x": 0, "k": [1, 2, 3]} +3 {"x": 0, "k": "a string"} +4 {"x": 0, "k": 42} +5 {"x": 0, "k": null} +6 {"x": 0, "k": true} +7 {"x": 0, "k": {"compact": [1, 2], "no": "spaces"}} +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document FROM t4 ORDER BY id; +id built_a_document +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +# a NULL value is spliced as the JSON null, not as the document +INSERT INTO t4 VALUES (8, NULL); +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id = 8; +id v +8 [null] +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 WHERE id = 8; +id v +8 {"k": null} +# +# 7. Whether what the constructors build is already in the loose +# form, or would still be changed by reformatting it. +# +# A value that is spliced is copied exactly as it was written, so +# the spacing of the result is the spacing the value arrived with. +# A constructor is therefore only in the loose form when the +# documents given to it are. +# +CREATE TABLE t5 (id INT, j JSON); +INSERT INTO t5 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t5 VALUES (2, '{"a":1,"b":[1,2]}'); +INSERT INTO t5 VALUES (3, '[ 1 ,2, 3 ]'); +# with nothing spliced, and with a document spliced in +SELECT JSON_ARRAY(1, 'a', TRUE, NULL) AS built, +JSON_ARRAY(1, 'a', TRUE, NULL) = JSON_LOOSE(JSON_ARRAY(1, 'a', TRUE, NULL)) +AS already_loose; +built already_loose +[1, "a", true, null] 1 +SELECT JSON_OBJECT('a', 1, 'b', 'x') AS built, +JSON_OBJECT('a', 1, 'b', 'x') = JSON_LOOSE(JSON_OBJECT('a', 1, 'b', 'x')) +AS already_loose; +built already_loose +{"a": 1, "b": "x"} 1 +SELECT id, JSON_ARRAY(j) AS built, +JSON_ARRAY(j) = JSON_LOOSE(JSON_ARRAY(j)) AS already_loose +FROM t5 ORDER BY id; +id built already_loose +1 [{"a": 1, "b": [1, 2]}] 1 +2 [{"a":1,"b":[1,2]}] 0 +3 [[ 1 ,2, 3 ]] 0 +SELECT id, JSON_OBJECT('k', j) AS built, +JSON_OBJECT('k', j) = JSON_LOOSE(JSON_OBJECT('k', j)) AS already_loose +FROM t5 ORDER BY id; +id built already_loose +1 {"k": {"a": 1, "b": [1, 2]}} 1 +2 {"k": {"a":1,"b":[1,2]}} 0 +3 {"k": [ 1 ,2, 3 ]} 0 +# the aggregates write their own separators, which are not the +# loose ones. A group of one never writes a separator at all, so +# both a single row and several are recorded here. +INSERT INTO t5 VALUES (4, '{"c": 3}'); +SELECT JSON_ARRAYAGG(j) AS built, +JSON_ARRAYAGG(j) = JSON_LOOSE(JSON_ARRAYAGG(j)) AS already_loose +FROM t5 WHERE id = 1; +built already_loose +[{"a": 1, "b": [1, 2]}] 1 +SELECT JSON_ARRAYAGG(j ORDER BY id) AS built, +JSON_ARRAYAGG(j ORDER BY id) = JSON_LOOSE(JSON_ARRAYAGG(j ORDER BY id)) +AS already_loose +FROM t5 WHERE id IN (1, 4); +built already_loose +[{"a": 1, "b": [1, 2]},{"c": 3}] 0 +SELECT JSON_OBJECTAGG(id, j) AS built, +JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose +FROM t5 WHERE id = 1; +built already_loose +{"1":{"a": 1, "b": [1, 2]}} 0 +SELECT JSON_OBJECTAGG(id, j) AS built, +JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose +FROM t5 WHERE id IN (1, 4); +built already_loose +{"1":{"a": 1, "b": [1, 2]}, "4":{"c": 3}} 0 +# a mutator reads back everything it built, so its spacing is its +# own regardless of what was spliced +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS built, +JSON_SET('{"x": 0}', '$.k', j) = JSON_LOOSE(JSON_SET('{"x": 0}', '$.k', j)) +AS already_loose +FROM t5 ORDER BY id; +id built already_loose +1 {"x": 0, "k": {"a": 1, "b": [1, 2]}} 1 +2 {"x": 0, "k": {"a": 1, "b": [1, 2]}} 1 +3 {"x": 0, "k": [1, 2, 3]} 1 +4 {"x": 0, "k": {"c": 3}} 1 +# +# 8. What the aggregates do with a row they cannot write out. +# +# An aggregate writes the separator before it asks for the element, +# so a row that yields nothing leaves a separator with nothing on +# one side of it. There are two reasons a row can yield nothing +# and they are not answered alike. A character that cannot be +# written into a document at all is a property of the data: the +# element was dropped and the separator left standing before, and +# it still is, now with a note saying so. A buffer that would not +# grow is not a property of the data, and has already stopped the +# statement by the time it is noticed, so the group is refused +# whole rather than returned with a gap in it. +# +# cp1250 is used here because it admits every byte, so a column can +# hold one that stands for no character at all. +# +CREATE TABLE t6 (id INT, c VARCHAR(16) CHARACTER SET cp1250); +INSERT INTO t6 VALUES (1, X'81'), (2, 'ok'), (3, X'8881'); +SELECT id, HEX(c) AS stored FROM t6 ORDER BY id; +id stored +1 81 +2 6F6B +3 8881 +SELECT JSON_ARRAYAGG(c) AS v FROM t6; +v +[,"ok",] +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +SELECT JSON_OBJECTAGG(id, c) AS v FROM t6; +v +{"1":", "2":"ok", "3":"} +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_objectagg' at position 0 +Note 4035 Broken JSON string in argument 2 to function 'json_objectagg' at position 0 +# and where it is the key that cannot be written +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +v +{"":1, "ok":2, "":3} +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_objectagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_objectagg' at position 0 +# a group with no such row is untouched +SELECT JSON_ARRAYAGG(c) AS v FROM t6 WHERE id = 2; +v +["ok"] +# the same rows through the tree path, which writes the elements out +# when the group is asked for rather than as they arrive +SELECT JSON_ARRAYAGG(c ORDER BY id) AS v FROM t6; +v +[,"ok",] +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +SELECT JSON_ARRAYAGG(DISTINCT c) AS v FROM t6; +v +["ok",,] +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +# a copy of the aggregate, which is made without fixing it again. +# The values here are valid, so nothing should be said about them. +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v +FROM t6 GROUP BY id WITH ROLLUP; +id v +1 5B005B002200780022005D5D +2 5B005B002200780022005D5D +3 5B005B002200780022005D5D +NULL 5B005B002200780022005D002C005B002200780022005D002C005B002200780022005D5D +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v +FROM t6 GROUP BY id; +id v +1 5B005B002200780022005D5D +2 5B005B002200780022005D5D +3 5B005B002200780022005D5D +# nesting composed by the aggregate itself, which puts each value +# inside one array or object of its own +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, +JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj +FROM t2 WHERE id = 31; +wrapped_once wrapped_once_obj +0 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 63 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 63 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 37 +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, +JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj +FROM t2 WHERE id = 30; +wrapped_once wrapped_once_obj +1 1 +# +# 9. The same character in a key of a constructor. +# +# A key is written by the same routine that writes one for the +# aggregate, and a character no document can carry stops it the +# same way. The constructors read nothing back, so a key and a +# value alike throw the whole result away here; all that is left +# of either is what is said about it, and both sides say which +# argument it was. +# +# the value side, which has always been named +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 0 +SELECT JSON_OBJECT('k', c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_object' at position 0 +# the key side, in the first pair and in a later one +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_object' at position 0 +SELECT JSON_OBJECT('a', 1, c, 2) AS v FROM t6 WHERE id = 3; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 3 to function 'json_object' at position 0 +# a key and a value of the same row, which stops at the key +SELECT JSON_OBJECT(c, c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_object' at position 0 +# a row whose key can be written says nothing +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 2; +v +{"ok": 1} +# +# 10. The same character through JSON_QUOTE, which writes a value out +# with the routine the constructors use but is not building a +# document of its own. +# +# It has always answered NULL for a character it cannot write and +# still does. What it never did was say so, and it was the only +# caller of that routine which did not. +# +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_quote' at position 0 +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 3; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_quote' at position 0 +# a row it can write says nothing +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 2; +v +"ok" +# the same value reaching a constructor instead, which has always +# named the argument +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 0 +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/main/func_json_embed.test b/mysql-test/main/func_json_embed.test new file mode 100644 index 0000000000000..36a3d9728302f --- /dev/null +++ b/mysql-test/main/func_json_embed.test @@ -0,0 +1,336 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Splicing a value that is typed as JSON into a document. +--echo # +--echo # A value whose type is JSON is spliced into the result as it +--echo # stands rather than quoted as a string. This is not only about +--echo # objects and arrays: a JSON number, string or boolean is spliced +--echo # the same way, which is what keeps a stored "a string" from coming +--echo # back out with its quotes escaped. +--echo # +--echo # Being typed as JSON and being JSON are different things: the type +--echo # comes from a check constraint, and a constraint can be switched +--echo # off for a statement, leaving bytes behind that it would have +--echo # rejected. Nothing between the column and the result reads those +--echo # bytes. +--echo # +--echo # The constructors are the interesting half, because they have no +--echo # closing pass over what they built. Whatever they splice is what +--echo # the caller gets. +--echo # +--echo # A value that does not parse is spliced all the same, byte for +--echo # byte, and a note is raised saying which argument it was. Reading +--echo # the result back is still where a caller finds out that what came +--echo # out is not a document; the note is what says so at the time. +--echo # +--echo # Non ASCII characters below are written as the bytes that encode +--echo # them, so this file needs no character set of its own: +--echo # C3 A9 is U+00E9, one byte in latin1 (E9) +--echo # E6 BC A2 is U+6F22, which latin1 has no room for +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Which values are spliced as documents and which are quoted. +--echo # + +CREATE TABLE t1 (id INT, j JSON, txt LONGTEXT); +INSERT INTO t1 VALUES (1, '{"a": 1}', '{"a": 1}'); +SELECT JSON_ARRAY(j) AS typed, JSON_ARRAY(txt) AS untyped FROM t1; +--echo # a function that returns JSON is typed the same way +SELECT JSON_ARRAY(JSON_COMPACT('[1, 2]')) AS from_function; +SELECT JSON_ARRAY(CONCAT(j, '')) AS through_concat FROM t1; + +--echo # +--echo # 2. A column holding bytes its constraint would have rejected. +--echo # + +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,', NULL); +INSERT INTO t1 VALUES (11, 'not json at all', NULL); +INSERT INTO t1 VALUES (12, '{"a":1} and then some', NULL); +SET SESSION check_constraint_checks = ON; + +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS built_a_document + FROM t1 WHERE id >= 10 ORDER BY id; + +--echo # more than one element, so the value is spliced in the middle of a +--echo # document rather than as the whole of it +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_OBJECT('before', 1, 'k', j, 'after', 2) AS v + FROM t1 WHERE id >= 10 ORDER BY id; + +--echo # the aggregates, which splice a value per row +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10; +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS built_a_document + FROM t1 WHERE id >= 10; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10; +SELECT JSON_VALID(JSON_OBJECTAGG(id, j)) AS built_a_document + FROM t1 WHERE id >= 10; +--echo # a group mixing a good row with a bad one +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id = 1 OR id = 10; + +--echo # the mutators splice the same way, but read back what they built +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_ARRAY_APPEND('[0]', '$', j) AS v + FROM t1 WHERE id >= 10 ORDER BY id; + +--echo # +--echo # 3. Bytes that are not a document but do not look wrong afterwards. +--echo # +--echo # An empty value leaves nothing behind at all, so the result parses +--echo # and reports one element fewer than it was given. The document +--echo # itself gives no sign of this, which is what makes the note the +--echo # only way to tell it apart from an array that really was short. +--echo # + +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (20, '', NULL); +INSERT INTO t1 VALUES (21, ' ', NULL); +INSERT INTO t1 VALUES (22, '// nothing here', NULL); +SET SESSION check_constraint_checks = ON; + +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS looks_fine, + JSON_LENGTH(JSON_ARRAY(j)) AS elements + FROM t1 WHERE id >= 20 ORDER BY id; +SELECT id, JSON_ARRAY(1, j, 2) AS v, JSON_VALID(JSON_ARRAY(1, j, 2)) AS looks_fine, + JSON_LENGTH(JSON_ARRAY(1, j, 2)) AS elements + FROM t1 WHERE id >= 20 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v, JSON_VALID(JSON_OBJECT('k', j)) AS looks_fine + FROM t1 WHERE id >= 20 ORDER BY id; +SELECT JSON_ARRAYAGG(j) AS v, JSON_VALID(JSON_ARRAYAGG(j)) AS looks_fine + FROM t1 WHERE id >= 20; + +--echo # +--echo # 4. Nesting that only goes over the limit once the value is wrapped. +--echo # +--echo # The scanner takes 31 nested structures. A document of exactly 31 +--echo # is a document; put it inside one more and it is not. +--echo # + +CREATE TABLE t2 (id INT, j JSON); +INSERT INTO t2 VALUES (31, CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))); +INSERT INTO t2 VALUES (30, CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30))); +SELECT id, JSON_VALID(j) AS on_its_own, JSON_DEPTH(j) AS deep FROM t2 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS wrapped_once, + LENGTH(JSON_ARRAY(j)) AS bytes FROM t2 ORDER BY id; +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS wrapped_once FROM t2 ORDER BY id; +--echo # without a table in the way +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT( + CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))))) AS wrapped_once; +--echo # a mutator splices at the depth the path reaches, and reads back +SELECT id, JSON_VALID(JSON_SET('{}', '$.k', j)) AS spliced FROM t2 ORDER BY id; + +--echo # +--echo # 5. A document written in a different character set from the one it +--echo # is being spliced into. +--echo # +--echo # The constructors agree a character set across their arguments +--echo # first, so the value arrives already converted. The mutators take +--echo # the character set of the document they are editing and convert +--echo # nothing, so the value arrives as it was written. +--echo # + +CREATE TABLE t3 (id INT, j JSON); +INSERT INTO t3 VALUES (1, _utf8mb4 0x5B22C3A9225D); +INSERT INTO t3 VALUES (2, _utf8mb4 0x5B22E6BCA2225D); +SELECT id, CHARSET(j) AS column_cs, HEX(j) AS bytes FROM t3 ORDER BY id; + +--echo # spliced by a constructor whose other argument is latin1 +SELECT id, CHARSET(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS result_cs, + HEX(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS bytes + FROM t3 ORDER BY id; + +--echo # spliced by a mutator into a latin1 document +SELECT id, CHARSET(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) + AS result_cs, + HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) AS bytes + FROM t3 ORDER BY id; +SELECT id, HEX(JSON_ARRAY_APPEND(CONVERT(_utf8mb4'[1]' USING latin1), '$', j)) + AS bytes + FROM t3 ORDER BY id; +--echo # read the spliced value back out of the document it landed in +SELECT id, HEX(JSON_EXTRACT( + JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j), + '$.b')) AS read_back + FROM t3 ORDER BY id; + +--echo # +--echo # 6. Values that are what they say they are, which must not move. +--echo # +--echo # Note the scalars among them. A column holding a JSON string is +--echo # spliced with one set of quotes, not two, which is the difference +--echo # between being spliced as JSON and being quoted as a string. +--echo # + +CREATE TABLE t4 (id INT, j JSON); +INSERT INTO t4 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t4 VALUES (2, '[1, 2, 3]'); +INSERT INTO t4 VALUES (3, '"a string"'); +INSERT INTO t4 VALUES (4, '42'); +INSERT INTO t4 VALUES (5, 'null'); +INSERT INTO t4 VALUES (6, 'true'); +INSERT INTO t4 VALUES (7, '{"compact":[1,2],"no":"spaces"}'); +SELECT id, JSON_ARRAY(j) AS v FROM t4 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 ORDER BY id; +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t4 ORDER BY id; +SELECT JSON_ARRAYAGG(j) AS v FROM t4; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4; +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v FROM t4 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document FROM t4 ORDER BY id; +--echo # a NULL value is spliced as the JSON null, not as the document +INSERT INTO t4 VALUES (8, NULL); +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id = 8; +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 WHERE id = 8; + +--echo # +--echo # 7. Whether what the constructors build is already in the loose +--echo # form, or would still be changed by reformatting it. +--echo # +--echo # A value that is spliced is copied exactly as it was written, so +--echo # the spacing of the result is the spacing the value arrived with. +--echo # A constructor is therefore only in the loose form when the +--echo # documents given to it are. +--echo # + +CREATE TABLE t5 (id INT, j JSON); +INSERT INTO t5 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t5 VALUES (2, '{"a":1,"b":[1,2]}'); +INSERT INTO t5 VALUES (3, '[ 1 ,2, 3 ]'); + +--echo # with nothing spliced, and with a document spliced in +SELECT JSON_ARRAY(1, 'a', TRUE, NULL) AS built, + JSON_ARRAY(1, 'a', TRUE, NULL) = JSON_LOOSE(JSON_ARRAY(1, 'a', TRUE, NULL)) + AS already_loose; +SELECT JSON_OBJECT('a', 1, 'b', 'x') AS built, + JSON_OBJECT('a', 1, 'b', 'x') = JSON_LOOSE(JSON_OBJECT('a', 1, 'b', 'x')) + AS already_loose; +SELECT id, JSON_ARRAY(j) AS built, + JSON_ARRAY(j) = JSON_LOOSE(JSON_ARRAY(j)) AS already_loose + FROM t5 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS built, + JSON_OBJECT('k', j) = JSON_LOOSE(JSON_OBJECT('k', j)) AS already_loose + FROM t5 ORDER BY id; + +--echo # the aggregates write their own separators, which are not the +--echo # loose ones. A group of one never writes a separator at all, so +--echo # both a single row and several are recorded here. +INSERT INTO t5 VALUES (4, '{"c": 3}'); +SELECT JSON_ARRAYAGG(j) AS built, + JSON_ARRAYAGG(j) = JSON_LOOSE(JSON_ARRAYAGG(j)) AS already_loose + FROM t5 WHERE id = 1; +SELECT JSON_ARRAYAGG(j ORDER BY id) AS built, + JSON_ARRAYAGG(j ORDER BY id) = JSON_LOOSE(JSON_ARRAYAGG(j ORDER BY id)) + AS already_loose + FROM t5 WHERE id IN (1, 4); +SELECT JSON_OBJECTAGG(id, j) AS built, + JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose + FROM t5 WHERE id = 1; +SELECT JSON_OBJECTAGG(id, j) AS built, + JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose + FROM t5 WHERE id IN (1, 4); + +--echo # a mutator reads back everything it built, so its spacing is its +--echo # own regardless of what was spliced +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS built, + JSON_SET('{"x": 0}', '$.k', j) = JSON_LOOSE(JSON_SET('{"x": 0}', '$.k', j)) + AS already_loose + FROM t5 ORDER BY id; + +--echo # +--echo # 8. What the aggregates do with a row they cannot write out. +--echo # +--echo # An aggregate writes the separator before it asks for the element, +--echo # so a row that yields nothing leaves a separator with nothing on +--echo # one side of it. There are two reasons a row can yield nothing +--echo # and they are not answered alike. A character that cannot be +--echo # written into a document at all is a property of the data: the +--echo # element was dropped and the separator left standing before, and +--echo # it still is, now with a note saying so. A buffer that would not +--echo # grow is not a property of the data, and has already stopped the +--echo # statement by the time it is noticed, so the group is refused +--echo # whole rather than returned with a gap in it. +--echo # +--echo # cp1250 is used here because it admits every byte, so a column can +--echo # hold one that stands for no character at all. +--echo # + +CREATE TABLE t6 (id INT, c VARCHAR(16) CHARACTER SET cp1250); +INSERT INTO t6 VALUES (1, X'81'), (2, 'ok'), (3, X'8881'); +SELECT id, HEX(c) AS stored FROM t6 ORDER BY id; +SELECT JSON_ARRAYAGG(c) AS v FROM t6; +SELECT JSON_OBJECTAGG(id, c) AS v FROM t6; +--echo # and where it is the key that cannot be written +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +--echo # a group with no such row is untouched +SELECT JSON_ARRAYAGG(c) AS v FROM t6 WHERE id = 2; + +--echo # the same rows through the tree path, which writes the elements out +--echo # when the group is asked for rather than as they arrive +SELECT JSON_ARRAYAGG(c ORDER BY id) AS v FROM t6; +SELECT JSON_ARRAYAGG(DISTINCT c) AS v FROM t6; + +--echo # a copy of the aggregate, which is made without fixing it again. +--echo # The values here are valid, so nothing should be said about them. +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v + FROM t6 GROUP BY id WITH ROLLUP; +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v + FROM t6 GROUP BY id; + +--echo # nesting composed by the aggregate itself, which puts each value +--echo # inside one array or object of its own +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, + JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj + FROM t2 WHERE id = 31; +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, + JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj + FROM t2 WHERE id = 30; + +--echo # +--echo # 9. The same character in a key of a constructor. +--echo # +--echo # A key is written by the same routine that writes one for the +--echo # aggregate, and a character no document can carry stops it the +--echo # same way. The constructors read nothing back, so a key and a +--echo # value alike throw the whole result away here; all that is left +--echo # of either is what is said about it, and both sides say which +--echo # argument it was. +--echo # + +--echo # the value side, which has always been named +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; +SELECT JSON_OBJECT('k', c) AS v FROM t6 WHERE id = 1; +--echo # the key side, in the first pair and in a later one +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 1; +SELECT JSON_OBJECT('a', 1, c, 2) AS v FROM t6 WHERE id = 3; +--echo # a key and a value of the same row, which stops at the key +SELECT JSON_OBJECT(c, c) AS v FROM t6 WHERE id = 1; +--echo # a row whose key can be written says nothing +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 2; + +--echo # +--echo # 10. The same character through JSON_QUOTE, which writes a value out +--echo # with the routine the constructors use but is not building a +--echo # document of its own. +--echo # +--echo # It has always answered NULL for a character it cannot write and +--echo # still does. What it never did was say so, and it was the only +--echo # caller of that routine which did not. +--echo # + +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 1; +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 3; +--echo # a row it can write says nothing +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 2; +--echo # the same value reaching a constructor instead, which has always +--echo # named the argument +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; + +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/main/func_json_oom.result b/mysql-test/main/func_json_oom.result new file mode 100644 index 0000000000000..6fd7573d3ae67 --- /dev/null +++ b/mysql-test/main/func_json_oom.result @@ -0,0 +1,309 @@ +# +# Reformatting a document when there is no room to grow the buffer. +# +# The reformatting pass asks for the length of its input plus a +# little, then writes the loose form into it. The loose form spends +# two characters where the compact form spends one, so an input +# carrying enough separators needs more room than was asked for, and +# the buffer has to grow part way through being written. +# +# These tests make that growing fail. What is being pinned is that +# every one of them ends either in an error or in a value that +# parses. A document with a piece missing out of the middle is not +# an answer, and no caller checks for one. +# +SET NAMES utf8mb4; +# +# 1. How much bigger the loose form is, which is what makes the +# room asked for too little. +# +SET @compact = CONCAT('[', REPEAT('1,', 49), '1]'); +SET @pairs = CONCAT('{', REPEAT('"a":1,', 24), '"a":1}'); +SET @nested = CONCAT('{"k":[', REPEAT('1,', 49), '1]}'); +SET @small = '[1,2,3]'; +SELECT LENGTH(@compact) AS compact_len, +LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; +compact_len grew_by +101 49 +SELECT LENGTH(@pairs) AS compact_len, +LENGTH(JSON_LOOSE(@pairs)) - LENGTH(@pairs) AS grew_by; +compact_len grew_by +151 49 +# a document short enough that the room asked for is enough +SELECT LENGTH(@small) AS compact_len, +LENGTH(JSON_LOOSE(@small)) - LENGTH(@small) AS grew_by; +compact_len grew_by +7 2 +# +# 2. Which uses of a formatter actually reformat anything. +# +# A formatter handed to something that wants JSON gives it the +# argument untouched, formatting being no business of a reader. +# Only a use that wants text runs the reformatting pass at all. +# +SELECT JSON_LOOSE(@pairs) AS as_text; +as_text +{"a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1} +SELECT JSON_TYPE(JSON_LOOSE(@pairs)) AS asked_as_json; +asked_as_json +OBJECT +SELECT CONCAT(JSON_LOOSE(@pairs), '') AS asked_as_text; +asked_as_text +{"a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1} +SELECT JSON_VALID(JSON_LOOSE(@pairs)) AS reader_sees_the_argument, +JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS reader_sees_the_output; +reader_sees_the_argument reader_sees_the_output +1 1 +# +# 3. The formatters, with no room to grow. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; +SELECT LENGTH(JSON_LOOSE(@compact)) AS len; +SET SESSION debug_dbug = DEFAULT; +# the same statements once the room is there again +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; +parses +1 +# +# 4. A document short enough not to need the room, which must be +# unaffected. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@small), '')) AS parses; +parses +1 +SELECT JSON_LOOSE(@small) AS v; +v +[1, 2, 3] +SET SESSION debug_dbug = DEFAULT; +# +# 5. The editing functions, which reformat what they have built +# before returning it. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; +# and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; +parses +1 +# +# 6. Reading a value out, which reformats what it read. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; +parses +1 +# +# 7. Where the value would go if it were returned at all. +# +# The allocator reports the failure itself, and reports it as +# fatal, so the statement ends before anything can be assigned and +# a handler cannot let execution carry on past it. This is what +# keeps a half written document from being seen. +# +SET @out = 'untouched'; +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SET @out = JSON_LOOSE(@compact); +SET SESSION debug_dbug = DEFAULT; +SELECT @out AS assigned_value; +assigned_value +untouched +SET @reached = 0; +SET @out2 = 'untouched'; +CREATE PROCEDURE p() +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET @out2 = JSON_LOOSE(@compact); +SET @reached = 1; +END $$ +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +CALL p(); +SET SESSION debug_dbug = DEFAULT; +SELECT @reached AS handler_carried_on, @out2 AS assigned_value; +handler_carried_on assigned_value +0 untouched +DROP PROCEDURE p; +# +# 8. Several rows, so the failure is met more than once and the +# statement does not turn on it happening first. +# +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2,3]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses +FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses +FROM t1 ORDER BY id; +id parses +1 1 +2 1 +3 1 +DROP TABLE t1; +# +# The sections above fail a buffer that is being grown, which +# reaches every write the reformatting pass makes. The ones below +# fail a single write instead, and they exist because growing +# cannot reach those writes at all: the room a document takes is +# asked for before it is written, so they never run short, and the +# arms that cope with their failing are arms nobody has run. Each +# is a pair - the statement without the failure and with it - so +# that what comes back is read against what the statement answers +# when nothing is wrong. +# +# +# 9. The quotes around a value. One at each end of the escaping, +# and failing the first never reaches the second, so they are +# asked about one at a time. +# +SELECT JSON_ARRAY('abc', 'def') AS whole, +JSON_OBJECT('k', 'abc') AS whole_value; +whole whole_value +["abc", "def"] {"k": "abc"} +SET SESSION debug_dbug = '+d,json_value_open_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_open_quote; +no_open_quote +NULL +SELECT JSON_OBJECT('k', 'abc') AS no_open_quote_value; +no_open_quote_value +NULL +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_value_close_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_close_quote; +no_close_quote +NULL +SELECT JSON_OBJECT('k', 'abc') AS no_close_quote_value; +no_close_quote_value +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 10. The room the escaping asks for, which is asked for once and +# up front at the widest the value could come to. +# +SET SESSION debug_dbug = '+d,json_escape_reserve_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_room_to_escape; +no_room_to_escape +NULL +SELECT JSON_QUOTE('abc') AS no_room_to_quote; +no_room_to_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 11. The quote that opens a key. A key is a string and nothing +# else, so it is written the way a value that is a string is. +# +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole; +whole +{"k": 1, "l": 2} +SET SESSION debug_dbug = '+d,json_keyname_quote_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_key_quote; +no_key_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 12. The quotes JSON_QUOTE puts round its argument, which are the +# same two writes as section 9 made by the function whose +# whole answer they are. +# +SELECT JSON_QUOTE('abc') AS whole, JSON_QUOTE('a"b') AS whole_escaped; +whole whole_escaped +"abc" "a\"b" +SET SESSION debug_dbug = '+d,json_quote_open_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_open_quote; +no_open_quote +NULL +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_quote_close_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_close_quote; +no_close_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 13. JSON_UNQUOTE returning an argument that is not a quoted +# string. It comes back as it stands, which takes a +# conversion where it carries no character set and a copy +# where it carries another one. Both roads are here, failing +# as they do in different places. +# +SELECT JSON_UNQUOTE(_binary'abc') AS from_binary, +JSON_UNQUOTE(_latin1'abc') AS from_latin1; +from_binary from_latin1 +abc abc +SET SESSION debug_dbug = '+d,json_unquote_as_is_out_of_memory'; +SELECT JSON_UNQUOTE(_binary'abc') AS no_room_binary; +no_room_binary +NULL +SELECT JSON_UNQUOTE(_latin1'abc') AS no_room_latin1; +no_room_latin1 +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 14. A value that is a document written in another character set +# than the one it is joining, which is converted on the way in +# so that what is spliced is written the way the rest of the +# document is. Both directions, a narrow value into a wide +# document and a wide one into a narrow document. +# +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS whole_into_ucs2; +whole_into_ucs2 +1 +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) +AS whole_from_latin1; +whole_from_latin1 +1 +SET SESSION debug_dbug = '+d,json_splice_convert_out_of_memory'; +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS no_room_into_ucs2; +no_room_into_ucs2 +NULL +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) +AS no_room_from_latin1; +no_room_from_latin1 +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 15. A path whose key is written in another character set, which +# is converted before it can be spliced into the document - a +# write of its own, and one that reaches nothing else. +# +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole, +JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS whole_set; +whole whole_set +{"a": 1, "b": 2} {"a": 1, "c": 3} +SET SESSION debug_dbug = '+d,json_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS no_room_for_key; +no_room_for_key +NULL +SELECT JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS no_room_for_key_set; +no_room_for_key_set +NULL +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_oom.test b/mysql-test/main/func_json_oom.test new file mode 100644 index 0000000000000..2c13fc5ccc99d --- /dev/null +++ b/mysql-test/main/func_json_oom.test @@ -0,0 +1,275 @@ +--source include/have_debug.inc + +--echo # +--echo # Reformatting a document when there is no room to grow the buffer. +--echo # +--echo # The reformatting pass asks for the length of its input plus a +--echo # little, then writes the loose form into it. The loose form spends +--echo # two characters where the compact form spends one, so an input +--echo # carrying enough separators needs more room than was asked for, and +--echo # the buffer has to grow part way through being written. +--echo # +--echo # These tests make that growing fail. What is being pinned is that +--echo # every one of them ends either in an error or in a value that +--echo # parses. A document with a piece missing out of the middle is not +--echo # an answer, and no caller checks for one. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. How much bigger the loose form is, which is what makes the +--echo # room asked for too little. +--echo # + +SET @compact = CONCAT('[', REPEAT('1,', 49), '1]'); +SET @pairs = CONCAT('{', REPEAT('"a":1,', 24), '"a":1}'); +SET @nested = CONCAT('{"k":[', REPEAT('1,', 49), '1]}'); +SET @small = '[1,2,3]'; +SELECT LENGTH(@compact) AS compact_len, + LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; +SELECT LENGTH(@pairs) AS compact_len, + LENGTH(JSON_LOOSE(@pairs)) - LENGTH(@pairs) AS grew_by; +--echo # a document short enough that the room asked for is enough +SELECT LENGTH(@small) AS compact_len, + LENGTH(JSON_LOOSE(@small)) - LENGTH(@small) AS grew_by; + +--echo # +--echo # 2. Which uses of a formatter actually reformat anything. +--echo # +--echo # A formatter handed to something that wants JSON gives it the +--echo # argument untouched, formatting being no business of a reader. +--echo # Only a use that wants text runs the reformatting pass at all. +--echo # + +SELECT JSON_LOOSE(@pairs) AS as_text; +SELECT JSON_TYPE(JSON_LOOSE(@pairs)) AS asked_as_json; +SELECT CONCAT(JSON_LOOSE(@pairs), '') AS asked_as_text; +SELECT JSON_VALID(JSON_LOOSE(@pairs)) AS reader_sees_the_argument, + JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS reader_sees_the_output; + +--echo # +--echo # 3. The formatters, with no room to grow. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; +--error 0,5 +SELECT LENGTH(JSON_LOOSE(@compact)) AS len; +SET SESSION debug_dbug = DEFAULT; + +--echo # the same statements once the room is there again +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; + +--echo # +--echo # 4. A document short enough not to need the room, which must be +--echo # unaffected. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@small), '')) AS parses; +SELECT JSON_LOOSE(@small) AS v; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 5. The editing functions, which reformat what they have built +--echo # before returning it. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; + +--echo # and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; + +--echo # +--echo # 6. Reading a value out, which reformats what it read. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; + +--echo # +--echo # 7. Where the value would go if it were returned at all. +--echo # +--echo # The allocator reports the failure itself, and reports it as +--echo # fatal, so the statement ends before anything can be assigned and +--echo # a handler cannot let execution carry on past it. This is what +--echo # keeps a half written document from being seen. +--echo # + +SET @out = 'untouched'; +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SET @out = JSON_LOOSE(@compact); +SET SESSION debug_dbug = DEFAULT; +SELECT @out AS assigned_value; + +SET @reached = 0; +SET @out2 = 'untouched'; +--delimiter $$ +CREATE PROCEDURE p() +BEGIN + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET @out2 = JSON_LOOSE(@compact); + SET @reached = 1; +END $$ +--delimiter ; +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +CALL p(); +SET SESSION debug_dbug = DEFAULT; +SELECT @reached AS handler_carried_on, @out2 AS assigned_value; +DROP PROCEDURE p; + +--echo # +--echo # 8. Several rows, so the failure is met more than once and the +--echo # statement does not turn on it happening first. +--echo # + +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2,3]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses + FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses + FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # The sections above fail a buffer that is being grown, which +--echo # reaches every write the reformatting pass makes. The ones below +--echo # fail a single write instead, and they exist because growing +--echo # cannot reach those writes at all: the room a document takes is +--echo # asked for before it is written, so they never run short, and the +--echo # arms that cope with their failing are arms nobody has run. Each +--echo # is a pair - the statement without the failure and with it - so +--echo # that what comes back is read against what the statement answers +--echo # when nothing is wrong. +--echo # + +--echo # +--echo # 9. The quotes around a value. One at each end of the escaping, +--echo # and failing the first never reaches the second, so they are +--echo # asked about one at a time. +--echo # +SELECT JSON_ARRAY('abc', 'def') AS whole, + JSON_OBJECT('k', 'abc') AS whole_value; + +SET SESSION debug_dbug = '+d,json_value_open_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_open_quote; +SELECT JSON_OBJECT('k', 'abc') AS no_open_quote_value; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_value_close_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_close_quote; +SELECT JSON_OBJECT('k', 'abc') AS no_close_quote_value; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 10. The room the escaping asks for, which is asked for once and +--echo # up front at the widest the value could come to. +--echo # +SET SESSION debug_dbug = '+d,json_escape_reserve_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_room_to_escape; +SELECT JSON_QUOTE('abc') AS no_room_to_quote; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 11. The quote that opens a key. A key is a string and nothing +--echo # else, so it is written the way a value that is a string is. +--echo # +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole; + +SET SESSION debug_dbug = '+d,json_keyname_quote_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_key_quote; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 12. The quotes JSON_QUOTE puts round its argument, which are the +--echo # same two writes as section 9 made by the function whose +--echo # whole answer they are. +--echo # +SELECT JSON_QUOTE('abc') AS whole, JSON_QUOTE('a"b') AS whole_escaped; + +SET SESSION debug_dbug = '+d,json_quote_open_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_open_quote; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_quote_close_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_close_quote; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 13. JSON_UNQUOTE returning an argument that is not a quoted +--echo # string. It comes back as it stands, which takes a +--echo # conversion where it carries no character set and a copy +--echo # where it carries another one. Both roads are here, failing +--echo # as they do in different places. +--echo # +SELECT JSON_UNQUOTE(_binary'abc') AS from_binary, + JSON_UNQUOTE(_latin1'abc') AS from_latin1; + +SET SESSION debug_dbug = '+d,json_unquote_as_is_out_of_memory'; +SELECT JSON_UNQUOTE(_binary'abc') AS no_room_binary; +SELECT JSON_UNQUOTE(_latin1'abc') AS no_room_latin1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 14. A value that is a document written in another character set +--echo # than the one it is joining, which is converted on the way in +--echo # so that what is spliced is written the way the rest of the +--echo # document is. Both directions, a narrow value into a wide +--echo # document and a wide one into a narrow document. +--echo # +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS whole_into_ucs2; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) + AS whole_from_latin1; + +SET SESSION debug_dbug = '+d,json_splice_convert_out_of_memory'; +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS no_room_into_ucs2; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) + AS no_room_from_latin1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 15. A path whose key is written in another character set, which +--echo # is converted before it can be spliced into the document - a +--echo # write of its own, and one that reaches nothing else. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole, + JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS whole_set; + +SET SESSION debug_dbug = '+d,json_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS no_room_for_key; +SELECT JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS no_room_for_key_set; +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_pathkey.result b/mysql-test/main/func_json_pathkey.result new file mode 100644 index 0000000000000..c707bca01ce05 --- /dev/null +++ b/mysql-test/main/func_json_pathkey.result @@ -0,0 +1,202 @@ +# +# The last step of a path, when the path and the document are written +# in different character sets. +# +# A path is parsed in its own character set. Every step of it is +# therefore a sequence of characters, and comparing a step against a +# key of the document is a comparison of characters, not of bytes. +# These tests cover the last step of the path, which is the one the +# mutating functions handle themselves instead of leaving it to the +# path walker. The interior steps are covered alongside as the +# contrast case. +# +# The keys below are non ASCII characters, written as the bytes that +# write them so that this file needs no character set of its own: +# C3 A9 is U+00E9, one byte in latin1 (E9) +# E2 82 AC is U+20AC, one byte in this server's latin1 (80) +# E6 BC A2 is U+6F22, which latin1 has no room for +# +SET NAMES utf8mb4; +# +# 1. Replacing a key that is already in the document. The key is the +# same character on both sides, encoded in two bytes in the path +# and with one in the document. +# +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"e":1}' USING latin1), +_utf8mb4'$."e"', 2)) AS ascii_key; +ascii_key +7B2265223A20327D +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS wide_key; +wide_key +7B22E9223A20327D +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS set_existing; +set_existing +7B22E9223A20327D +SELECT JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS still_valid; +still_valid +1 +# +# 2. The same document and path in one character set, as the control. +# +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, +_utf8mb4 0x242E22C3A922, 2)) AS same_cs; +same_cs +7B22C3A9223A20327D +SELECT CHARSET(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, +_utf8mb4 0x242E22C3A922, 2)) AS same_cs_charset; +same_cs_charset +utf8mb4 +# +# 3. An interior step, which the path walker handles. This one has +# always compared characters. +# +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A7B2262223A317D7D USING latin1), +_utf8mb4 0x242E22C3A9222E62, 2)) AS interior_step; +interior_step +7B22E9223A207B2262223A20327D7D +# +# 4. Adding a key that is not in the document. The bytes of the key +# have to arrive in the document's character set. +# +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS insert_wide_key; +insert_wide_key +7B2261223A20312C2022E9223A20327D +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS set_new_key; +set_new_key +7B2261223A20312C2022E9223A20327D +# read the key back with a path in the document's own character set +SELECT JSON_EXTRACT(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22C3A922, 2), +CONVERT(_utf8mb4 0x242E22C3A922 USING latin1)) AS read_back; +read_back +2 +# +# 5. The other direction: a document in the wider character set and a +# path in the narrower one. +# +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, +CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) +AS narrow_path; +narrow_path +7B22C3A9223A20327D +SELECT HEX(JSON_INSERT(_utf8mb4'{"a":1}', +CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) +AS narrow_insert; +narrow_insert +7B2261223A20312C2022C3A9223A20327D +# +# 6. Removing a key. JSON_REMOVE only has to match, never to write. +# +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B22C3A9223A312C2261223A327D USING latin1), +_utf8mb4 0x242E22C3A922)) AS remove_wide; +remove_wide +7B2261223A20327D +SELECT HEX(JSON_REMOVE(_utf8mb4 0x7B22C3A9223A312C2261223A327D, +_utf8mb4 0x242E22C3A922)) AS remove_same_cs; +remove_same_cs +7B2261223A20327D +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B2261223A7B22C3A9223A317D2C2262223A327D USING latin1), +_utf8mb4 0x242E612E22C3A922)) AS remove_interior; +remove_interior +7B2261223A207B7D2C202262223A20327D +# +# 7. A key the document's character set has no room for. U+20AC is +# in this server's latin1, U+6F22 is not. +# +# A key that will not convert is spliced in as it arrived, which is +# what was done with every key before any of them was converted, and +# a note says so. Whether the document that results can be read +# back is then decided by the document's character set, on the same +# terms as before. latin1 has a character at every byte, so the +# document always comes back. ucs2 reads its bytes two at a time, +# so what comes back depends on how many there are: an even number +# reads as characters and the document is kept, an odd one leaves a +# byte over and the document is refused, which is where a key that +# would not convert was refused before. +# +# U+6F22 is in ucs2 and so is converted like any other key. +# U+1F600 is not in ucs2, and takes four bytes; 'a' before it makes +# five. +# +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E282AC22, 2)) AS euro_into_latin1; +euro_into_latin1 +7B2261223A20312C202280223A20327D +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable; +unrepresentable +7B2261223A20312C2022E6BCA2223A20327D +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_set; +unrepresentable_set +7B2261223A20312C2022E6BCA2223A20327D +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_set' at position 0 +# a ucs2 document, which has room for this key and so takes it +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), +_utf8mb4 0x242E22E6BCA222, 2)) AS representable_ucs2; +representable_ucs2 +007B002200610022003A00200031002C002000226F220022003A00200032007D +# a key ucs2 has no room for, whose bytes go in as they are. Four +# of them read back as two characters, so the document is kept +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), +_utf8mb4 0x242E22F09F988022, 2)) AS unrepresentable_ucs2; +unrepresentable_ucs2 +007B002200610022003A00200031002C00200022F09F98800022003A00200032007D +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +# and five of them do not, so it is refused on the way back out +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), +_utf8mb4 0x242E2261F09F988022, 2)) AS odd_length_ucs2; +odd_length_ucs2 +NULL +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +Warning 4035 Broken JSON string in argument 1 to function 'json_insert' at position 30 +# and inside a statement that stops on a warning +CREATE TABLE t1 (id INT, doc LONGTEXT CHARACTER SET latin1); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t1 VALUES +(1, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)), +(2, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4'$.b', 2)); +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +SELECT id, HEX(doc) AS stored FROM t1 ORDER BY id; +id stored +1 7B2261223A20312C2022E6BCA2223A20327D +2 7B2261223A20312C202262223A20327D +SET SESSION sql_mode=DEFAULT; +DROP TABLE t1; +# matching an absent key needs no conversion, so it still just misses +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_replace; +unrepresentable_replace +7B2261223A20317D +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222)) AS unrepresentable_remove; +unrepresentable_remove +7B2261223A20317D +# +# 8. A key holding the character that ends a key in the document. +# This is not valid to splice, and stays a failure. +# +SELECT JSON_INSERT('{}', '$.a"b', 1) AS quote_in_key; +quote_in_key +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 +SELECT JSON_INSERT(CONVERT(_utf8mb4'{}' USING latin1), +_utf8mb4'$.a"b', 1) AS quote_in_key_cross_cs; +quote_in_key_cross_cs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 diff --git a/mysql-test/main/func_json_pathkey.test b/mysql-test/main/func_json_pathkey.test new file mode 100644 index 0000000000000..7f3a99196d235 --- /dev/null +++ b/mysql-test/main/func_json_pathkey.test @@ -0,0 +1,153 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # The last step of a path, when the path and the document are written +--echo # in different character sets. +--echo # +--echo # A path is parsed in its own character set. Every step of it is +--echo # therefore a sequence of characters, and comparing a step against a +--echo # key of the document is a comparison of characters, not of bytes. +--echo # These tests cover the last step of the path, which is the one the +--echo # mutating functions handle themselves instead of leaving it to the +--echo # path walker. The interior steps are covered alongside as the +--echo # contrast case. +--echo # +--echo # The keys below are non ASCII characters, written as the bytes that +--echo # write them so that this file needs no character set of its own: +--echo # C3 A9 is U+00E9, one byte in latin1 (E9) +--echo # E2 82 AC is U+20AC, one byte in this server's latin1 (80) +--echo # E6 BC A2 is U+6F22, which latin1 has no room for +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Replacing a key that is already in the document. The key is the +--echo # same character on both sides, encoded in two bytes in the path +--echo # and with one in the document. +--echo # + +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"e":1}' USING latin1), + _utf8mb4'$."e"', 2)) AS ascii_key; +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS wide_key; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS set_existing; +SELECT JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS still_valid; + +--echo # +--echo # 2. The same document and path in one character set, as the control. +--echo # + +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, + _utf8mb4 0x242E22C3A922, 2)) AS same_cs; +SELECT CHARSET(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, + _utf8mb4 0x242E22C3A922, 2)) AS same_cs_charset; + +--echo # +--echo # 3. An interior step, which the path walker handles. This one has +--echo # always compared characters. +--echo # + +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A7B2262223A317D7D USING latin1), + _utf8mb4 0x242E22C3A9222E62, 2)) AS interior_step; + +--echo # +--echo # 4. Adding a key that is not in the document. The bytes of the key +--echo # have to arrive in the document's character set. +--echo # + +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS insert_wide_key; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS set_new_key; +--echo # read the key back with a path in the document's own character set +SELECT JSON_EXTRACT(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22C3A922, 2), + CONVERT(_utf8mb4 0x242E22C3A922 USING latin1)) AS read_back; + +--echo # +--echo # 5. The other direction: a document in the wider character set and a +--echo # path in the narrower one. +--echo # + +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, + CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) + AS narrow_path; +SELECT HEX(JSON_INSERT(_utf8mb4'{"a":1}', + CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) + AS narrow_insert; + +--echo # +--echo # 6. Removing a key. JSON_REMOVE only has to match, never to write. +--echo # + +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B22C3A9223A312C2261223A327D USING latin1), + _utf8mb4 0x242E22C3A922)) AS remove_wide; +SELECT HEX(JSON_REMOVE(_utf8mb4 0x7B22C3A9223A312C2261223A327D, + _utf8mb4 0x242E22C3A922)) AS remove_same_cs; +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B2261223A7B22C3A9223A317D2C2262223A327D USING latin1), + _utf8mb4 0x242E612E22C3A922)) AS remove_interior; + +--echo # +--echo # 7. A key the document's character set has no room for. U+20AC is +--echo # in this server's latin1, U+6F22 is not. +--echo # +--echo # A key that will not convert is spliced in as it arrived, which is +--echo # what was done with every key before any of them was converted, and +--echo # a note says so. Whether the document that results can be read +--echo # back is then decided by the document's character set, on the same +--echo # terms as before. latin1 has a character at every byte, so the +--echo # document always comes back. ucs2 reads its bytes two at a time, +--echo # so what comes back depends on how many there are: an even number +--echo # reads as characters and the document is kept, an odd one leaves a +--echo # byte over and the document is refused, which is where a key that +--echo # would not convert was refused before. +--echo # +--echo # U+6F22 is in ucs2 and so is converted like any other key. +--echo # U+1F600 is not in ucs2, and takes four bytes; 'a' before it makes +--echo # five. +--echo # + +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E282AC22, 2)) AS euro_into_latin1; +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_set; +--echo # a ucs2 document, which has room for this key and so takes it +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), + _utf8mb4 0x242E22E6BCA222, 2)) AS representable_ucs2; +--echo # a key ucs2 has no room for, whose bytes go in as they are. Four +--echo # of them read back as two characters, so the document is kept +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), + _utf8mb4 0x242E22F09F988022, 2)) AS unrepresentable_ucs2; +--echo # and five of them do not, so it is refused on the way back out +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), + _utf8mb4 0x242E2261F09F988022, 2)) AS odd_length_ucs2; +--echo # and inside a statement that stops on a warning +CREATE TABLE t1 (id INT, doc LONGTEXT CHARACTER SET latin1); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t1 VALUES + (1, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)), + (2, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4'$.b', 2)); +SELECT id, HEX(doc) AS stored FROM t1 ORDER BY id; +SET SESSION sql_mode=DEFAULT; +DROP TABLE t1; +--echo # matching an absent key needs no conversion, so it still just misses +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_replace; +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222)) AS unrepresentable_remove; + +--echo # +--echo # 8. A key holding the character that ends a key in the document. +--echo # This is not valid to splice, and stays a failure. +--echo # + +SELECT JSON_INSERT('{}', '$.a"b', 1) AS quote_in_key; +SELECT JSON_INSERT(CONVERT(_utf8mb4'{}' USING latin1), + _utf8mb4'$.a"b', 1) AS quote_in_key_cross_cs; diff --git a/mysql-test/main/func_json_unquote.result b/mysql-test/main/func_json_unquote.result new file mode 100644 index 0000000000000..a7a158db8fa06 --- /dev/null +++ b/mysql-test/main/func_json_unquote.result @@ -0,0 +1,159 @@ +# +# What JSON_UNQUOTE reports its result to be, against what it +# actually returns. +# +# The function settles on utf8mb4_bin when the statement is prepared, +# before any argument has been read, so that is what every consumer +# downstream of it is told the bytes are. The bytes therefore have to +# arrive in that character set no matter what the argument was +# written in. Only one of the ways out of the function converts them. +# +SET NAMES utf8mb4; +# +# 1. What the function says about itself. +# +SELECT CHARSET(JSON_UNQUOTE(CONVERT('["a"]' USING latin1))) AS declared_cs, +COLLATION(JSON_UNQUOTE('["a"]')) AS declared_coll; +declared_cs declared_coll +utf8mb4 utf8mb4_bin +# +# 2. A string value. This is the path that unescapes, and it +# converts on the way out. +# +SELECT HEX(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS str_value; +str_value +C3A9 +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS chars; +chars +1 +# +# 3. A value that is not a string. There is nothing to unquote, so +# the argument is returned unchanged. +# +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS array_value; +array_value +5B22C3A9225D +SELECT HEX(JSON_UNQUOTE(CONVERT(X'7B2261223A22E9227D' USING latin1))) AS object_value; +object_value +7B2261223A22C3A9227D +SELECT HEX(JSON_UNQUOTE(CONVERT('123' USING latin1))) AS number_value; +number_value +313233 +SELECT HEX(JSON_UNQUOTE(CONVERT('true' USING latin1))) AS bool_value; +bool_value +74727565 +# +# 4. A document that does not parse. Also returned unchanged. +# +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E922' USING latin1))) AS broken_value; +broken_value +5B22C3A922 +# +# 5. The result used where its declared character set is taken at +# its word. +# +SELECT HEX(CONCAT(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)), 'z')) AS concatenated; +concatenated +5B22C3A9225D7A +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS chars; +chars +5 +SELECT JSON_VALID(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS still_json; +still_json +1 +SELECT HEX(UPPER(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)))) AS uppercased; +uppercased +5B22C389225D +# +# 6. A document held as bytes, which has no character set to +# convert from. +# +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22FF225D' AS BINARY))) AS from_binary; +from_binary +5B22C3BF225D +SELECT HEX(JSON_UNQUOTE(CAST(X'5B226122' AS BINARY))) AS from_binary_broken; +from_binary_broken +5B226122 +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22615D' AS BINARY))) AS from_binary_ascii; +from_binary_ascii +5B22615D +# the two ways out of the function have to say the same thing about +# the same byte. A value that IS a string is unescaped, reading each +# byte as the character of the same number; anything else is handed +# back converted, and it has to be read the same way. 0x80 and 0x9F +# are two of the 27 positions in 0x80-0x9F where latin1 would not: +# MariaDB's latin1 is cp1252 and makes 0x80 the euro sign. +SELECT HEX(JSON_UNQUOTE(CAST(X'228022' AS BINARY))) AS str_80; +str_80 +C280 +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2280225D' AS BINARY))) AS doc_80; +doc_80 +5B22C280225D +SELECT HEX(JSON_UNQUOTE(CAST(X'229F22' AS BINARY))) AS str_9f; +str_9f +C29F +SELECT HEX(JSON_UNQUOTE(CAST(X'5B229F225D' AS BINARY))) AS doc_9f; +doc_9f +5B22C29F225D +# and one of the five where it would agree, for contrast +SELECT HEX(JSON_UNQUOTE(CAST(X'228122' AS BINARY))) AS str_81; +str_81 +C281 +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2281225D' AS BINARY))) AS doc_81; +doc_81 +5B22C281225D +# +# 7. An argument already in the declared character set, which is +# the common case and must not move. +# +SELECT HEX(JSON_UNQUOTE(_utf8mb4'["a"]')) AS same_cs; +same_cs +5B2261225D +SELECT HEX(JSON_UNQUOTE(_utf8mb4'"a"')) AS same_cs_str; +same_cs_str +61 +# a document holding U+00E9, written here as its utf8mb4 bytes C3 A9 +SELECT HEX(JSON_UNQUOTE(_utf8mb4 0x5B22C3A9225D)) AS same_cs_wide; +same_cs_wide +5B22C3A9225D +# +# 8. An argument in a character set that admits every byte but has +# no character at some of them. +# +# The conversion has nowhere to put such a byte. What it does put +# there instead is not the value it was given, so the argument is +# returned as it arrived, which is the answer this call has +# always given, and the difference is said in a note. A note and +# not a warning: a warning becomes an error inside a statement +# running under strict mode, and a statement that used to finish +# would then stop finishing. +# +# cp1250 has no character at 0x81. +# +CREATE TABLE t1 (id INT, c TEXT CHARACTER SET cp1250); +INSERT INTO t1 VALUES (1, X'5B3181'), (2, X'5B2261225D'), (3, X'2281'); +SELECT id, HEX(c) AS stored, HEX(JSON_UNQUOTE(c)) AS unquoted +FROM t1 ORDER BY id; +id stored unquoted +1 5B3181 5B3181 +2 5B2261225D 5B2261225D +3 2281 2281 +Warnings: +Note 4035 Broken JSON string in argument 0 to function 'unquote' at position 0 +Warning 4035 Broken JSON string in argument 1 to function 'json_unquote' at position 1 +Note 4035 Broken JSON string in argument 0 to function 'unquote' at position 0 +# the same values inside a statement that stops on a warning. Row 3 +# is left out of it: reading a STRING that will not scan has always +# raised a warning of its own, from before any of this, and strict +# mode has always stopped on that one. +CREATE TABLE t2 (b BLOB); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t2 SELECT JSON_UNQUOTE(c) FROM t1 WHERE id < 3; +Warnings: +Note 4035 Broken JSON string in argument 0 to function 'unquote' at position 0 +SELECT HEX(b) AS stored FROM t2 ORDER BY b; +stored +5B2261225D +5B3181 +SET SESSION sql_mode=DEFAULT; +DROP TABLE t1, t2; diff --git a/mysql-test/main/func_json_unquote.test b/mysql-test/main/func_json_unquote.test new file mode 100644 index 0000000000000..66974764e7029 --- /dev/null +++ b/mysql-test/main/func_json_unquote.test @@ -0,0 +1,120 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # What JSON_UNQUOTE reports its result to be, against what it +--echo # actually returns. +--echo # +--echo # The function settles on utf8mb4_bin when the statement is prepared, +--echo # before any argument has been read, so that is what every consumer +--echo # downstream of it is told the bytes are. The bytes therefore have to +--echo # arrive in that character set no matter what the argument was +--echo # written in. Only one of the ways out of the function converts them. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. What the function says about itself. +--echo # + +SELECT CHARSET(JSON_UNQUOTE(CONVERT('["a"]' USING latin1))) AS declared_cs, + COLLATION(JSON_UNQUOTE('["a"]')) AS declared_coll; + +--echo # +--echo # 2. A string value. This is the path that unescapes, and it +--echo # converts on the way out. +--echo # + +SELECT HEX(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS str_value; +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS chars; + +--echo # +--echo # 3. A value that is not a string. There is nothing to unquote, so +--echo # the argument is returned unchanged. +--echo # + +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS array_value; +SELECT HEX(JSON_UNQUOTE(CONVERT(X'7B2261223A22E9227D' USING latin1))) AS object_value; +SELECT HEX(JSON_UNQUOTE(CONVERT('123' USING latin1))) AS number_value; +SELECT HEX(JSON_UNQUOTE(CONVERT('true' USING latin1))) AS bool_value; + +--echo # +--echo # 4. A document that does not parse. Also returned unchanged. +--echo # + +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E922' USING latin1))) AS broken_value; + +--echo # +--echo # 5. The result used where its declared character set is taken at +--echo # its word. +--echo # + +SELECT HEX(CONCAT(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)), 'z')) AS concatenated; +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS chars; +SELECT JSON_VALID(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS still_json; +SELECT HEX(UPPER(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)))) AS uppercased; + +--echo # +--echo # 6. A document held as bytes, which has no character set to +--echo # convert from. +--echo # + +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22FF225D' AS BINARY))) AS from_binary; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B226122' AS BINARY))) AS from_binary_broken; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22615D' AS BINARY))) AS from_binary_ascii; + +--echo # the two ways out of the function have to say the same thing about +--echo # the same byte. A value that IS a string is unescaped, reading each +--echo # byte as the character of the same number; anything else is handed +--echo # back converted, and it has to be read the same way. 0x80 and 0x9F +--echo # are two of the 27 positions in 0x80-0x9F where latin1 would not: +--echo # MariaDB's latin1 is cp1252 and makes 0x80 the euro sign. +SELECT HEX(JSON_UNQUOTE(CAST(X'228022' AS BINARY))) AS str_80; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2280225D' AS BINARY))) AS doc_80; +SELECT HEX(JSON_UNQUOTE(CAST(X'229F22' AS BINARY))) AS str_9f; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B229F225D' AS BINARY))) AS doc_9f; +--echo # and one of the five where it would agree, for contrast +SELECT HEX(JSON_UNQUOTE(CAST(X'228122' AS BINARY))) AS str_81; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2281225D' AS BINARY))) AS doc_81; + +--echo # +--echo # 7. An argument already in the declared character set, which is +--echo # the common case and must not move. +--echo # + +SELECT HEX(JSON_UNQUOTE(_utf8mb4'["a"]')) AS same_cs; +SELECT HEX(JSON_UNQUOTE(_utf8mb4'"a"')) AS same_cs_str; +--echo # a document holding U+00E9, written here as its utf8mb4 bytes C3 A9 +SELECT HEX(JSON_UNQUOTE(_utf8mb4 0x5B22C3A9225D)) AS same_cs_wide; + +--echo # +--echo # 8. An argument in a character set that admits every byte but has +--echo # no character at some of them. +--echo # +--echo # The conversion has nowhere to put such a byte. What it does put +--echo # there instead is not the value it was given, so the argument is +--echo # returned as it arrived, which is the answer this call has +--echo # always given, and the difference is said in a note. A note and +--echo # not a warning: a warning becomes an error inside a statement +--echo # running under strict mode, and a statement that used to finish +--echo # would then stop finishing. +--echo # +--echo # cp1250 has no character at 0x81. +--echo # + +CREATE TABLE t1 (id INT, c TEXT CHARACTER SET cp1250); +INSERT INTO t1 VALUES (1, X'5B3181'), (2, X'5B2261225D'), (3, X'2281'); +SELECT id, HEX(c) AS stored, HEX(JSON_UNQUOTE(c)) AS unquoted + FROM t1 ORDER BY id; + +--echo # the same values inside a statement that stops on a warning. Row 3 +--echo # is left out of it: reading a STRING that will not scan has always +--echo # raised a warning of its own, from before any of this, and strict +--echo # mode has always stopped on that one. +CREATE TABLE t2 (b BLOB); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t2 SELECT JSON_UNQUOTE(c) FROM t1 WHERE id < 3; +SELECT HEX(b) AS stored FROM t2 ORDER BY b; +SET SESSION sql_mode=DEFAULT; + +DROP TABLE t1, t2; diff --git a/mysql-test/suite/json/r/json_no_table.result b/mysql-test/suite/json/r/json_no_table.result index 5b838a5414765..eda51094356ff 100644 --- a/mysql-test/suite/json/r/json_no_table.result +++ b/mysql-test/suite/json/r/json_no_table.result @@ -2930,7 +2930,7 @@ Warnings: Warning 4036 Character disallowed in JSON in argument 1 to function 'json_compact' at position 1 select json_unquote(json_compact(st_geomfromtext('point(1 1)'))); json_unquote(json_compact(st_geomfromtext('point(1 1)'))) -ð?ð? +ð?ð? SELECT JSON_UNQUOTE( '"abc"' ); JSON_UNQUOTE( '"abc"' ) abc diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index 2509f8f63e2a2..021251ed0eb79 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -152,7 +152,26 @@ void *my_realloc(PSI_memory_key key, void *old_point, size_t size, myf my_flags) DBUG_ASSERT((old_flags & 1) == MY_TEST(my_flags & MY_THREAD_SPECIFIC)); size= ALIGN_SIZE(size); - mh= sf_realloc(old_mh, size + HEADER_SIZE, my_flags); + + /* + Growing an allocation can fail where the first one succeeded, and a + buffer that is already held is where most of the unchecked writes in + the server are: the code that asked for the memory is long gone, and + what is left is an append that assumes it will fit. my_malloc() + above can be made to fail for a test; without the same here, none of + those paths can be reached at all. + + Kept apart from "simulate_out_of_memory" rather than folded into it, + so that a test can fail the growing of a buffer without also failing + every first allocation that happens to fall in the same window - + which is usually the whole point, the two failing in different + places for different reasons. A request that shrinks is left alone, + keeping the guarantee below that a smaller size never fails. + */ + if (size > old_size && DBUG_IF("simulate_realloc_out_of_memory")) + mh= NULL; + else + mh= sf_realloc(old_mh, size + HEADER_SIZE, my_flags); if (mh == NULL) { diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 6acf31a8f6753..47c435536956a 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -94,6 +94,54 @@ append_simple(String *s, const uchar *a, size_t a_len) } +static void report_bad_chr_note(const char *fname, int n_arg); + + +/* + Appends the key of a path step to the document being built. + + A path is written in its own character set, which is not necessarily + the one of the document, so the key may have to be converted before it + can be spliced in. Only a conversion that lost nothing is taken: a + key holding a character the document's character set has no room for + goes in as it arrived, which is what was done with every key before + any key was converted at all, and a note says so. + + Whether the document that results can still be read is then decided + by the document's character set, exactly as it was decided before: + one that admits every byte keeps the document, and one that does not + refuses it on the way back out. Deciding it here instead would take + away documents that have always been given. + + Nothing else is done to the key: one holding a character that is not + allowed unescaped inside a JSON string stays invalid, and is caught + where it was caught before. +*/ +static bool __attribute__((warn_unused_result)) +append_json_path_key(String *s, const json_path_step_t *step, + CHARSET_INFO *path_cs, String *tmp_key, + const char *fname, int n_arg) +{ + size_t key_len= (size_t) (step->key_end - step->key); + uint errors; + + if (!my_charset_same(path_cs, s->charset())) + { + if (tmp_key->copy((const char *) step->key, key_len, path_cs, + s->charset(), &errors) || + DBUG_IF("json_path_key_out_of_memory")) + return true; /* Out of memory. */ + + if (!errors) + return append_simple(s, tmp_key->ptr(), tmp_key->length()); + + report_bad_chr_note(fname, n_arg); + } + + return append_simple(s, step->key, key_len); +} + + /* Appends JSON string to the String object taking charsets in consideration. @@ -135,7 +183,7 @@ bool st_append_json(String *s, Appends arbitrary String to the JSON string taking charsets in consideration. */ -int st_append_escaped(String *s, const String *a) +json_append_result st_append_escaped(String *s, const String *a) { /* In the worst case one character from the 'a' string @@ -143,17 +191,19 @@ int st_append_escaped(String *s, const String *a) */ int str_len= a->length() * 12 * s->charset()->mbmaxlen / a->charset()->mbminlen; - if (!s->reserve(str_len, 1024) && - (str_len= - json_escape(a->charset(), (uchar *) a->ptr(), (uchar *)a->end(), - s->charset(), - (uchar *) s->end(), (uchar *)s->end() + str_len)) > 0) - { - s->length(s->length() + str_len); - return 0; - } - - return a->length(); + if (s->reserve(str_len, 1024) || + DBUG_IF("json_escape_reserve_out_of_memory")) + return JSON_APPEND_OOM; + + str_len= json_escape(a->charset(), (uchar *) a->ptr(), (uchar *)a->end(), + s->charset(), + (uchar *) s->end(), (uchar *)s->end() + str_len); + if (str_len < 0) + return str_len == JSON_ERROR_ILLEGAL_SYMBOL ? + JSON_APPEND_BAD_CHR : JSON_APPEND_OOM; + + s->length(s->length() + str_len); + return JSON_APPEND_OK; } @@ -320,7 +370,20 @@ static int json_nice(json_engine_t *je, String *nice_js, if (nice_js->alloc(je->s.str_end - je->s.c_str + 32)) goto error; - + + /* + The buffer above is a guess, and the loose form can outgrow it: it + spends two characters where the compact form spends one, so an input + with more than sixteen separators in it needs more room than was + asked for. Everything written below therefore may have to grow the + buffer, and this is where a test arranges for that growing to fail. + Disarmed at both ways out, so that only the writing done here is + affected. + */ + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("+d,simulate_realloc_out_of_memory");); + + DBUG_ASSERT(mode != Item_func_json_format::DETAILED || (tab_size >= 0 && tab_size <= TAB_SIZE_LIMIT)); @@ -358,25 +421,25 @@ static int json_nice(json_engine_t *je, String *nice_js, if (unlikely(je->s.error)) goto error; - if (!first_value) - nice_js->append(comma, comma_len); + if (!first_value && nice_js->append(comma, comma_len)) + goto error; if (mode == Item_func_json_format::DETAILED && append_tab(nice_js, depth, tab_size)) goto error; - nice_js->append('"'); - if (append_simple(nice_js, key_start, key_end - key_start)) + if (nice_js->append('"') || + append_simple(nice_js, key_start, key_end - key_start) || + nice_js->append(colon, colon_len)) goto error; - nice_js->append(colon, colon_len); } /* now we have key value to handle, so no 'break'. */ DBUG_ASSERT(je->state == JST_VALUE); goto handle_value; case JST_VALUE: - if (!first_value) - nice_js->append(comma, comma_len); + if (!first_value && nice_js->append(comma, comma_len)) + goto error; if (mode == Item_func_json_format::DETAILED && depth > 0 && @@ -392,8 +455,9 @@ static int json_nice(json_engine_t *je, String *nice_js, je->value_end - je->value_begin)) goto error; - curr_str.copy((const char *)je->value_begin, - je->value_end - je->value_begin, je->s.cs); + if (curr_str.copy((const char *)je->value_begin, + je->value_end - je->value_begin, je->s.cs)) + goto error; value_len= je->value_end - je->value_begin; first_value= 0; if (value_size != -1) @@ -405,7 +469,9 @@ static int json_nice(json_engine_t *je, String *nice_js, depth > 0 && !(curr_state != JST_KEY) && append_tab(nice_js, depth, tab_size)) goto error; - nice_js->append((je->value_type == JSON_VALUE_OBJECT) ? "{" : "[", 1); + if (nice_js->append((je->value_type == JSON_VALUE_OBJECT) ? + "{" : "[", 1)) + goto error; first_value= 1; value_size= (je->value_type == JSON_VALUE_OBJECT) ? -1: 0; depth++; @@ -426,10 +492,12 @@ static int json_nice(json_engine_t *je, String *nice_js, nice_js->length(nice_js->length() - value_len); for (auto i = 0; i < (depth + 1) * tab_size + 1; i++) nice_js->chop(); - nice_js->append(curr_str); + if (nice_js->append(curr_str)) + goto error; } - - nice_js->append((je->state == JST_OBJ_END) ? "}": "]", 1); + + if (nice_js->append((je->state == JST_OBJ_END) ? "}": "]", 1)) + goto error; first_value= 0; value_size= -1; break; @@ -439,9 +507,13 @@ static int json_nice(json_engine_t *je, String *nice_js, }; } while (json_scan_next(je) == 0); + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("-d,simulate_realloc_out_of_memory");); return je->s.error || *je->killed_ptr; error: + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("-d,simulate_realloc_out_of_memory");); return 1; } @@ -508,6 +580,30 @@ void report_json_error_ex(const char *js, json_engine_t *je, } +/* + Says that a character had nowhere to go, in the cases where that is + all that is being said: the bytes are passed on unchanged, as they + always were, and this note is the only sign that anything was amiss. + + A note and not a warning. A warning becomes an error inside a + statement running under strict mode, and every caller of this is + somewhere the released server said nothing at all, so a warning would + stop statements that used to finish. + + The argument is numbered the way the caller numbers it in its other + diagnostics, and the position is the beginning of it: what is + reported here is that the writing stopped, not where it stopped. +*/ +static void report_bad_chr_note(const char *fname, int n_arg) +{ + THD *thd= current_thd; + + if (thd) + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_JSON_BAD_CHR, + ER_THD(thd, ER_JSON_BAD_CHR), n_arg, fname, 0); +} + + #define NO_WILDCARD_ALLOWED 1 #define SHOULD_END_WITH_ARRAY 2 @@ -906,6 +1002,7 @@ bool Item_func_json_quote::fix_length_and_dec(THD *thd) String *Item_func_json_quote::val_str(String *str) { String *s= args[0]->val_str(&tmp_s); + json_append_result rc; if ((null_value= (args[0]->null_value || args[0]->result_type() != STRING_RESULT))) @@ -914,16 +1011,34 @@ String *Item_func_json_quote::val_str(String *str) str->length(0); str->set_charset(&my_charset_utf8mb4_bin); - if (str->append('"') || - st_append_escaped(str, s) || - str->append('"')) + if (str->append('"') || DBUG_IF("json_quote_open_out_of_memory")) + goto error; + + if ((rc= st_append_escaped(str, s))) { - /* Report an error. */ - null_value= 1; - return 0; + /* + A character no document can carry stops the writing, and this + function has always answered NULL for it. What it has never done + is SAY so: the three other JSON functions that write a value + through st_append_escaped() all report the note, and a value moved + from one of them to this one lost the only signal there is for the + condition. The optimizer trace calls it as well and reports + nothing, a trace not being an answer anybody asked for. The answer + is unchanged - only the silence is. + */ + if (rc == JSON_APPEND_BAD_CHR) + report_bad_chr_note(func_name(), 1); + goto error; } + if (str->append('"') || DBUG_IF("json_quote_close_out_of_memory")) + goto error; + return str; + +error: + null_value= 1; + return 0; } @@ -960,6 +1075,83 @@ String *Item_func_json_unquote::read_json(json_engine_t *je) } +/* + Returns the argument unchanged, but in the character set this item + declares. The bytes and the label have to agree: a value encoded one + way and labelled another is read wrongly by everything downstream of + it, and the character set is settled at fix time, so it is the bytes + that have to move. + + An argument that carries no character set at all is read one byte to + one character, which is how json_unescape() already reads it when the + value turns out to be a string. Both ways out of the function then + say the same thing about the same argument. +*/ +String *Item_func_json_unquote::return_as_is(String *str, String *js) +{ + CHARSET_INFO *from_cs= js->charset(); + uint errors; + + if (my_charset_same(from_cs, collation.collation)) + return js; + + if (from_cs == &my_charset_bin) + { + /* + An argument that carries no character set is read one byte to one + character, which is the reading json_unescape() gives it on the + other way out: my_charset_bin's mb_wc returns the byte itself. + Reading it any other way would have the two exits of this function + name the same byte as two different characters. latin1 in + particular is NOT that reading - MariaDB's latin1 is cp1252, which + differs from it at 27 of the 32 positions in 0x80-0x9F. + + Converted rather than copied, and not through String::copy(), + which would not convert: needs_conversion() answers false for a + binary source whose length divides the destination's mbminlen, and + utf8mb4's is 1, so the bytes would go across untouched and be + relabelled - 0x80 would leave here as a lone continuation byte, + which is not a character of the set the result says it is in. + copy_and_convert() runs the conversion that short-circuit skips. + */ + uint32 room= (uint32) (collation.collation->mbmaxlen * js->length()); + + if (str->alloc(room) || DBUG_IF("json_unquote_as_is_out_of_memory")) + { + null_value= 1; + return NULL; + } + str->length(copy_and_convert((char *) str->ptr(), room, + collation.collation, js->ptr(), + js->length(), from_cs, &errors)); + str->set_charset(collation.collation); + } + else if (str->copy(js->ptr(), js->length(), from_cs, + collation.collation, &errors) || + DBUG_IF("json_unquote_as_is_out_of_memory")) + { + null_value= 1; + return NULL; + } + + /* + Only a conversion that lost nothing is worth having. A character + with nowhere to go comes out of the conversion as a substitute for + itself, and returning that would answer differently than this + call has always answered - the argument would come back altered + rather than merely mislabelled. The label is the lesser of the two + wrongs, so the bytes stay as they arrived and the note says why. + */ + if (errors) + { + report_bad_chr_note("unquote", 0); + return js; + } + + return str; +} + + String *Item_func_json_unquote::val_str(String *str) { json_engine_t je; @@ -970,7 +1162,7 @@ String *Item_func_json_unquote::val_str(String *str) return NULL; if (unlikely(je.s.error) || je.value_type != JSON_VALUE_STRING) - return js; + return return_as_is(str, js); int buf_len= je.value_len; if (js->charset()->cset != my_charset_utf8mb4_bin.cset) @@ -1006,7 +1198,7 @@ String *Item_func_json_unquote::val_str(String *str) ER_JSON_BAD_CHR, ER_THD(current_thd, ER_JSON_BAD_CHR), 0, "unquote", 0); } - return js; + return return_as_is(str, js); } @@ -1936,7 +2128,180 @@ bool is_json_type(const Item *item) } -static int append_json_value(String *str, Item *item, String *tmp_val) +/* + Appends a value whose type is JSON, which is spliced into the result + as it stands instead of being quoted as a string. Any complete JSON + value counts, a bare number or string as much as an object: what the + type buys the value is that its own punctuation is kept rather than + escaped away. + + Being typed as JSON is not the same as being JSON. The type comes + from a check constraint, the constraint can be switched off for the + duration of a statement, and the bytes it would have rejected stay in + the column afterwards. Nothing on the way from the column to here + reads them. + + So the value is parsed as it is copied, but a value that does not + parse is copied ALL THE SAME, with a note saying so. It has always + been copied, callers have always been able to see the result, and + what a released server does is not something to take away from the + people relying on it; the note is what is new. Reading the result + back is where a caller finds out, exactly as before. What the parse + is for is to know WHETHER to say anything - and, once the trust + predicates exist, to keep an unparseable value from ever being taken + on trust by a later reader. + + A value written in another character set is converted first, because + the result is read in the character set it is being built in, not in + the one the value arrived in. The quoting arm converts the same way, + through json_escape(); a value spliced as JSON cannot go through + json_escape() without its punctuation being escaped along with + everything else, so it is converted whole and then parsed as whatever + it has become. A character set that cannot hold the value leaves the + bytes where they were, again because that is what was done before. + + 'depth' is how many structures the value will sit inside once it has + been spliced. Its own nesting adds to that, and it is the total that + the scanner's limit applies to. +*/ +static int append_json_typed_value(String *str, const String *sv, uint depth, + const char *fname, int n_param) +{ + StringBuffer cnv; + const char *ptr= sv->ptr(); + size_t length= sv->length(); + json_engine_t je; + int max_level= 0; + + /* + Bytes that carry no character set are left where they are, in both + directions and for opposite reasons. Going into a result that is + bytes, there is no character set to convert to. Coming out of a + value that is bytes, String::copy() cannot convert at all - + needs_conversion() answers false across that boundary, so the copy + would relabel the bytes and change nothing else, which is the one + thing this must not do. The scan below still reads whatever gets + appended in the character set of the result, so bytes that do not + encode a JSON value there are still refused. + */ + if (!my_charset_same(sv->charset(), str->charset()) && + sv->charset() != &my_charset_bin && str->charset() != &my_charset_bin) + { + uint errors; + + if (cnv.copy(sv->ptr(), sv->length(), sv->charset(), str->charset(), + &errors) || + DBUG_IF("json_splice_convert_out_of_memory")) + return 1; /* Out of memory. */ + + /* + Only a conversion that lost nothing is worth having. One that + substituted a character has changed the value, and putting a + changed value in would be worse than leaving the bytes as they + arrived, which is what was always done with them. The parse + below reads them as the result will be read and says so. + */ + if (!errors) + { + ptr= cnv.ptr(); + length= cnv.length(); + } + } + + /* + The deepest the value goes is read off the scanner as it runs rather + than charged to it in advance. Starting the scan with stack_p + already raised looks like the shorter way to ask the same question, + but json_scan_start() marks stack[0] as the end of the document and + the scanner stops when it pops back to that mark, so raising the + pointer past it leaves unwritten slots underneath and the scan never + finishes. + */ + json_scan_start(&je, str->charset(), (const uchar *) ptr, + (const uchar *) ptr + length); + while (json_scan_next(&je) == 0) + { + if (je.stack_p > max_level) + max_level= je.stack_p; + } + + if (je.s.error == 0 && max_level + (int) depth >= JSON_DEPTH_LIMIT) + je.s.error= JE_DEPTH; + + /* + Said here rather than left to the caller: the callers report through + the engine they scanned the DOCUMENT with, which knows nothing about + what went wrong with a value, and reports nothing at all when that + engine is not itself in error. + + A note and not a warning, because a warning becomes an error inside + a statement running under strict mode, and a statement that used to + finish would then stop finishing. What is being added here is + something to read, not a new way to fail. json_valid() says the + same thing at the same level for the same reason. + */ + if (je.s.error) + report_json_error_ex(ptr, &je, fname, n_param, + Sql_condition::WARN_LEVEL_NOTE); + + return append_simple(str, ptr, length); +} + + +/* + Writes a value that is not itself a document, as a JSON string when + the value is one and bare otherwise. + + A character that cannot be written into a document at all leaves the + value part written and stops there, which is what has always happened + to it. The caller is told which of the two failures it was, because + only one of them is a reason to give up on the whole result. +*/ +static json_append_result __attribute__((warn_unused_result)) +append_escaped_value(String *str, const String *sv, bool quoted, + const char *fname, int n_param) +{ + json_append_result rc; + + if ((quoted && str->append('"')) || + DBUG_IF("json_value_open_quote_out_of_memory")) + return JSON_APPEND_OOM; + + if ((rc= st_append_escaped(str, sv))) + { + if (rc == JSON_APPEND_BAD_CHR) + report_bad_chr_note(fname, n_param + 1); + return rc; + } + + if ((quoted && str->append('"')) || + DBUG_IF("json_value_close_quote_out_of_memory")) + return JSON_APPEND_OOM; + + return JSON_APPEND_OK; +} + + +__attribute__((warn_unused_result)) +/* + Appends one value to the document being built. + + 'depth' is how many structures the value will end up inside, and it is + consulted for EVERY value rather than only for one whose type is JSON: + it is recorded before the type is looked at, because a value goes in + where the caller says whatever it turns out to be. The functions that + BUILD a document pass 1, the value going directly inside the array or + object they are making. The functions that EDIT one pass the depth + the path reached, taken off the scanner as je.stack_p and raised by + whatever wrap they are adding - none of them passes 0. + + 'fname' and 'n_param' name the function and the argument a complaint + is about, and they reach BOTH arms: a value spliced in as a document + complains through the trailing-text and depth notes, and one written + out as a string complains through the bad-character note. +*/ +static int append_json_value(String *str, Item *item, String *tmp_val, + uint depth, const char *fname, int n_param) { if (item->type_handler()->is_bool_type()) { @@ -1965,15 +2330,10 @@ static int append_json_value(String *str, Item *item, String *tmp_val) if (item->null_value) goto append_null; if (is_json_type(item)) - return str->append(sv->ptr(), sv->length()); + return append_json_typed_value(str, sv, depth, fname, n_param); - if (item->result_type() == STRING_RESULT) - { - return str->append('"') || - st_append_escaped(str, sv) || - str->append('"'); - } - return st_append_escaped(str, sv); + return append_escaped_value(str, sv, item->result_type() == STRING_RESULT, + fname, n_param); } append_null: @@ -1981,8 +2341,11 @@ static int append_json_value(String *str, Item *item, String *tmp_val) } +/* The same, reading the value out of a row rather than off an Item. */ +__attribute__((warn_unused_result)) static int append_json_value_from_field(String *str, - Item *i, Field *f, const uchar *key, size_t offset, String *tmp_val) + Item *i, Field *f, const uchar *key, size_t offset, String *tmp_val, + uint depth, const char *fname, int n_param) { if (i->type_handler()->is_bool_type()) { @@ -2011,15 +2374,10 @@ static int append_json_value_from_field(String *str, if (f->is_null_in_record(key)) goto append_null; if (is_json_type(i)) - return str->append(sv->ptr(), sv->length()); + return append_json_typed_value(str, sv, depth, fname, n_param); - if (i->result_type() == STRING_RESULT) - { - return str->append('"') || - st_append_escaped(str, sv) || - str->append('"'); - } - return st_append_escaped(str, sv); + return append_escaped_value(str, sv, i->result_type() == STRING_RESULT, + fname, n_param); } append_null: @@ -2027,18 +2385,41 @@ static int append_json_value_from_field(String *str, } -static int append_json_keyname(String *str, Item *item, String *tmp_val) +/* + Writes a key, which is a JSON string and nothing else. + + A character that cannot be written into a document at all leaves the + key part written and stops there, the same as it does in a value - + and the caller is told which of the two failures it was for the same + reason. What the caller then does with a key it could not write is + its own business: a constructor gives up on the whole document and an + aggregate finishes the pair around what went in, but neither of them + can say so without being told, and a key that could not be written is + worth as much saying as a value that could not. +*/ +static json_append_result append_json_keyname(String *str, Item *item, + String *tmp_val, + const char *fname, int n_param) { + json_append_result rc; String *sv= item->val_str(tmp_val); if (item->null_value) goto append_null; - return str->append('"') || - st_append_escaped(str, sv) || - str->append("\": ", 3); + if (str->append('"') || DBUG_IF("json_keyname_quote_out_of_memory")) + return JSON_APPEND_OOM; + + if ((rc= st_append_escaped(str, sv))) + { + if (rc == JSON_APPEND_BAD_CHR) + report_bad_chr_note(fname, n_param + 1); + return rc; + } + + return str->append("\": ", 3) ? JSON_APPEND_OOM : JSON_APPEND_OK; append_null: - return str->append("\"\": ", 4); + return str->append("\"\": ", 4) ? JSON_APPEND_OOM : JSON_APPEND_OK; } @@ -2096,13 +2477,15 @@ String *Item_func_json_array::val_str(String *str) str->set_charset(collation.collation); if (str->append('[') || - ((arg_count > 0) && append_json_value(str, args[0], &tmp_val))) + ((arg_count > 0) && + append_json_value(str, args[0], &tmp_val, 1, func_name(), 0))) goto err_return; for (n_arg=1; n_arg < arg_count; n_arg++) { if (str->append(", ", 2) || - append_json_value(str, args[n_arg], &tmp_val)) + append_json_value(str, args[n_arg], &tmp_val, 1, func_name(), + (int) n_arg)) goto err_return; } @@ -2222,7 +2605,8 @@ String *Item_func_json_array_append::val_str(String *str) str->q_append(js->ptr(), ar_end-(const uchar *) js->ptr()); if (n_items) str->append(", ", 2); - if (append_json_value(str, args[n_arg+1], &tmp_val)) + if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1)) goto return_null; /* Out of memory. */ if (str->reserve(str_rest_len, 1024)) @@ -2249,7 +2633,8 @@ String *Item_func_json_array_append::val_str(String *str) if (str->append('[') || str->append((const char *) c_from, c_to - c_from) || str->append(", ", 2) || - append_json_value(str, args[n_arg+1], &tmp_val) || + append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1) || str->append(']') || str->append((const char *) je.s.c_str, js->end() - (const char *) je.s.c_str)) @@ -2406,7 +2791,8 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 1); goto return_null; /* Out of memory. */ } - if (append_json_value(str, args[n_arg+1], &tmp_val)) + if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ @@ -2445,7 +2831,8 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 2); goto return_null; /* Out of memory. */ } - if (append_json_value(str, args[n_arg+1], &tmp_val)) + if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ @@ -2500,15 +2887,17 @@ String *Item_func_json_object::val_str(String *str) if (str->append('{') || (arg_count > 0 && - (append_json_keyname(str, args[0], &tmp_val) || - append_json_value(str, args[1], &tmp_val)))) + (append_json_keyname(str, args[0], &tmp_val, func_name(), 0) || + append_json_value(str, args[1], &tmp_val, 1, func_name(), 1)))) goto err_return; for (n_arg=2; n_arg < arg_count; n_arg+=2) { if (str->append(", ", 2) || - append_json_keyname(str, args[n_arg], &tmp_val) || - append_json_value(str, args[n_arg+1], &tmp_val)) + append_json_keyname(str, args[n_arg], &tmp_val, func_name(), + (int) n_arg) || + append_json_value(str, args[n_arg+1], &tmp_val, 1, func_name(), + (int) n_arg + 1)) goto err_return; } @@ -3403,6 +3792,7 @@ String *Item_func_json_insert::val_str(String *str) String *js= args[0]->val_json(&tmp_js); uint n_arg, n_path; json_string_t key_name; + StringBuffer tmp_key; THD *thd; DBUG_ASSERT(fixed()); @@ -3415,7 +3805,6 @@ String *Item_func_json_insert::val_str(String *str) str->set_charset(collation.collation); tmp_js.set_charset(collation.collation); - json_string_set_cs(&key_name, collation.collation); for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { @@ -3513,7 +3902,8 @@ String *Item_func_json_insert::val_str(String *str) if ((do_array_autowrap && (append_simple(str, v_from, je.s.c_str - v_from) || str->append(", ", 2))) || - append_json_value(str, args[n_arg+1], &tmp_val) || + append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1) || (do_array_autowrap && str->append(']')) || append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) goto js_error; /* Out of memory. */ @@ -3555,7 +3945,8 @@ String *Item_func_json_insert::val_str(String *str) str->length(0); if (append_simple(str, js->ptr(), v_to - js->ptr()) || (n_item > 0 && str->append(", ", 2)) || - append_json_value(str, args[n_arg+1], &tmp_val) || + append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1) || append_simple(str, v_to, js->end() - v_to)) goto js_error; /* Out of memory. */ } @@ -3566,6 +3957,14 @@ String *Item_func_json_insert::val_str(String *str) if (je.value_type != JSON_VALUE_OBJECT) continue; + /* + The key of the step is written in the character set of the path, + which is not necessarily the one the document is written in. + Comparing it against a key of the document compares characters, + so it has to be read in its own character set. + */ + json_string_set_cs(&key_name, c_path->p.s.cs); + while (json_scan_next(&je) == 0 && je.state != JST_OBJ_END) { switch (je.state) @@ -3593,10 +3992,16 @@ String *Item_func_json_insert::val_str(String *str) str->length(0); if (append_simple(str, js->ptr(), v_to - js->ptr()) || (n_key > 0 && str->append(", ", 2)) || - str->append('"') || - append_simple(str, lp->key, lp->key_end - lp->key) || - str->append("\":", 2) || - append_json_value(str, args[n_arg+1], &tmp_val) || + str->append('"')) + goto js_error; /* Out of memory. */ + + if (append_json_path_key(str, lp, c_path->p.s.cs, &tmp_key, + func_name(), (int) n_arg + 1)) + goto js_error; /* Out of memory. */ + + if (str->append("\":", 2) || + append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1) || append_simple(str, v_to, js->end() - v_to)) goto js_error; /* Out of memory. */ } @@ -3620,7 +4025,8 @@ String *Item_func_json_insert::val_str(String *str) } if (append_simple(str, js->ptr(), v_to - js->ptr()) || - append_json_value(str, args[n_arg+1], &tmp_val) || + append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), + (int) n_arg + 1) || append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) goto js_error; /* Out of memory. */ continue_point: @@ -3683,7 +4089,6 @@ String *Item_func_json_remove::val_str(String *str) JSON_DO_PAUSE_EXECUTION(thd, 0.0002); str->set_charset(js->charset()); - json_string_set_cs(&key_name, js->charset()); for (n_arg=1, n_path=0; n_arg < arg_count; n_arg++, n_path++) { @@ -3786,6 +4191,13 @@ String *Item_func_json_remove::val_str(String *str) if (je.value_type != JSON_VALUE_OBJECT) continue; + /* + Set here and not once before the loop: each path is written in + its own character set, and the key has to be read in the one the + path it came from was written in. + */ + json_string_set_cs(&key_name, c_path->p.s.cs); + while (json_scan_next(&je) == 0 && je.state != JST_OBJ_END) { switch (je.state) @@ -4461,11 +4873,55 @@ bool Item_func_json_arrayagg::fix_fields(THD *thd, Item **ref) } +/* + func_name() ends with '(' so that "Row %lu was cut by %s)" reads + properly. A diagnostic that names the function on its own wants the + name without it. +*/ +static const char json_arrayagg_name[]= "json_arrayagg"; + + +/* + Cleared where a group begins rather than where its result is asked + for. Without ORDER BY or DISTINCT the rows are written out as they + arrive, long before anything asks for the result, so clearing the mark + at that point would throw away what the rows had already reported. +*/ +void Item_func_json_arrayagg::clear() +{ + m_bad_element= false; + Item_func_group_concat::clear(); +} + + +/* + Returning nothing for a row is not the same as the row not being + there. The separator between one element and the next is written + before the element is asked for (dump_leaf_key(), item_sum.cc), and it + stays written whether an element follows it or not, so a row declined + here leaves a separator with nothing on one side of it. + + That is what has always been returned for a value holding a + character no document can carry, and it still is: the note is the new + part, the shape is not. A row lost to a buffer that would not grow + is a different matter - nothing about the data asked for it, and the + statement is over anyway - so the group is marked and refused whole + when it is asked for. + + A row holding something that does not parse as JSON reaches neither + of these. It is written out as it stands, the same as it always was. +*/ String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) { + int rc; + m_tmp_json.length(0); - if (append_json_value(&m_tmp_json, i, tmp)) + if ((rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0))) + { + if (rc == JSON_APPEND_OOM) + m_bad_element= true; return NULL; + } return &m_tmp_json; } @@ -4473,10 +4929,17 @@ String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) String *Item_func_json_arrayagg::get_str_from_field(Item *i,Field *f, String *tmp, const uchar *key, size_t offset) { + int rc; + m_tmp_json.length(0); - if (append_json_value_from_field(&m_tmp_json, i, f, key, offset, tmp)) + if ((rc= append_json_value_from_field(&m_tmp_json, i, f, key, offset, tmp, + 1, json_arrayagg_name, 0))) + { + if (rc == JSON_APPEND_OOM) + m_bad_element= true; return NULL; + } return &m_tmp_json; @@ -4511,18 +4974,42 @@ String* Item_func_json_arrayagg::val_str(String *str) if ((str= Item_func_group_concat::val_str(str))) { String s; - s.append('['); + + /* + A row that could not be written out left the elements it sits + between joined by nothing but their separator, so there is no + array here to return. Reachable only through a failure to + allocate, which has already raised an error of its own. + */ + if (m_bad_element) + { + null_value= 1; + return NULL; + } + + /* + The brackets are put on last, so a buffer that will not take them + leaves behind something that reads as a value of its own rather + than as the array it is meant to be. + */ + if (s.append('[')) + goto bad_result; + s.swap(*str); - str->append(s); - str->append(']'); + if (str->append(s) || str->append(']')) + goto bad_result; } return str; + +bad_result: + null_value= 1; + return NULL; } Item_func_json_objectagg:: Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item) - :Item_sum(thd, item) + :Item_sum(thd, item), m_bad_pair(false) { quick_group= FALSE; result.set_charset(collation.collation); @@ -4594,6 +5081,7 @@ void Item_func_json_objectagg::clear() { result.length(1); null_value= 1; + m_bad_pair= false; } @@ -4601,23 +5089,50 @@ bool Item_func_json_objectagg::add() { StringBuffer buf; String *key; + int rc; key= args[0]->val_str(&buf); if (args[0]->is_null()) return 0; null_value= 0; - if (result.length() > 1) - result.append(STRING_WITH_LEN(", ")); + if (result.length() > 1 && result.append(STRING_WITH_LEN(", "))) + goto bad_pair; + + if (result.append('"')) + goto bad_pair; + + /* + A key with a character no document can carry has always gone in as + however much of it could be written, and the pair has always been + finished around it. Only a buffer that would not grow gives up. + */ + if ((rc= st_append_escaped(&result, key))) + { + if (rc == JSON_APPEND_OOM) + goto bad_pair; + report_bad_chr_note(func_name(), 1); + } - result.append('"'); - st_append_escaped(&result,key); - result.append(STRING_WITH_LEN("\":")); + if (result.append(STRING_WITH_LEN("\":"))) + goto bad_pair; buf.length(0); - append_json_value(&result, args[1], &buf); + if (append_json_value(&result, args[1], &buf, 1, func_name(), 1) == + JSON_APPEND_OOM) + goto bad_pair; return 0; + +bad_pair: + /* + The pair has been written out in part, and the buffer will not take + the rest of it. Whatever else the group goes on to add, the object + it is building is already broken, so it is refused as a whole + rather than returned half written. + */ + m_bad_pair= true; + return 0; } @@ -4627,7 +5142,12 @@ String* Item_func_json_objectagg::val_str(String* str) if (null_value) return 0; - result.append('}'); + if (m_bad_pair || result.append('}')) + { + null_value= 1; + return 0; + } + return &result; } diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index cfa6a15077fe3..00445285aa86c 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -48,7 +48,23 @@ void report_json_error_ex(const char *js, json_engine_t *je, const char *fname, int n_param, Sql_condition::enum_warning_level lv); bool check_overlaps(json_engine_t *js, json_engine_t *value, bool compare_whole); -int st_append_escaped(String *s, const String *a); + +/* + What an append of escaped text to a document came to. The two ways + of failing are kept apart because they are not answered alike: a + buffer that would not grow has already raised an error of its own and + the statement is over, while a character that cannot be written into + a document is an ordinary property of the data, which callers have + always carried on past. +*/ +enum json_append_result +{ + JSON_APPEND_OK= 0, + JSON_APPEND_OOM= 1, + JSON_APPEND_BAD_CHR= 2 +}; + +json_append_result st_append_escaped(String *s, const String *a); int json_find_overlap_with_object(json_engine_t *js, json_engine_t *value, bool compare_whole); @@ -269,6 +285,7 @@ class Item_func_json_unquote: public Item_str_func protected: String tmp_s; String *read_json(json_engine_t *je); + String *return_as_is(String *str, String *js); public: Item_func_json_unquote(THD *thd, Item *s): Item_str_func(thd, s) {} LEX_CSTRING func_name_cstring() const override @@ -758,6 +775,15 @@ class Item_func_json_arrayagg : public Item_func_group_concat const uchar *key, size_t offset) override; void cut_max_length(String *result, uint old_length, uint max_length) const override; + /* + A row of this group could not be written out at all, the buffer + having failed to grow. Neither a row whose value does not parse as + JSON nor one holding a character no document can carry is this: + the first is written out as it stands and the second is dropped + where it always was, both with a note. Reset for each group by + clear(). + */ + bool m_bad_element; public: String m_tmp_json; /* Used in get_str_from_*.. */ Item_func_json_arrayagg(THD *thd, Name_resolution_context *context_arg, @@ -765,11 +791,22 @@ class Item_func_json_arrayagg : public Item_func_group_concat const SQL_I_List &is_order, String *is_separator, bool limit_clause, Item *row_limit, Item *offset_limit): Item_func_group_concat(thd, context_arg, is_distinct, is_select, is_order, - is_separator, limit_clause, row_limit, offset_limit) + is_separator, limit_clause, row_limit, offset_limit), + m_bad_element(false) { } + /* + A copy is not fixed again, so anything fix_fields() settled has to + be settled here too. The buffer's character set is one of those: + left at its default the copy would read the values it is given as + bytes rather than as the characters they are, and say they were + not JSON. + */ Item_func_json_arrayagg(THD *thd, Item_func_json_arrayagg *item) : - Item_func_group_concat(thd, item) {} + Item_func_group_concat(thd, item), m_bad_element(false) + { + m_tmp_json.set_charset(collation.collation); + } const Type_handler *type_handler() const override { return Type_handler_json_common::json_type_handler_sum(this); @@ -783,6 +820,7 @@ class Item_func_json_arrayagg : public Item_func_group_concat bool fix_fields(THD *thd, Item **ref) override; enum Sumfunctype sum_func() const override { return JSON_ARRAYAGG_FUNC; } + void clear() override; String* val_str(String *str) override; Item *copy_or_same(THD* thd) override; @@ -796,9 +834,15 @@ class Item_func_json_arrayagg : public Item_func_group_concat class Item_func_json_objectagg : public Item_sum { String result; + /* + A pair of this group could not be written out in full, the buffer + having failed to grow part way through it. Reset for each group by + clear(). + */ + bool m_bad_pair; public: Item_func_json_objectagg(THD *thd, Item *key, Item *value) : - Item_sum(thd, key, value) + Item_sum(thd, key, value), m_bad_pair(false) { quick_group= FALSE; result.append('{'); From a45fa225ff8b5743b515c36acaf33acf55a90b34 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 3 Aug 2026 02:05:31 -0400 Subject: [PATCH 03/30] MDEV-40605 Write both ends of an aggregated object in one character set `JSON_OBJECTAGG` wrote its opening brace in the constructor, which runs while the expression is still being parsed - before `fix_fields()` has resolved the character set the result will be in. The brace therefore went down as a single byte. The closing brace is written when the result is asked for, by which time the character set is settled, so that one went down at full width. In `ucs2`, `utf16` and `utf32` the two ends did not match and everything between them sat a byte out of step. `JSON_VALID()` answered 0, and converting the result to `utf8mb4` raised Warning 1300 Invalid ucs2 character string: '}' which is the server refusing to read a document it had just written. Two further places measured that brace rather than writing it, and both are right only where a character is one byte: `clear()` kept byte 0 with `result.length(1)`, and `add()` decided whether a pair needed a separator in front of it by asking whether `result.length() > 1`. The brace is now written in `clear()`, once per group, where the result already carries the character set it will be read in, so `String::append()` puts it down at the width that character set asks for and the two ends agree. `add()` asks `null_value` whether a pair has been written already instead of measuring the buffer, and `cleanup()` empties the buffer rather than keeping one byte of what may be a wider character. A result computed where one byte is one character does not move, `latin1` and `utf8mb4` included. New test `func_json_agg_charset`, covering both aggregates because they are asked the same question and answer it differently. Section 7 is this change, recorded before it: the three wide character sets; one pair, where no separator is ever written, and several, where one is; a group each, so that the opening brace is written more than once; and a `WITH ROLLUP` copy, which is made without fixing the item again and so has to settle its own character set. Sections 1 to 6 are the sibling aggregate, which is in the same difficulty for an unrelated reason and is recorded here as it stands. `swe7` is the vehicle for them: having no code point for the JSON punctuation, it keeps every recorded byte ASCII while still showing which results were converted on the way out and which were not. A later commit puts that half right. --- mysql-test/main/func_json_agg_charset.result | 280 +++++++++++++++++++ mysql-test/main/func_json_agg_charset.test | 187 +++++++++++++ sql/item_jsonfunc.cc | 28 +- sql/item_jsonfunc.h | 8 +- 4 files changed, 497 insertions(+), 6 deletions(-) create mode 100644 mysql-test/main/func_json_agg_charset.result create mode 100644 mysql-test/main/func_json_agg_charset.test diff --git a/mysql-test/main/func_json_agg_charset.result b/mysql-test/main/func_json_agg_charset.result new file mode 100644 index 0000000000000..a5fb8019f4c1d --- /dev/null +++ b/mysql-test/main/func_json_agg_charset.result @@ -0,0 +1,280 @@ +# +# How the JSON aggregates deliver their result to the client. +# +# A result computed in one character set and asked for in another is +# converted on the way out. Whether that conversion happens at all is +# decided by the character set the returned value carries, which need +# not be the one the function says it has. +# +# swe7 is the vehicle throughout. It has no code point for the +# brackets and braces a JSON document is built from, so a result that +# was converted arrives with those replaced by '?', and one passed on +# unconverted arrives intact. The difference is therefore visible +# without a single non-ASCII byte in this file. A client asking for +# latin1 is told the same two stories, but the second one reaches it as +# utf8mb4 bytes under a latin1 label, which is not printable and not +# recoverable. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES (1, 'aa'), (1, 'bb'), (2, 'cc'), (2, 'dd'); +# +# 1. An aggregate delivered next to the functions that are not. +# +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('aa','bb') AS v; +v +?"aa", "bb"? +SELECT JSON_OBJECT('aa','bb') AS v; +v +?"aa": "bb"? +SELECT JSON_SET('{}','$.a','b') AS v; +v +?"a": "b"? +SELECT JSON_KEYS('{"aa":1}') AS v; +v +?"aa"? +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +["aa","bb","cc","dd"] +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +v +?"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"? +# one group per row of output, so the delivery is repeated +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ["aa","bb"] +2 ["cc","dd"] +SELECT g, JSON_OBJECTAGG(a,a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ?"aa":"aa", "bb":"bb"? +2 ?"cc":"cc", "dd":"dd"? +SET SESSION character_set_results = DEFAULT; +# +# 2. The value itself, which the delivery does not touch. +# +# Read inside the server rather than sent to the client, the aggregate +# is the same either way. HEX() answers in ASCII, so the bytes can be +# shown under both settings and compared. +# +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, +JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +valid len +1 4 +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1; +v +[["aa","bb","cc","dd"]] +SET SESSION character_set_results = swe7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, +JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +valid len +1 4 +SET SESSION character_set_results = DEFAULT; +# +# 3. What the function says it returns. +# +# CHARSET() and COLLATION() report what the expression was resolved to, +# not what the returned value ended up labelled with, so neither of +# them can see a difference in delivery. A probe built from these two +# alone, or from ASCII values under a character set that can hold them, +# observes nothing at all. +# +SELECT CHARSET(JSON_ARRAYAGG(a)) AS cs, COLLATION(JSON_ARRAYAGG(a)) AS co +FROM t1; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_OBJECTAGG(a,a)) AS cs, COLLATION(JSON_OBJECTAGG(a,a)) AS co +FROM t1; +cs co +utf8mb4 utf8mb4_general_ci +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +["aa","bb","cc","dd"] +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +v +{"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"} +SET SESSION character_set_results = DEFAULT; +# +# 4. Stored rather than delivered. +# +# A value on its way into a column is converted from the character set +# the expression was resolved to, which is a separate question from the +# one above. +# +CREATE TABLE t2 (v VARCHAR(50) CHARACTER SET latin1); +INSERT INTO t2 SELECT JSON_ARRAYAGG(a) FROM t1; +INSERT INTO t2 SELECT JSON_OBJECTAGG(a,a) FROM t1; +SELECT v, HEX(v) AS h FROM t2 ORDER BY v; +v h +["aa","bb","cc","dd"] 5B226161222C226262222C226363222C226464225D +{"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"} 7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +SET SESSION character_set_results = swe7; +SELECT v FROM t2 ORDER BY v; +v +?"aa","bb","cc","dd"? +?"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"? +SET SESSION character_set_results = DEFAULT; +# +# 5. The other ways into the same result. +# +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1; +v +["aa","bb","cc","dd"] +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1; +v +["dd","cc","bb","aa"] +# a group of one writes no separator, so it exercises less than it looks +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'aa'; +v +["aa"] +# no rows at all: NULL, which is never delivered as a string +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +v +NULL +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1 WHERE g = 99; +v +NULL +SET SESSION character_set_results = DEFAULT; +# +# 6. A result computed in a character set that ASCII is not a part of. +# +# Every character of such a result is two or four bytes wide, the +# brackets the aggregate puts around the elements included. Written +# as single bytes those two would leave everything between them one +# byte out of step, and the result would read as an entirely +# different set of characters rather than as a document. +# +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1; +h +5B0022006100610022002C0022006200620022002C0022006300630022002C00220064006400225D +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS valid FROM t1; +valid +0 +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 2 +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS h FROM t1; +h +5B0022006100610022002C0022006200620022002C0022006300630022002C00220064006400225D +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS valid FROM t1; +valid +0 +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 2 +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS h FROM t1; +h +5B000000220000006100000061000000220000002C000000220000006200000062000000220000002C000000220000006300000063000000220000002C000000220000006400000064000000225D +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS valid FROM t1; +valid +0 +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 2 +# a group of one, which is bracket, element, bracket and nothing else +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1 WHERE a = 'aa'; +h +5B00220061006100225D +# the same values where one byte is one character, which is the case +# the brackets were always written for +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING latin1))) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf8mb4))) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +# +# 7. The sibling aggregate, in the same character sets. +# +# It reaches the same question by another route. Its opening brace +# is not written next to its closing one: the closing brace goes on +# when the result is asked for, and the opening brace has to be there +# before the first pair is added. Where that brace is written decides +# whether it can be written in the character set the result is in, and +# whether the two ends of the object match. +# +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h +FROM t1; +h +007B0022006100610022003A0022006100610022002C00200022006200620022003A0022006200620022002C00200022006300630022003A0022006300630022002C00200022006400640022003A0022006400640022007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid FROM t1; +valid +1 +SELECT CONVERT(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2)) USING utf8mb4) AS v +FROM t1; +v +{"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"} +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf16), CONVERT(a USING utf16))) AS h +FROM t1; +h +007B0022006100610022003A0022006100610022002C00200022006200620022003A0022006200620022002C00200022006300630022003A0022006300630022002C00200022006400640022003A0022006400640022007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf16), +CONVERT(a USING utf16))) AS valid FROM t1; +valid +1 +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf32), CONVERT(a USING utf32))) AS h +FROM t1; +h +0000007B000000220000006100000061000000220000003A000000220000006100000061000000220000002C00000020000000220000006200000062000000220000003A000000220000006200000062000000220000002C00000020000000220000006300000063000000220000003A000000220000006300000063000000220000002C00000020000000220000006400000064000000220000003A000000220000006400000064000000220000007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf32), +CONVERT(a USING utf32))) AS valid FROM t1; +valid +1 +# one pair, where no separator is written between pairs at all +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h +FROM t1 WHERE a = 'aa'; +h +007B0022006100610022003A0022006100610022007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid +FROM t1 WHERE a = 'aa'; +valid +1 +# a group each, so the opening brace is written more than once +SELECT g, HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS h +FROM t1 GROUP BY g ORDER BY g; +g h +1 007B0022006100610022003A0022006100610022002C00200022006200620022003A0022006200620022007D +2 007B0022006300630022003A0022006300630022002C00200022006400640022003A0022006400640022007D +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid +FROM t1 GROUP BY g ORDER BY g; +g valid +1 1 +2 1 +# a copy of the aggregate, which is made without fixing it again +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid +FROM t1 GROUP BY g WITH ROLLUP; +g valid +1 1 +2 1 +NULL 1 +# no rows, which is a NULL and never an empty object +SELECT JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2)) AS v +FROM t1 WHERE g = 99; +v +NULL +# and where one byte is one character, which must not move +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING latin1), +CONVERT(a USING latin1))) AS h FROM t1; +h +7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf8mb4), +CONVERT(a USING utf8mb4))) AS h FROM t1; +h +7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +SELECT g, HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1 GROUP BY g ORDER BY g; +g h +1 7B226161223A226161222C20226262223A226262227D +2 7B226363223A226363222C20226464223A226464227D +DROP TABLE t1, t2; +SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_agg_charset.test b/mysql-test/main/func_json_agg_charset.test new file mode 100644 index 0000000000000..ff8aa271fbe8b --- /dev/null +++ b/mysql-test/main/func_json_agg_charset.test @@ -0,0 +1,187 @@ +--source include/have_utf8mb4.inc +--source include/have_ucs2.inc +--source include/have_utf16.inc +--source include/have_utf32.inc + +--echo # +--echo # How the JSON aggregates hand their result to the client. +--echo # +--echo # A result computed in one character set and asked for in another is +--echo # converted on the way out. Whether that conversion happens at all is +--echo # decided by the character set the returned value carries, which need +--echo # not be the one the function says it has. +--echo # +--echo # swe7 is the vehicle throughout. It has no code point for the +--echo # brackets and braces a JSON document is built from, so a result that +--echo # was converted arrives with those replaced by '?', and one passed on +--echo # unconverted arrives intact. The difference is therefore visible +--echo # without a single non-ASCII byte in this file. A client asking for +--echo # latin1 is told the same two stories, but the second one reaches it as +--echo # utf8mb4 bytes under a latin1 label, which is not printable and not +--echo # recoverable. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES (1, 'aa'), (1, 'bb'), (2, 'cc'), (2, 'dd'); + +--echo # +--echo # 1. An aggregate delivered next to the functions that are not. +--echo # + +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('aa','bb') AS v; +SELECT JSON_OBJECT('aa','bb') AS v; +SELECT JSON_SET('{}','$.a','b') AS v; +SELECT JSON_KEYS('{"aa":1}') AS v; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +--echo # one group per row of output, so the delivery is repeated +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +SELECT g, JSON_OBJECTAGG(a,a) AS v FROM t1 GROUP BY g ORDER BY g; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 2. The value itself, which the delivery does not touch. +--echo # +--echo # Read inside the server rather than sent to the client, the aggregate +--echo # is the same either way. HEX() answers in ASCII, so the bytes can be +--echo # shown under both settings and compared. +--echo # + +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, + JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1; +SET SESSION character_set_results = swe7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, + JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 3. What the function says it returns. +--echo # +--echo # CHARSET() and COLLATION() report what the expression was resolved to, +--echo # not what the returned value ended up labelled with, so neither of +--echo # them can see a difference in delivery. A probe built from these two +--echo # alone, or from ASCII values under a character set that can hold them, +--echo # observes nothing at all. +--echo # + +SELECT CHARSET(JSON_ARRAYAGG(a)) AS cs, COLLATION(JSON_ARRAYAGG(a)) AS co + FROM t1; +SELECT CHARSET(JSON_OBJECTAGG(a,a)) AS cs, COLLATION(JSON_OBJECTAGG(a,a)) AS co + FROM t1; +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 4. Stored rather than delivered. +--echo # +--echo # A value on its way into a column is converted from the character set +--echo # the expression was resolved to, which is a separate question from the +--echo # one above. +--echo # + +CREATE TABLE t2 (v VARCHAR(50) CHARACTER SET latin1); +INSERT INTO t2 SELECT JSON_ARRAYAGG(a) FROM t1; +INSERT INTO t2 SELECT JSON_OBJECTAGG(a,a) FROM t1; +SELECT v, HEX(v) AS h FROM t2 ORDER BY v; +SET SESSION character_set_results = swe7; +SELECT v FROM t2 ORDER BY v; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 5. The other ways into the same result. +--echo # + +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1; +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1; +--echo # a group of one writes no separator, so it exercises less than it looks +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'aa'; +--echo # no rows at all: NULL, which is never delivered as a string +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1 WHERE g = 99; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 6. A result computed in a character set that ASCII is not a part of. +--echo # +--echo # Every character of such a result is two or four bytes wide, the +--echo # brackets the aggregate puts around the elements included. Written +--echo # as single bytes those two would leave everything between them one +--echo # byte out of step, and the result would read as an entirely +--echo # different set of characters rather than as a document. +--echo # + +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS valid FROM t1; +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS valid FROM t1; +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS valid FROM t1; +--echo # a group of one, which is bracket, element, bracket and nothing else +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1 WHERE a = 'aa'; +--echo # the same values where one byte is one character, which is the case +--echo # the brackets were always written for +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING latin1))) AS h FROM t1; +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf8mb4))) AS h FROM t1; +--echo # +--echo # 7. The sibling aggregate, in the same character sets. +--echo # +--echo # It reaches the same question by another route. Its opening brace +--echo # is not written next to its closing one: the closing brace goes on +--echo # when the result is asked for, and the opening brace has to be there +--echo # before the first pair is added. Where that brace is written decides +--echo # whether it can be written in the character set the result is in, and +--echo # whether the two ends of the object match. +--echo # + +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h + FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid FROM t1; +SELECT CONVERT(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2)) USING utf8mb4) AS v + FROM t1; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf16), CONVERT(a USING utf16))) AS h + FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf16), + CONVERT(a USING utf16))) AS valid FROM t1; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf32), CONVERT(a USING utf32))) AS h + FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf32), + CONVERT(a USING utf32))) AS valid FROM t1; +--echo # one pair, where no separator is written between pairs at all +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h + FROM t1 WHERE a = 'aa'; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid + FROM t1 WHERE a = 'aa'; +--echo # a group each, so the opening brace is written more than once +SELECT g, HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS h + FROM t1 GROUP BY g ORDER BY g; +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid + FROM t1 GROUP BY g ORDER BY g; +--echo # a copy of the aggregate, which is made without fixing it again +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid + FROM t1 GROUP BY g WITH ROLLUP; +--echo # no rows, which is a NULL and never an empty object +SELECT JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2)) AS v + FROM t1 WHERE g = 99; +--echo # and where one byte is one character, which must not move +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING latin1), + CONVERT(a USING latin1))) AS h FROM t1; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf8mb4), + CONVERT(a USING utf8mb4))) AS h FROM t1; +SELECT g, HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1 GROUP BY g ORDER BY g; + +DROP TABLE t1, t2; +SET NAMES utf8mb4; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 47c435536956a..a37aabbe1c43c 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -5013,7 +5013,6 @@ Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item) { quick_group= FALSE; result.set_charset(collation.collation); - result.append('{'); } @@ -5066,7 +5065,7 @@ void Item_func_json_objectagg::cleanup() DBUG_ENTER("Item_func_json_objectagg::cleanup"); Item_sum::cleanup(); - result.length(1); + result.length(0); DBUG_VOID_RETURN; } @@ -5077,11 +5076,24 @@ Item *Item_func_json_objectagg::copy_or_same(THD* thd) } +/* + The opening brace is written here rather than where the object is + built up, because here the result has the character set it will be + read in and the brace can be written at the width that asks for. The + closing one is written when the result is asked for, by which time the + same is true of it, so the two ends match. +*/ void Item_func_json_objectagg::clear() { - result.length(1); + result.length(0); null_value= 1; + /* + The brace is a write into the object like any other, and a group + that lost one is refused whole rather than returned short. + */ m_bad_pair= false; + if (result.append('{')) + m_bad_pair= true; } @@ -5095,9 +5107,15 @@ bool Item_func_json_objectagg::add() if (args[0]->is_null()) return 0; - null_value= 0; - if (result.length() > 1 && result.append(STRING_WITH_LEN(", "))) + /* + Whether this pair needs a separator in front of it is whether a pair + has been written already, which is what null_value says until this + row changes it. Comparing the buffer against the opening brace would + be comparing it against a width that is not always one byte. + */ + if (!null_value && result.append(STRING_WITH_LEN(", "))) goto bad_pair; + null_value= 0; if (result.append('"')) goto bad_pair; diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 00445285aa86c..c84471efa08ff 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -841,11 +841,17 @@ class Item_func_json_objectagg : public Item_sum */ bool m_bad_pair; public: + /* + The opening brace is not written here. This runs while the + expression is being parsed, before there is a character set to write + it in, and a brace put down now would be one byte wide however wide a + character of the result turns out to be. clear() writes it instead, + once per group and once the width is known. + */ Item_func_json_objectagg(THD *thd, Item *key, Item *value) : Item_sum(thd, key, value), m_bad_pair(false) { quick_group= FALSE; - result.append('{'); } Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item); From 9b030ec2bf0aa593cda9b728bd210cbbcd1c1470 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 3 Aug 2026 02:24:34 -0400 Subject: [PATCH 04/30] MDEV-40590 JSON_OVERLAPS crashes on an array nested past the depth limit The scanner refuses to open a structure nested deeper than it can keep track of, and records that it has, but it moved its nesting counter before working out whether there was room. A refusal therefore left the counter one past the end of the stack it indexes. `stack_p` is declared immediately after `stack`, and `stack` is exactly `JSON_DEPTH_LIMIT` long, so `j->stack[j->stack_p]` at that point reads the counter itself and yields `JSON_DEPTH_LIMIT` as though it were a scanner state. `json_scan_next()` then indexes a table of `NR_JSON_STATES` rows with it and calls what it finds there: json_actions[32][t_next](j) /* the table has 10 rows */ Getting that far needs a caller that keeps scanning after being told the document is too deep, and `json_compare_arr_and_obj()` is one: `json_skip_level()`'s result is not looked at, and the loop around it asks only whether the next scan step succeeded. So SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '{"a":1}'); killed the server, in either argument position and whatever the other argument held. The counter is now moved only once the room for it is known to be there, so a refusal leaves it where it was and every read of `stack[stack_p]` stays inside the stack. Making that a property of the scanner rather than of its callers matters here: six further places in this family also skip levels without looking at the result, and each of them was one ignored refusal away from the same ending. **A refusal is kept, and so is the place where it happened.** That is the other half of making it the scanner's property, and three things rest on it. Every error is reported by subtracting the scanner's read pointer from the start of the document, and `json_scan_next()` moves that pointer before it looks at anything, so each further step taken after a refusal moves the place the refusal will be said to have happened. For one document and four different second arguments: SELECT JSON_OVERLAPS(@d,'{"a":1}'), JSON_OVERLAPS(@d,'"s"'), JSON_OVERLAPS(@d,'[[1]]'), JSON_OVERLAPS(@d,'[1,2]'); at position 34 at position 34 at position 34 at position 33 while `JSON_DEPTH(@d)` says 32 for the same bytes, and so do `JSON_CONTAINS`, `JSON_EQUALS` and `JSON_TYPE` four lines away in the same recorded file. The reading is now taken when the refusal happens rather than when it is reported. `json_error()` sets the code and the position together, and what reports a refusal reads the recorded position instead of the live pointer, so where the scanner afterwards comes to rest no longer decides what the caller is told. Every site that refused by assigning to `error` calls it instead, which is why the two cannot come apart: there is nowhere left to set one without the other. It keeps the first refusal for the same reason it saves the position - a scan that carries on after failing reaches handlers that refuse it again on their own account, and what the caller is told has to be what stopped the scan and not whatever it ran into afterwards. Stopping the engine outright would answer this too, and was the first way it was done here, but it costs a load, a test and a branch on every token of every document the server reads - paid by the documents that never fail, which is nearly all of them. Recording the position where it is set puts that cost only on the refusals. Two places carry a refusal from one engine to another and they are not the same case. `json_skip_array_and_count()` scans a copy of the engine over the same string, so where that copy stopped is a position in the caller's string too and travels with the code. The normalizing comparisons run over a converted copy of the argument, so where the normalizer stopped is a position in that copy and means nothing outside it; only the code is carried over. `json_normalize_engine()` already mapped the read pointer back from the converted copy for exactly this reporting, and now maps the saved position, which is what is read - the copy is freed on the next line, so a position left pointing into it points into nothing. The geometry reader puts codes of its own into `error` and still assigns them: it reports those by name, never asks where, and means to replace whatever the scanner said. Both comparison walks remember where an engine stood and put it back to try the next candidate, and the snapshot predates the failure, so a restore rolled the error away with it. `json_resume_scan()` states the rule once and the five restore sites go through it; without it `JSON_OVERLAPS('[1,2]', <32 deep>)` answers 0 and says nothing at all. `compare_nested_object()` measured its two fragments from where the skip stopped and normalized them. After a refusal that is a piece of a document, so normalizing found it unterminated and said so, and SELECT JSON_OVERLAPS(CONCAT('[',REPEAT('{"a":',31),'1', REPEAT('}',31),']'), '{"a":1}'); drew `4037 Unexpected end of JSON text` for a document that is complete and balanced and whose only fault is depth - beside the pure-object case one screen above, which says 4040 and where. The refusal is honoured instead, and the engine's own reason survives to the caller: 4040 at position 152, which is what `JSON_DEPTH` says of the same bytes. The two places that ask whether the walk has already been refused, both of them closing a key-name loop, say so with `unlikely`. A document that reads cleanly answers no every time, and marking it keeps the refusal out of the straight line through the scanner. Neither is on the per-token road; nothing asks that question once per token. `JSON_OVERLAPS` now answers 0 with warning 4040 for an over-deep argument, which is what it has always answered when the same depth is reached through objects instead of arrays - the crash was particular to arrays. `Item_func_json_overlaps::val_bool()` was already reporting the engine's error once the comparison returned; it simply never got to. Present since `JSON_OVERLAPS` was added, so 10.9 onwards. **Not one answer moves; what moves is where a complaint is placed, and whether it is made at all.** Against a released server: JSON_OVERLAPS('{"b":9}', '{"a":1 "b":2}') 9 -> 8 JSON_OVERLAPS('{"z":0}', '{"a":1, "b":2 "c":3}') 16 -> 15 JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1 "q":2}}') 15 -> 13 JSON_OVERLAPS('[{"z":9},{"b":2 "c":3}]', '[{"a":1}]') 18 -> 17 JSON_OVERLAPS('[9]', '[1,2,3 4]') silent -> 8 JSON_OVERLAPS('[{"a":1}]', '[{"z":9},{"b":2 "c":3}]') silent -> 17 The new positions are the right ones, and the tests say so by something other than themselves: `JSON_VALID` and `JSON_EXTRACT` read the same bytes with no comparison around them, and they give 8, 15, 13, 17 and 8 for these documents. Both readings are recorded side by side, so a later change that moves one without the other fails there. Nothing in the suite stood on any of this. The only `json_overlaps` syntax-error warning recorded anywhere is a whole document rejected at position 1, which never reaches the walk at all. The two silent cases could have come back in silence and the four positions could have drifted with nothing to say so. Two new tests. `func_json_overlaps_depth`: both argument positions, the other argument of each kind, both arguments too deep, the same depth reached through objects and through a mix of the two, one container inside the limit where the comparison does run, what the neighbouring comparisons answer on the same input, and a table of such rows so that one engine is reused across evaluations; its section 6 puts one first argument against four second arguments in a single statement, so that four positions stand in one block and any drift between them is visible, with `JSON_DEPTH` of the same value beneath it, and the mixed chain in section 2 gains the same anchor. `func_json_overlaps_diag` takes the break in either argument, at the top level and below it, on the road where every candidate is tried and put back, and out of a table in both argument orders so no one statement settles another; whole documents and a document refused before the walk begins stand beside them, unmoved. Three recorded lines move outside the new tests, all of them a diagnostic position and none of them an answer. Each moves onto the number the same file already records for the same bytes: `func_json_columns`, 34 to 32; and `func_json_depth`, `JSON_KEYS` 160 to 156 where `JSON_DEPTH(@o32)` in that file says 156, and `JSON_OVERLAPS` 33 to 32 where `JSON_DEPTH(@d32)` says 32. The recorded baseline `func_json_columns` no longer holds its over-deep row back from the `JSON_OVERLAPS` statement. The comment there said the row was excluded until this was fixed; the row now answers like every other function in that statement. --- include/json_lib.h | 40 +++- mysql-test/main/func_json_columns.result | 8 +- mysql-test/main/func_json_columns.test | 6 +- mysql-test/main/func_json_depth.result | 4 +- .../main/func_json_overlaps_depth.result | 198 ++++++++++++++++++ mysql-test/main/func_json_overlaps_depth.test | 106 ++++++++++ .../main/func_json_overlaps_diag.result | 155 ++++++++++++++ mysql-test/main/func_json_overlaps_diag.test | 92 ++++++++ sql/item_jsonfunc.cc | 76 +++++-- strings/json_lib.c | 131 +++++++----- strings/json_normalize.c | 10 + 11 files changed, 745 insertions(+), 81 deletions(-) create mode 100644 mysql-test/main/func_json_overlaps_depth.result create mode 100644 mysql-test/main/func_json_overlaps_depth.test create mode 100644 mysql-test/main/func_json_overlaps_diag.result create mode 100644 mysql-test/main/func_json_overlaps_diag.test diff --git a/include/json_lib.h b/include/json_lib.h index d71fcd243be76..53d6548694853 100644 --- a/include/json_lib.h +++ b/include/json_lib.h @@ -12,7 +12,7 @@ extern "C" { /* When error happens, the c_next of the JSON engine contains the - character that caused the error, and the c_str is the position + character that caused the error, and the error_pos is the position in string where the error occurs. */ enum json_errors { @@ -41,6 +41,7 @@ typedef struct st_json_string_t my_wc_t c_next; /* UNICODE of the last read character */ int c_next_len; /* character lenght of the last read character. */ int error; /* error code. */ + const uchar *error_pos; /* Where in the string that error happened. */ CHARSET_INFO *cs; /* Character set of the JSON string. */ @@ -52,6 +53,43 @@ typedef struct st_json_string_t void json_string_set_cs(json_string_t *s, CHARSET_INFO *i_cs); void json_string_set_str(json_string_t *s, const uchar *str, const uchar *end); + +/* + Refuse the string, and remember where the refusing happened. + + The position is taken here rather than read off c_str at the point + the refusal is reported, because the scanner does not come to rest + where it failed. A caller is free to go on asking for the next token + after being told no - not all of them are obliged to stop, and see + where the depth refusals are written for why they cannot be made to - + and c_str then walks past the place the refusal happened, so a reading + taken later describes somewhere else. + + The first refusal is the one kept, for the same reason: a scan that + carries on after failing reaches handlers that refuse it again on + their own account, and what the caller is told has to be what stopped + the scan rather than whatever it ran into afterwards. + + Returns the code it was given, so the sites written as + `return s->error= JE_SYN` keep their shape. A code of zero refuses + nothing, which is what the sites that pass a result along need. + + Everything that a position is reported for is refused through here, + so error and error_pos are set together and mean each other. The + geometry reader puts codes of its own into error without coming here; + it reports those by name and never asks where, and it means to + replace whatever the scanner said, which is why it still assigns. +*/ +static inline int json_error(json_string_t *s, int code) +{ + if (code && !s->error) + { + s->error= code; + s->error_pos= s->c_str; + } + return code; +} + #define json_next_char(j) \ ((j)->c_next_len= (j)->wc((j)->cs, &(j)->c_next, (j)->c_str, (j)->str_end)) #define json_eos(j) ((j)->c_str >= (j)->str_end) diff --git a/mysql-test/main/func_json_columns.result b/mysql-test/main/func_json_columns.result index d114ff93f00fe..770491e1f88f4 100644 --- a/mysql-test/main/func_json_columns.result +++ b/mysql-test/main/func_json_columns.result @@ -429,18 +429,16 @@ Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_equals' Warning 4038 Syntax error in JSON text in argument 1 to function 'json_equals' at position 1 Warning 4038 Syntax error in JSON text in argument 1 to function 'json_equals' at position 9 Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_equals' at position 32 -# id 13, an array nested past the scanner limit, is left out here: -# JSON_OVERLAPS crashes the server on such an argument, so the row is -# excluded until that is fixed. Every other function above takes it. -SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v -FROM t1 WHERE id >= 10 AND id <> 13 ORDER BY id; +SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; id v 10 0 11 0 12 1 +13 0 Warnings: Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_overlaps' Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 # # 5. The column as an embedded argument. A JSON column is embedded as # a document, so whatever it holds is copied into the result as is. diff --git a/mysql-test/main/func_json_columns.test b/mysql-test/main/func_json_columns.test index 05dee4d531fe0..a8ca999345499 100644 --- a/mysql-test/main/func_json_columns.test +++ b/mysql-test/main/func_json_columns.test @@ -155,11 +155,7 @@ SELECT id, JSON_EXISTS(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; SELECT id, JSON_CONTAINS(j, '1', '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; SELECT id, JSON_SEARCH(j, 'one', '1') AS v FROM t1 WHERE id >= 10 ORDER BY id; SELECT id, JSON_EQUALS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; ---echo # id 13, an array nested past the scanner limit, is left out here: ---echo # JSON_OVERLAPS crashes the server on such an argument, so the row is ---echo # excluded until that is fixed. Every other function above takes it. -SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v - FROM t1 WHERE id >= 10 AND id <> 13 ORDER BY id; +SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; --echo # --echo # 5. The column as an embedded argument. A JSON column is embedded as diff --git a/mysql-test/main/func_json_depth.result b/mysql-test/main/func_json_depth.result index 82a90ead1280b..1bdfeda7a16a5 100644 --- a/mysql-test/main/func_json_depth.result +++ b/mysql-test/main/func_json_depth.result @@ -131,7 +131,7 @@ SELECT JSON_KEYS(@o32) AS over_limit; over_limit NULL Warnings: -Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_keys' at position 160 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_keys' at position 156 SELECT JSON_SET(@d32, '$[0]', 2) AS over_limit; over_limit NULL @@ -196,7 +196,7 @@ SELECT JSON_OVERLAPS(@d32, '[1]') AS over_limit; over_limit 0 Warnings: -Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 33 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 SELECT JSON_EQUALS(@d32, @d32) AS over_limit; over_limit NULL diff --git a/mysql-test/main/func_json_overlaps_depth.result b/mysql-test/main/func_json_overlaps_depth.result new file mode 100644 index 0000000000000..fae7164bbb195 --- /dev/null +++ b/mysql-test/main/func_json_overlaps_depth.result @@ -0,0 +1,198 @@ +# +# A document nested deeper than the parser will go, handed to the +# functions that compare one document against another. +# +# The parser stops at a fixed depth and records that it has. What it +# does with its nesting counter on the way out decides what a caller +# that keeps scanning afterwards will read, the counter being the +# index into the stack of states the scanner works from. +# +SET NAMES utf8mb4; +# +# 1. One container past the limit, in either argument. +# +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'{"a":1}') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_OVERLAPS('{"a":1}', +CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +# the other argument's kind does not matter +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'"s"') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'[[1]]') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'[1,2]') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +# both arguments past the limit +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +# +# 2. The same depth reached through objects rather than arrays. +# +SELECT JSON_OVERLAPS(CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32)), +'{"a":1}') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 156 +SELECT JSON_OVERLAPS('{"a":1}', +CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32))) AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 156 +# and mixed, an array holding an object chain. It is one container +# past the limit exactly as the two above are, and the complaint it +# draws has to say so: the document is complete and balanced, and +# too deep is the only thing wrong with it. +SELECT JSON_OVERLAPS(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']'), +'{"a":1}') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 152 +SELECT JSON_DEPTH(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']')) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 152 +# +# 3. One container inside the limit, which is a document and compares. +# +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), +'{"a":1}') AS v; +v +0 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), +CONCAT(REPEAT('[',31),'1',REPEAT(']',31))) AS v; +v +1 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), +'[1,2]') AS v; +v +0 +# +# 4. What the neighbouring functions answer on the same input. +# +SELECT JSON_CONTAINS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains' at position 32 +SELECT JSON_EQUALS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_equals' at position 32 +SELECT JSON_DEPTH(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +SELECT JSON_VALID(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_TYPE(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_type' at position 32 +# +# 5. Rows of them, so the engine is reused across evaluations. +# +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES +(1, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))), +(2, '[1,2]'), +(3, CONCAT(REPEAT('[',31),'1',REPEAT(']',31))), +(4, '{"a":1}'); +SELECT id, JSON_OVERLAPS(j, '[1,2]') AS v FROM t1 ORDER BY id; +id v +1 0 +2 1 +3 0 +4 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT id, JSON_OVERLAPS('[1,2]', j) AS v FROM t1 ORDER BY id; +id v +1 0 +2 1 +3 0 +4 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +SELECT id, JSON_OVERLAPS(j, j) AS v FROM t1 ORDER BY id; +id v +1 0 +2 1 +3 1 +4 1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +DROP TABLE t1; +# +# 6. Where the complaint says the document went too deep. +# +# The position is a property of the document alone. The second +# argument decides how much comparing is attempted before the depth +# is reached, and must not decide where the scanner is said to have +# stopped: all four below are the same first argument, so all four +# positions have to be the same one. +# +# It is also the same document the other functions are given in +# section 4, so it has to be the position they report for it. +# +SET @d = CONCAT(REPEAT('[',32),'1',REPEAT(']',32)); +SELECT JSON_OVERLAPS(@d,'{"a":1}') AS obj, JSON_OVERLAPS(@d,'"s"') AS scalar, +JSON_OVERLAPS(@d,'[[1]]') AS nested, JSON_OVERLAPS(@d,'[1,2]') AS flat; +obj scalar nested flat +0 0 0 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_DEPTH(@d) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +# and the other way round, where it is argument 2 that is too deep +SELECT JSON_OVERLAPS('{"a":1}',@d) AS obj, JSON_OVERLAPS('"s"',@d) AS scalar, +JSON_OVERLAPS('[[1]]',@d) AS nested, JSON_OVERLAPS('[1,2]',@d) AS flat; +obj scalar nested flat +0 0 0 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +SET @d = NULL; diff --git a/mysql-test/main/func_json_overlaps_depth.test b/mysql-test/main/func_json_overlaps_depth.test new file mode 100644 index 0000000000000..ea050230613dc --- /dev/null +++ b/mysql-test/main/func_json_overlaps_depth.test @@ -0,0 +1,106 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # A document nested deeper than the parser will go, handed to the +--echo # functions that compare one document against another. +--echo # +--echo # The parser stops at a fixed depth and records that it has. What it +--echo # does with its nesting counter on the way out decides what a caller +--echo # that keeps scanning afterwards will read, the counter being the +--echo # index into the stack of states the scanner works from. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. One container past the limit, in either argument. +--echo # + +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '{"a":1}') AS v; +SELECT JSON_OVERLAPS('{"a":1}', + CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +--echo # the other argument's kind does not matter +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '"s"') AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '[[1]]') AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '[1,2]') AS v; +--echo # both arguments past the limit +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; + +--echo # +--echo # 2. The same depth reached through objects rather than arrays. +--echo # + +SELECT JSON_OVERLAPS(CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32)), + '{"a":1}') AS v; +SELECT JSON_OVERLAPS('{"a":1}', + CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32))) AS v; +--echo # and mixed, an array holding an object chain. It is one container +--echo # past the limit exactly as the two above are, and the complaint it +--echo # draws has to say so: the document is complete and balanced, and +--echo # too deep is the only thing wrong with it. +SELECT JSON_OVERLAPS(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']'), + '{"a":1}') AS v; +SELECT JSON_DEPTH(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']')) AS v; + +--echo # +--echo # 3. One container inside the limit, which is a document and compares. +--echo # + +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), + '{"a":1}') AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), + CONCAT(REPEAT('[',31),'1',REPEAT(']',31))) AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), + '[1,2]') AS v; + +--echo # +--echo # 4. What the neighbouring functions answer on the same input. +--echo # + +SELECT JSON_CONTAINS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +SELECT JSON_EQUALS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +SELECT JSON_DEPTH(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +SELECT JSON_VALID(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +SELECT JSON_TYPE(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; + +--echo # +--echo # 5. Rows of them, so the engine is reused across evaluations. +--echo # + +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES + (1, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))), + (2, '[1,2]'), + (3, CONCAT(REPEAT('[',31),'1',REPEAT(']',31))), + (4, '{"a":1}'); +SELECT id, JSON_OVERLAPS(j, '[1,2]') AS v FROM t1 ORDER BY id; +SELECT id, JSON_OVERLAPS('[1,2]', j) AS v FROM t1 ORDER BY id; +SELECT id, JSON_OVERLAPS(j, j) AS v FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 6. Where the complaint says the document went too deep. +--echo # +--echo # The position is a property of the document alone. The second +--echo # argument decides how much comparing is attempted before the depth +--echo # is reached, and must not decide where the scanner is said to have +--echo # stopped: all four below are the same first argument, so all four +--echo # positions have to be the same one. +--echo # +--echo # It is also the same document the other functions are given in +--echo # section 4, so it has to be the position they report for it. +--echo # + +SET @d = CONCAT(REPEAT('[',32),'1',REPEAT(']',32)); +SELECT JSON_OVERLAPS(@d,'{"a":1}') AS obj, JSON_OVERLAPS(@d,'"s"') AS scalar, + JSON_OVERLAPS(@d,'[[1]]') AS nested, JSON_OVERLAPS(@d,'[1,2]') AS flat; +SELECT JSON_DEPTH(@d) AS v; +--echo # and the other way round, where it is argument 2 that is too deep +SELECT JSON_OVERLAPS('{"a":1}',@d) AS obj, JSON_OVERLAPS('"s"',@d) AS scalar, + JSON_OVERLAPS('[[1]]',@d) AS nested, JSON_OVERLAPS('[1,2]',@d) AS flat; +SET @d = NULL; diff --git a/mysql-test/main/func_json_overlaps_diag.result b/mysql-test/main/func_json_overlaps_diag.result new file mode 100644 index 0000000000000..b3b56fb721216 --- /dev/null +++ b/mysql-test/main/func_json_overlaps_diag.result @@ -0,0 +1,155 @@ +# +# Where JSON_OVERLAPS says a document broke. +# +# The comparison walks both arguments together and puts an engine +# back where it was to try the next candidate. A refusal met +# part-way through has to survive being put back, and the place it +# is reported from has to be where the document broke, not +# wherever the walk happened to stop afterwards. +# +# There is one right attest to each of these documents and it can +# be had elsewhere: JSON_VALID and JSON_EXTRACT read the same bytes +# with no comparison around them. Every case below is recorded +# beside those two, so the position is pinned to something other +# than itself. +# +# +# 1. What the same bytes are worth to a reader with nothing else +# going on. These are the positions every case below must give. +# +SELECT JSON_VALID('{"a":1 "b":2}') AS ok; +ok +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +SELECT JSON_EXTRACT('{"a":1, "b":2 "c":3}', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 15 +SELECT JSON_EXTRACT('{"a":{"p":1 "q":2}}', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +SELECT JSON_EXTRACT('[{"z":9},{"b":2 "c":3}]', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 17 +SELECT JSON_EXTRACT('[1,2,3 4]', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +# +# 2. The break is in argument 2, met after the walk has turned at +# least once. +# +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1 "b":2}') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +SELECT JSON_OVERLAPS('{"z":0}', '{"a":1, "b":2 "c":3}') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 15 +# +# 3. The break is below the top level, so the walk is further in +# when it meets it. +# +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1 "q":2}}') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 13 +# +# 4. The break is in argument 1, and inside an element of it. +# +SELECT JSON_OVERLAPS('[{"z":9},{"b":2 "c":3}]', '[{"a":1}]') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 17 +# +# 5. A refusal that is reached only after every candidate has been +# tried and put back. Nothing said so before: the walk ended on +# the refusal exactly as it ends on running out of candidates, +# and the answer came back with no diagnostic at all. +# +SELECT JSON_OVERLAPS('[9]', '[1,2,3 4]') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +SELECT JSON_OVERLAPS('[{"a":1}]', '[{"z":9},{"b":2 "c":3}]') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 17 +# +# 6. The same documents out of a table, so no one statement +# settles another, and both argument orders together. +# +CREATE TABLE t1 (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t1 VALUES +(1, '{"b":9}', '{"a":1 "b":2}'), +(2, '{"a":{"p":1}}', '{"a":{"p":1 "q":2}}'), +(3, '[9]', '[1,2,3 4]'), +(4, '{"b":9}', '{"a":1, "b":2}'); +SELECT id, JSON_OVERLAPS(a, b) AS v FROM t1 ORDER BY id; +id v +1 0 +2 0 +3 0 +4 0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 13 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +SELECT id, JSON_OVERLAPS(b, a) AS v FROM t1 ORDER BY id; +id v +1 0 +2 0 +3 0 +4 0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 8 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 13 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 8 +DROP TABLE t1; +# +# 7. Whole documents beside them, which have nothing to report. +# +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1, "b":9}') AS v; +v +1 +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS v; +v +1 +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1, "q":2}}') AS v; +v +0 +SELECT JSON_OVERLAPS('[9]', '[1,2,3,4]') AS v; +v +0 +# +# 8. A document that is refused before the comparison begins is +# not this case, and is reported from position 1 as it always +# was. +# +SELECT JSON_OVERLAPS('{', '[1]') AS v; +v +0 +SELECT JSON_OVERLAPS('[1]', 'x') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 1 diff --git a/mysql-test/main/func_json_overlaps_diag.test b/mysql-test/main/func_json_overlaps_diag.test new file mode 100644 index 0000000000000..e7523c467ec81 --- /dev/null +++ b/mysql-test/main/func_json_overlaps_diag.test @@ -0,0 +1,92 @@ +--echo # +--echo # Where JSON_OVERLAPS says a document broke. +--echo # +--echo # The comparison walks both arguments together and puts an engine +--echo # back where it was to try the next candidate. A refusal met +--echo # part-way through has to survive being put back, and the place it +--echo # is reported from has to be where the document broke, not +--echo # wherever the walk happened to stop afterwards. +--echo # +--echo # There is one right attest to each of these documents and it can +--echo # be had elsewhere: JSON_VALID and JSON_EXTRACT read the same bytes +--echo # with no comparison around them. Every case below is recorded +--echo # beside those two, so the position is pinned to something other +--echo # than itself. +--echo # + +--echo # +--echo # 1. What the same bytes are worth to a reader with nothing else +--echo # going on. These are the positions every case below must give. +--echo # + +SELECT JSON_VALID('{"a":1 "b":2}') AS ok; +--error 0 +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS v; +SELECT JSON_EXTRACT('{"a":1, "b":2 "c":3}', '$') AS v; +SELECT JSON_EXTRACT('{"a":{"p":1 "q":2}}', '$') AS v; +SELECT JSON_EXTRACT('[{"z":9},{"b":2 "c":3}]', '$') AS v; +SELECT JSON_EXTRACT('[1,2,3 4]', '$') AS v; + +--echo # +--echo # 2. The break is in argument 2, met after the walk has turned at +--echo # least once. +--echo # + +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1 "b":2}') AS v; +SELECT JSON_OVERLAPS('{"z":0}', '{"a":1, "b":2 "c":3}') AS v; + +--echo # +--echo # 3. The break is below the top level, so the walk is further in +--echo # when it meets it. +--echo # + +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1 "q":2}}') AS v; + +--echo # +--echo # 4. The break is in argument 1, and inside an element of it. +--echo # + +SELECT JSON_OVERLAPS('[{"z":9},{"b":2 "c":3}]', '[{"a":1}]') AS v; + +--echo # +--echo # 5. A refusal that is reached only after every candidate has been +--echo # tried and put back. Nothing said so before: the walk ended on +--echo # the refusal exactly as it ends on running out of candidates, +--echo # and the answer came back with no diagnostic at all. +--echo # + +SELECT JSON_OVERLAPS('[9]', '[1,2,3 4]') AS v; +SELECT JSON_OVERLAPS('[{"a":1}]', '[{"z":9},{"b":2 "c":3}]') AS v; + +--echo # +--echo # 6. The same documents out of a table, so no one statement +--echo # settles another, and both argument orders together. +--echo # + +CREATE TABLE t1 (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t1 VALUES + (1, '{"b":9}', '{"a":1 "b":2}'), + (2, '{"a":{"p":1}}', '{"a":{"p":1 "q":2}}'), + (3, '[9]', '[1,2,3 4]'), + (4, '{"b":9}', '{"a":1, "b":2}'); +SELECT id, JSON_OVERLAPS(a, b) AS v FROM t1 ORDER BY id; +SELECT id, JSON_OVERLAPS(b, a) AS v FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 7. Whole documents beside them, which have nothing to report. +--echo # + +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1, "b":9}') AS v; +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS v; +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1, "q":2}}') AS v; +SELECT JSON_OVERLAPS('[9]', '[1,2,3,4]') AS v; + +--echo # +--echo # 8. A document that is refused before the comparison begins is +--echo # not this case, and is reported from position 1 as it always +--echo # was. +--echo # + +SELECT JSON_OVERLAPS('{', '[1]') AS v; +SELECT JSON_OVERLAPS('[1]', 'x') AS v; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index a37aabbe1c43c..4dd7fb6eed706 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -527,7 +527,13 @@ void report_json_error_ex(const char *js, json_engine_t *je, Sql_condition::enum_warning_level lv) { THD *thd= current_thd; - int position= (int)((const char *) je->s.c_str - js); + /* + Where the scanner refused, which is not where it came to rest: it + goes on being asked for the next token by callers that are not + obliged to stop, so json_error() takes the reading at the refusal + and this reports that one. + */ + int position= (int)((const char *) je->s.error_pos - js); uint code; n_param++; @@ -618,7 +624,7 @@ void report_path_error_ex(const char *ps, json_path_t *p, Sql_condition::enum_warning_level lv) { THD *thd= current_thd; - int position= (int)((const char *) p->s.c_str - ps + 1); + int position= (int)((const char *) p->s.error_pos - ps + 1); uint code; n_param++; @@ -677,7 +683,7 @@ static int path_setup_nwc(json_path_t *p, CHARSET_INFO *i_cs, if ((p->types_used & (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD | JSON_PATH_ARRAY_RANGE)) == 0) return 0; - p->s.error= NO_WILDCARD_ALLOWED; + json_error(&p->s, NO_WILDCARD_ALLOWED); } return 1; @@ -2226,7 +2232,7 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, } if (je.s.error == 0 && max_level + (int) depth >= JSON_DEPTH_LIMIT) - je.s.error= JE_DEPTH; + json_error(&je.s, JE_DEPTH); /* Said here rather than left to the caller: the callers report through @@ -2710,7 +2716,7 @@ String *Item_func_json_array_insert::val_str(String *str) c_path->p.last_step->type != JSON_PATH_ARRAY) { if (c_path->p.s.error == 0) - c_path->p.s.error= SHOULD_END_WITH_ARRAY; + json_error(&c_path->p.s, SHOULD_END_WITH_ARRAY); path_err: report_path_error(s_p, &c_path->p, n_arg); @@ -4116,7 +4122,7 @@ String *Item_func_json_remove::val_str(String *str) c_path->p.last_step--; if (c_path->p.last_step < c_path->p.steps) { - c_path->p.s.error= TRIVIAL_PATH_NOT_ALLOWED; + json_error(&c_path->p.s, TRIVIAL_PATH_NOT_ALLOWED); report_path_error(s_p, &c_path->p, n_arg); goto null_return; } @@ -5238,6 +5244,27 @@ void json_skip_current_level(json_engine_t *js, json_engine_t *value) } +/* + Put an engine back where it was remembered from, so the next candidate + can be tried against it. + + A refusal has to survive that. The snapshot was taken before the + engine failed, so restoring over the failure returns an engine that + reads as healthy, standing somewhere the scanner never reached; the + walk then goes on questioning a document it can no longer read, and + the complaint that is finally made - if one is made at all - names + wherever the walk ran out rather than what was wrong with the + document. Nonzero says the walk is over. +*/ +static int json_resume_scan(json_engine_t *je, const json_engine_t *saved) +{ + if (je->s.error) + return 1; + *je= *saved; + return 0; +} + + /* At least one of the two arguments is a scalar. */ bool json_find_overlap_with_scalar(json_engine_t *js, json_engine_t *value) { @@ -5303,7 +5330,8 @@ bool json_compare_arr_and_obj(json_engine_t *js, json_engine_t *value) int res1= json_find_overlap_with_object(js, value, true); if (res1) return TRUE; - *value= loc_val; + if (json_resume_scan(value, &loc_val)) + return FALSE; } if (js->value_type == JSON_VALUE_ARRAY) json_skip_level(js); @@ -5369,9 +5397,11 @@ int json_find_overlap_with_array(json_engine_t *js, json_engine_t *value, if (!json_value_scalar(value)) json_skip_level(value); } - *js= current_js; + if (json_resume_scan(js, ¤t_js)) + return FALSE; } - *value= loc_value; + if (json_resume_scan(value, &loc_value)) + return FALSE; if (!json_value_scalar(js)) json_skip_level(js); } @@ -5396,8 +5426,16 @@ int compare_nested_object(json_engine_t *js, json_engine_t *value) int result= 0; const char *value_begin= (const char*)value->s.c_str-1; const char *js_begin= (const char*)js->s.c_str-1; - json_skip_level(value); - json_skip_level(js); + /* + A refusal here means the fragment was not read to its end, so what + lies between begin and end is a piece of a document rather than one. + Normalizing that piece finds it unterminated and says so, which + replaces the reason the scanner gave with a complaint about where it + was made to stop. The engine already carries the reason; leave it + alone and let the caller report it. + */ + if (json_skip_level(value) || json_skip_level(js)) + return 0; const char *value_end= (const char*)value->s.c_str; const char *js_end= (const char*)js->s.c_str; json_engine_t je; @@ -5412,14 +5450,20 @@ int compare_nested_object(json_engine_t *js, json_engine_t *value) { goto error; } + /* + Only the code is carried over. The normalizing engine read a copy of + the value, so where it stopped is a position in that copy and says + nothing about this document; json_error() takes the reading from the + engine being refused, which is the one the caller reports through. + */ if (json_normalize_engine(&je, &a_res, a.ptr(), a.length(), value->s.cs)) { - value->s.error= je.s.error; + json_error(&value->s, je.s.error); goto error; } if (json_normalize_engine(&je, &b_res, b.ptr(), b.length(), value->s.cs)) { - js->s.error= je.s.error; + json_error(&js->s, je.s.error); goto error; } @@ -5498,7 +5542,8 @@ int json_find_overlap_with_object(json_engine_t *js, json_engine_t *value, only js (first argument i.e json document) and continue. */ - *js= loc_js; + if (json_resume_scan(js, &loc_js)) + return FALSE; continue; } } @@ -5515,7 +5560,8 @@ int json_find_overlap_with_object(json_engine_t *js, json_engine_t *value, return FALSE; if (!json_value_scalar(value)) json_skip_level(value); - *js= loc_js; + if (json_resume_scan(js, &loc_js)) + return FALSE; } } /* diff --git a/strings/json_lib.c b/strings/json_lib.c index 01df72bd522a7..1a77082d9ff8c 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -40,6 +40,7 @@ void json_string_set_cs(json_string_t *s, CHARSET_INFO *i_cs) { s->cs= i_cs; s->error= 0; + s->error_pos= NULL; s->wc= i_cs->cset->mb_wc; } @@ -123,7 +124,7 @@ typedef int (*json_state_handler)(json_engine_t *); /* The string is broken. */ static int unexpected_eos(json_engine_t *j) { - j->s.error= JE_EOS; + json_error(&j->s, JE_EOS); return 1; } @@ -131,21 +132,37 @@ static int unexpected_eos(json_engine_t *j) /* This symbol here breaks the JSON syntax. */ static int syntax_error(json_engine_t *j) { - j->s.error= JE_SYN; + json_error(&j->s, JE_SYN); return 1; } +/* + The four functions below open a nested structure, and each refuses to + open one deeper than the scanner can keep track of. + + A refusal must leave the nesting counter where it was. It indexes + 'stack', it is declared immediately after 'stack', and 'stack' is + exactly JSON_DEPTH_LIMIT long, so a counter moved past the end of the + stack addresses the counter itself: a caller that reads + j->stack[j->stack_p] then gets JSON_DEPTH_LIMIT back as though it were + a state, and acts on it. Callers are not all obliged to stop the + moment they are told the document is too deep, so this cannot be left + for them to get right - which is also why json_error() keeps the first + refusal and the reading taken at it, rather than the scanner being + stopped where it failed. +*/ + /* Value of object. */ static int mark_object(json_engine_t *j) { j->state= JST_OBJ_START; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_OBJ_CONT; + j->stack[++j->stack_p]= JST_OBJ_CONT; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -156,12 +173,12 @@ static int read_obj(json_engine_t *j) j->state= JST_OBJ_START; j->value_type= JSON_VALUE_OBJECT; j->value= j->value_begin; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_OBJ_CONT; + j->stack[++j->stack_p]= JST_OBJ_CONT; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -170,13 +187,13 @@ static int read_obj(json_engine_t *j) static int mark_array(json_engine_t *j) { j->state= JST_ARRAY_START; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_ARRAY_CONT; + j->stack[++j->stack_p]= JST_ARRAY_CONT; j->value= j->value_begin; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -186,12 +203,12 @@ static int read_array(json_engine_t *j) j->state= JST_ARRAY_START; j->value_type= JSON_VALUE_ARRAY; j->value= j->value_begin; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_ARRAY_CONT; + j->stack[++j->stack_p]= JST_ARRAY_CONT; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -264,10 +281,10 @@ static int read_4_hexdigits(json_string_t *s, uchar *dest) for (i=0; i<4; i++) { if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); if (s->c_next >= 128 || (t= json_instr_chr_map[s->c_next]) > S_F) - return s->error= JE_SYN; + return json_error(s, JE_SYN); s->c_str+= c_len; dest[i/2]+= (i % 2) ? t : t*16; @@ -281,7 +298,7 @@ static int json_handle_esc(json_string_t *s) int t, c_len; if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); s->c_str+= c_len; switch (s->c_next) @@ -306,7 +323,7 @@ static int json_handle_esc(json_string_t *s) if (s->c_next < 128 && (t= json_instr_chr_map[s->c_next]) == S_ERR) { s->c_str-= c_len; - return s->error= JE_ESCAPING; + return json_error(s, JE_ESCAPING); } @@ -328,18 +345,18 @@ static int json_handle_esc(json_string_t *s) return 0; if (c_len != MY_CS_TOOSMALL4) - return s->error= JE_BAD_CHR; + return json_error(s, JE_BAD_CHR); if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); if (s->c_next != '\\') - return s->error= JE_SYN; + return json_error(s, JE_SYN); s->c_str+= c_len; if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); if (s->c_next != 'u') - return s->error= JE_SYN; + return json_error(s, JE_SYN); s->c_str+= c_len; if (read_4_hexdigits(s, code+2)) @@ -348,7 +365,7 @@ static int json_handle_esc(json_string_t *s) if ((c_len= my_utf16_uni(0, &s->c_next, code, code+4)) == 4) return 0; } - return s->error= JE_BAD_CHR; + return json_error(s, JE_BAD_CHR); } @@ -361,7 +378,7 @@ int json_read_string_const_chr(json_string_t *js) js->c_str+= c_len; return (js->c_next == '\\') ? json_handle_esc(js) : 0; } - js->error= json_eos(js) ? JE_EOS : JE_BAD_CHR; + json_error(js, json_eos(js) ? JE_EOS : JE_BAD_CHR); return 1; } @@ -387,10 +404,10 @@ static int skip_str_constant(json_engine_t *j) continue; } /* Symbol not allowed in JSON. */ - return j->s.error= JE_NOT_JSON_CHR; + return json_error(&j->s, JE_NOT_JSON_CHR); } else - return j->s.error= json_eos(&j->s) ? JE_EOS : JE_BAD_CHR; + return json_error(&j->s, json_eos(&j->s) ? JE_EOS : JE_BAD_CHR); } j->state= j->stack[j->stack_p]; @@ -528,7 +545,7 @@ static int skip_num_constant(json_engine_t *j) break; } - if ((j->s.error= + if (json_error(&j->s, json_eos(&j->s) ? json_num_states[state][N_END] : JE_BAD_CHR) < 0) return 1; else @@ -574,9 +591,9 @@ static int skip_string_verbatim(json_string_t *s, const char *str) s->c_str+= c_len; continue; } - return s->error= JE_SYN; + return json_error(s, JE_SYN); } - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); } return 0; @@ -649,7 +666,7 @@ static int read_true(json_engine_t *j) /* Disallowed character. */ static int not_json_chr(json_engine_t *j) { - j->s.error= JE_NOT_JSON_CHR; + json_error(&j->s, JE_NOT_JSON_CHR); return 1; } @@ -657,7 +674,7 @@ static int not_json_chr(json_engine_t *j) /* Bad character. */ static int bad_chr(json_engine_t *j) { - j->s.error= JE_BAD_CHR; + json_error(&j->s, JE_BAD_CHR); return 1; } @@ -722,9 +739,9 @@ static int next_key(json_engine_t *j) return 0; } - j->s.error= (t_next == C_EOS) ? JE_EOS : - ((t_next == C_BAD) ? JE_BAD_CHR : - JE_SYN); + json_error(&j->s, (t_next == C_EOS) ? JE_EOS : + ((t_next == C_BAD) ? JE_BAD_CHR : + JE_SYN)); return 1; } @@ -831,9 +848,9 @@ static int skip_colon(json_engine_t *j) return json_actions[JST_VALUE][t_next](j); } - j->s.error= (t_next == C_EOS) ? JE_EOS : - ((t_next == C_BAD) ? JE_BAD_CHR: - JE_SYN); + json_error(&j->s, (t_next == C_EOS) ? JE_EOS : + ((t_next == C_BAD) ? JE_BAD_CHR: + JE_SYN)); return 1; } @@ -850,7 +867,7 @@ static int skip_key(json_engine_t *j) while (json_read_keyname_chr(j) == 0) {} - if (j->s.error) + if (unlikely(j->s.error)) return 1; get_first_nonspace(&j->s, &t_next, &c_len); @@ -927,10 +944,10 @@ int json_read_keyname_chr(json_engine_t *j) j->s.c_str+= c_len; continue; } - j->s.error= JE_SYN; + json_error(&j->s, JE_SYN); break; } - j->s.error= json_eos(&j->s) ? JE_EOS : JE_BAD_CHR; + json_error(&j->s, json_eos(&j->s) ? JE_EOS : JE_BAD_CHR); break; } return 1; @@ -938,11 +955,11 @@ int json_read_keyname_chr(json_engine_t *j) return json_handle_esc(&j->s); case S_ERR: j->s.c_str-= c_len; - j->s.error= JE_STRING_CONST; + json_error(&j->s, JE_STRING_CONST); return 1; } } - j->s.error= json_eos(&j->s) ? JE_EOS : JE_BAD_CHR; + json_error(&j->s, json_eos(&j->s) ? JE_EOS : JE_BAD_CHR); return 1; } @@ -956,7 +973,7 @@ int json_read_value(json_engine_t *j) { while (json_read_keyname_chr(j) == 0) {} - if (j->s.error) + if (unlikely(j->s.error)) return 1; } @@ -976,7 +993,7 @@ int json_scan_next(json_engine_t *j) get_first_nonspace(&j->s, &t_next, &j->sav_c_len); if (j->killed_ptr && *j->killed_ptr) { - j->s.error= JE_KILLED; + json_error(&j->s, JE_KILLED); return 1; } return json_actions[j->state][t_next](j); @@ -1149,19 +1166,19 @@ int json_path_setup(json_path_t *p, t_next= (p->s.c_next >= 128) ? P_ETC : json_path_chr_map[p->s.c_next]; if ((state= json_path_transitions[state][t_next]) < 0) - return p->s.error= state; + return json_error(&p->s, state); p->s.c_str+= c_len; switch (state) { case PS_LAX: - if ((p->s.error= skip_string_verbatim(&p->s, "ax"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "ax"))) return 1; p->mode_strict= FALSE; continue; case PS_SCT: - if ((p->s.error= skip_string_verbatim(&p->s, "rict"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "rict"))) return 1; p->mode_strict= TRUE; state= PS_LAX; @@ -1204,7 +1221,7 @@ int json_path_setup(json_path_t *p, is_negative_index= 0; is_last= 0; if (p->last_step - p->steps >= JSON_DEPTH_LIMIT) - return p->s.error= JE_DEPTH; + return json_error(&p->s, JE_DEPTH); p->types_used|= p->last_step->type= JSON_PATH_KEY | double_wildcard; double_wildcard= JSON_PATH_KEY_NULL; /* fall through */ @@ -1222,7 +1239,7 @@ int json_path_setup(json_path_t *p, prev_value= 0; is_negative_index= 0; if (p->last_step - p->steps >= JSON_DEPTH_LIMIT) - return p->s.error= JE_DEPTH; + return json_error(&p->s, JE_DEPTH); p->types_used|= p->last_step->type= JSON_PATH_ARRAY | double_wildcard; double_wildcard= JSON_PATH_KEY_NULL; p->last_step->n_item= 0; @@ -1249,7 +1266,7 @@ int json_path_setup(json_path_t *p, is_negative_index= 1; continue; case PS_LAST: - if ((p->s.error= skip_string_verbatim(&p->s, "ast"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "ast"))) return 1; p->types_used|= JSON_PATH_NEGATIVE_INDEX; is_last= 1; @@ -1259,7 +1276,7 @@ int json_path_setup(json_path_t *p, p->last_step->n_item= -1; continue; case PS_T: - if ((p->s.error= skip_string_verbatim(&p->s, "o"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "o"))) return 1; is_to= 1; is_negative_index= 0; @@ -1272,7 +1289,7 @@ int json_path_setup(json_path_t *p, }; } while (state != PS_OK); - return double_wildcard ? (p->s.error= JE_SYN) : 0; + return double_wildcard ? json_error(&p->s, JE_SYN) : 0; } @@ -1316,7 +1333,15 @@ int json_skip_array_and_count(json_engine_t *je, int *n_items) res= json_skip_level_and_count(&j, n_items); if (res) + { + /* + The copy scanned the same string, so where it stopped is a + position in the caller's string too, and it has to come back with + the code: what reports the refusal reads the two together. + */ je->s.error= j.s.error; + je->s.error_pos= j.s.error_pos; + } return res; } diff --git a/strings/json_normalize.c b/strings/json_normalize.c index 5243bd3c7064b..b74cd21a35bb6 100644 --- a/strings/json_normalize.c +++ b/strings/json_normalize.c @@ -931,7 +931,17 @@ json_normalize_engine(json_engine_t *je, DYNAMIC_STRING *result, { /* resulting error offset is mapped to original string */ if (je->s.error) + { je->s.c_str= (const uchar *) (s + (ptrdiff_t)(je->s.c_str - (const uchar *) s_utf8)); + /* + And the reading taken where the refusal happened, which is the + one reported. The converted copy is freed on the next line, so + a position left pointing into it is not merely in the wrong + string but in no string at all. + */ + je->s.error_pos= (const uchar *) + (s + (ptrdiff_t)(je->s.error_pos - (const uchar *) s_utf8)); + } my_free(s_utf8); } return err; From f593f0edcd06670cf3aab1e63e536958ff93b1b1 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 3 Aug 2026 12:56:26 -0400 Subject: [PATCH 05/30] MDEV-40606 Build an aggregated array in the character set it declares `JSON_ARRAYAGG` puts the brackets around its elements by building them in a scratch `String` and exchanging that with the result. The scratch one was left at its default character set, which is `my_charset_bin`, and `String::swap()` carries the character set across along with the bytes, so the result was returned saying it was bytes rather than text. That cost twice over: 1. **The brackets were written one byte wide** whatever width a character of the result takes. In `ucs2`, `utf16` and `utf32` that left everything between them a byte out of step and the result read as an entirely different set of characters: `JSON_VALID()` answered 0, and one recorded value came back as six CJK ideographs in place of `[["x"]]`. Only the two brackets were written this way; the elements between them, and the separators, always had the result's own character set. 2. **The result was not converted on its way to the client.** `Protocol::needs_conversion()` reads a binary label as needing no conversion, so `JSON_ARRAYAGG` alone among these functions ignored `character_set_results`. Under `character_set_results = swe7`, which has no code point for a bracket, every other JSON function arrived with those characters replaced and this one arrived untouched. One line settles both: the scratch buffer is given the character set the expression was resolved to, so the exchange hands that to the result, and `String::append(char)` writes a bracket at the width that character set asks for. A result computed in `latin1` or `utf8mb4` is unchanged to the byte. Stored copies are unaffected either way, `Item::save_str_in_field()` passing the item's declared collation to `Field::store()` rather than the returned value's own label. `func_json_agg_charset` was written one commit earlier and records this half, so the change shows there as a move rather than as a new file. Its header states the two questions the file asks: the delivery conversion, which `swe7` makes visible because it has no punctuation to convert into and `latin1` hides because it has, and the character set the result is computed in, which the later sections vary while leaving the client alone. Sections 1 and 5 stop arriving unconverted under `character_set_results = swe7`; section 6 puts the brackets at full width and `JSON_VALID()` answers 1 where it answered 0. There is no unconverted delivery left anywhere in the file: what section 3 records under `latin1` is both aggregates arriving intact, which is the conversion having happened and having had nothing to change. One statement is added there, reading the `ucs2` result back as `utf8mb4`. It could not be recorded before the change: the value being passed unconverted, that statement answered with the raw bytes and the NULs between them, which is not text and cannot be written down in a result file. Two recorded baselines move as well, both repairs: - `func_json_charset`, one line: the delivery is now converted. - `func_json_embed`, seven lines: `ucs2` brackets now at full width. One answer changes for a result the expression resolves to `swe7`, and that one is not a repair of a formatting but a different answer to a question: SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) FROM t1; Before, 1; now, 0. The bytes are the same either way, `5B226161222C226262222C226363222C226464225D`. What moved is the label they are read under: they used to arrive as `binary`, where a byte is a character and `5B` is a bracket, and they now arrive as `swe7`, where `5B` is a national letter. Neither `swe7` nor any other character set lacking the JSON punctuation can hold a document, so 0 is the answer that matches what is there; the 1 was an artifact of the wrong label, and removing that label is this change. The sibling aggregate already answered 0 for the same reason and does not move. Section 8 of `func_json_agg_charset` records both, `swe7` being what the expression resolves to there rather than what the client asked for, which is what every earlier section uses it for. --- mysql-test/main/func_json_agg_charset.result | 99 ++++++++++++++------ mysql-test/main/func_json_agg_charset.test | 50 +++++++--- mysql-test/main/func_json_charset.result | 2 +- mysql-test/main/func_json_embed.result | 14 +-- sql/item_jsonfunc.cc | 14 +++ 5 files changed, 130 insertions(+), 49 deletions(-) diff --git a/mysql-test/main/func_json_agg_charset.result b/mysql-test/main/func_json_agg_charset.result index a5fb8019f4c1d..86e964b42db84 100644 --- a/mysql-test/main/func_json_agg_charset.result +++ b/mysql-test/main/func_json_agg_charset.result @@ -2,18 +2,24 @@ # How the JSON aggregates deliver their result to the client. # # A result computed in one character set and asked for in another is -# converted on the way out. Whether that conversion happens at all is -# decided by the character set the returned value carries, which need -# not be the one the function says it has. +# converted on the way out. Two separate questions decide what the +# client is left holding: which character set the result was computed +# in, and whether what is in it survives the conversion to the one +# that was asked for. # -# swe7 is the vehicle throughout. It has no code point for the -# brackets and braces a JSON document is built from, so a result that -# was converted arrives with those replaced by '?', and one passed on -# unconverted arrives intact. The difference is therefore visible +# swe7 is the vehicle for the first of those. It has no code point +# for the brackets and braces a JSON document is built from, so a +# result converted into it arrives with those replaced by '?'. That +# is what a conversion having happened looks like, and it is visible # without a single non-ASCII byte in this file. A client asking for -# latin1 is told the same two stories, but the second one reaches it as -# utf8mb4 bytes under a latin1 label, which is not printable and not -# recoverable. +# latin1 sees nothing to remark on, latin1 having the punctuation: +# the conversion happens just the same and the result arrives intact. +# +# The later sections ask the other question, leaving the client alone +# and computing the result in a character set of its own. There the +# punctuation has to go down at the width that character set asks +# for, and where the character set has no punctuation to write, what +# is written is not a document at all. # SET NAMES utf8mb4; CREATE TABLE t1 (g INT, a VARCHAR(20) CHARACTER SET utf8mb4); @@ -36,15 +42,15 @@ v ?"aa"? SELECT JSON_ARRAYAGG(a) AS v FROM t1; v -["aa","bb","cc","dd"] +?"aa","bb","cc","dd"? SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; v ?"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"? # one group per row of output, so the delivery is repeated SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; g v -1 ["aa","bb"] -2 ["cc","dd"] +1 ?"aa","bb"? +2 ?"cc","dd"? SELECT g, JSON_OBJECTAGG(a,a) AS v FROM t1 GROUP BY g ORDER BY g; g v 1 ?"aa":"aa", "bb":"bb"? @@ -127,14 +133,14 @@ SET SESSION character_set_results = DEFAULT; SET SESSION character_set_results = swe7; SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1; v -["aa","bb","cc","dd"] +?"aa","bb","cc","dd"? SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1; v -["dd","cc","bb","aa"] +?"dd","cc","bb","aa"? # a group of one writes no separator, so it exercises less than it looks SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'aa'; v -["aa"] +?"aa"? # no rows at all: NULL, which is never delivered as a string SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; v @@ -154,32 +160,29 @@ SET SESSION character_set_results = DEFAULT; # SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1; h -5B0022006100610022002C0022006200620022002C0022006300630022002C00220064006400225D +005B0022006100610022002C0022006200620022002C0022006300630022002C0022006400640022005D SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS valid FROM t1; valid -0 -Warnings: -Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 2 +1 +SELECT CONVERT(JSON_ARRAYAGG(CONVERT(a USING ucs2)) USING utf8mb4) AS v FROM t1; +v +["aa","bb","cc","dd"] SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS h FROM t1; h -5B0022006100610022002C0022006200620022002C0022006300630022002C00220064006400225D +005B0022006100610022002C0022006200620022002C0022006300630022002C0022006400640022005D SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS valid FROM t1; valid -0 -Warnings: -Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 2 +1 SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS h FROM t1; h -5B000000220000006100000061000000220000002C000000220000006200000062000000220000002C000000220000006300000063000000220000002C000000220000006400000064000000225D +0000005B000000220000006100000061000000220000002C000000220000006200000062000000220000002C000000220000006300000063000000220000002C000000220000006400000064000000220000005D SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS valid FROM t1; valid -0 -Warnings: -Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 2 +1 # a group of one, which is bracket, element, bracket and nothing else SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1 WHERE a = 'aa'; h -5B00220061006100225D +005B0022006100610022005D # the same values where one byte is one character, which is the case # the brackets were always written for SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING latin1))) AS h FROM t1; @@ -276,5 +279,43 @@ SELECT g, HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1 GROUP BY g ORDER BY g; g h 1 7B226161223A226161222C20226262223A226262227D 2 7B226363223A226363222C20226464223A226464227D +# +# 8. swe7 as the character set the aggregate is resolved to. +# +# Everywhere above, swe7 is only what the client asked for. Here it +# is what the expression resolves to, so the punctuation is written +# in it - and swe7 puts national letters where the brackets and the +# braces belong. What is written is therefore not a document, and +# JSON_VALID() answers 0 for it. +# +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +SELECT CHARSET(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS cs FROM t1; +cs +swe7 +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid FROM t1; +valid +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT g, JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid +FROM t1 GROUP BY g ORDER BY g; +g valid +1 0 +2 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING swe7), +CONVERT(a USING swe7))) AS h FROM t1; +h +7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING swe7), +CONVERT(a USING swe7))) AS valid FROM t1; +valid +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 DROP TABLE t1, t2; SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_agg_charset.test b/mysql-test/main/func_json_agg_charset.test index ff8aa271fbe8b..4587ddd4e63da 100644 --- a/mysql-test/main/func_json_agg_charset.test +++ b/mysql-test/main/func_json_agg_charset.test @@ -4,21 +4,27 @@ --source include/have_utf32.inc --echo # ---echo # How the JSON aggregates hand their result to the client. +--echo # How the JSON aggregates deliver their result to the client. --echo # --echo # A result computed in one character set and asked for in another is ---echo # converted on the way out. Whether that conversion happens at all is ---echo # decided by the character set the returned value carries, which need ---echo # not be the one the function says it has. ---echo # ---echo # swe7 is the vehicle throughout. It has no code point for the ---echo # brackets and braces a JSON document is built from, so a result that ---echo # was converted arrives with those replaced by '?', and one passed on ---echo # unconverted arrives intact. The difference is therefore visible +--echo # converted on the way out. Two separate questions decide what the +--echo # client is left holding: which character set the result was computed +--echo # in, and whether what is in it survives the conversion to the one +--echo # that was asked for. +--echo # +--echo # swe7 is the vehicle for the first of those. It has no code point +--echo # for the brackets and braces a JSON document is built from, so a +--echo # result converted into it arrives with those replaced by '?'. That +--echo # is what a conversion having happened looks like, and it is visible --echo # without a single non-ASCII byte in this file. A client asking for ---echo # latin1 is told the same two stories, but the second one reaches it as ---echo # utf8mb4 bytes under a latin1 label, which is not printable and not ---echo # recoverable. +--echo # latin1 sees nothing to remark on, latin1 having the punctuation: +--echo # the conversion happens just the same and the result arrives intact. +--echo # +--echo # The later sections ask the other question, leaving the client alone +--echo # and computing the result in a character set of its own. There the +--echo # punctuation has to go down at the width that character set asks +--echo # for, and where the character set has no punctuation to write, what +--echo # is written is not a document at all. --echo # SET NAMES utf8mb4; @@ -120,6 +126,7 @@ SET SESSION character_set_results = DEFAULT; SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1; SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS valid FROM t1; +SELECT CONVERT(JSON_ARRAYAGG(CONVERT(a USING ucs2)) USING utf8mb4) AS v FROM t1; SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS h FROM t1; SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS valid FROM t1; SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS h FROM t1; @@ -183,5 +190,24 @@ SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf8mb4), CONVERT(a USING utf8mb4))) AS h FROM t1; SELECT g, HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1 GROUP BY g ORDER BY g; +--echo # +--echo # 8. swe7 as the character set the aggregate is resolved to. +--echo # +--echo # Everywhere above, swe7 is only what the client asked for. Here it +--echo # is what the expression resolves to, so the punctuation is written +--echo # in it - and swe7 puts national letters where the brackets and the +--echo # braces belong. What is written is therefore not a document, and +--echo # JSON_VALID() answers 0 for it. +--echo # +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS h FROM t1; +SELECT CHARSET(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS cs FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid FROM t1; +SELECT g, JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid + FROM t1 GROUP BY g ORDER BY g; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING swe7), + CONVERT(a USING swe7))) AS h FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING swe7), + CONVERT(a USING swe7))) AS valid FROM t1; + DROP TABLE t1, t2; SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result index 717ce0694f483..a2ad055dac99f 100644 --- a/mysql-test/main/func_json_charset.result +++ b/mysql-test/main/func_json_charset.result @@ -565,7 +565,7 @@ v ?a? SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; v -["a","b"] +?"a","b"? SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; v ?"a":"a", "b":"b"? diff --git a/mysql-test/main/func_json_embed.result b/mysql-test/main/func_json_embed.result index f9d3d88931428..3eb2be4e6ef48 100644 --- a/mysql-test/main/func_json_embed.result +++ b/mysql-test/main/func_json_embed.result @@ -568,16 +568,16 @@ Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at positi SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v FROM t6 GROUP BY id WITH ROLLUP; id v -1 5B005B002200780022005D5D -2 5B005B002200780022005D5D -3 5B005B002200780022005D5D -NULL 5B005B002200780022005D002C005B002200780022005D002C005B002200780022005D5D +1 005B005B002200780022005D005D +2 005B005B002200780022005D005D +3 005B005B002200780022005D005D +NULL 005B005B002200780022005D002C005B002200780022005D002C005B002200780022005D005D SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v FROM t6 GROUP BY id; id v -1 5B005B002200780022005D5D -2 5B005B002200780022005D5D -3 5B005B002200780022005D5D +1 005B005B002200780022005D005D +2 005B005B002200780022005D005D +3 005B005B002200780022005D005D # nesting composed by the aggregate itself, which puts each value # inside one array or object of its own SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 4dd7fb6eed706..b6b37f8467c19 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -4981,6 +4981,20 @@ String* Item_func_json_arrayagg::val_str(String *str) { String s; + /* + The brackets are built up in a buffer of their own which is then + exchanged with the result, and the exchange carries the character + set across along with the bytes. Left at its default the buffer + would say that what it holds is bytes rather than text, which + costs twice over: a bracket is then written one byte wide even + where a character of this result is two or four, leaving + everything between the two of them a byte out of step; and a + client asking for a character set other than the one the result + was computed in is handed the bytes as they stand, where every + other function here converts them first. + */ + s.set_charset(collation.collation); + /* A row that could not be written out left the elements it sits between joined by nothing but their separator, so there is no From 8e120bce359dfc1877161cba266aaa596b6eff7e Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 3 Aug 2026 23:20:34 -0400 Subject: [PATCH 06/30] MDEV-40612 Close a JSON aggregate's result once per group, not per call Both JSON aggregates put the last of their punctuation on in the call that returns the result. Nothing says how often a statement asks for the result of a group, and the buffer being written into belongs to the item and outlives the asking, so a statement that asks twice gets a value closed twice. `JSON_OBJECTAGG` returns a document with an extra closing brace, which is not JSON at all. `JSON_ARRAYAGG` returns one wrapped in an extra pair of brackets, which is valid JSON of the wrong value - the quieter of the two, since nothing downstream has any reason to reject it. Neither raises a warning or an error. `HAVING` on the alias is the shortest statement that asks twice, and the count follows the number of conditions: SELECT JSON_OBJECTAGG(a,a) AS v FROM t1 HAVING v LIKE '%'; {"x":"x", "y":"y"}} SELECT JSON_OBJECTAGG(a,a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '{%'; {"x":"x", "y":"y"}}} Every group of a `GROUP BY` is affected, not only the first. The parent of one of the two has the same exposure and guards against it: `GROUP_CONCAT` keeps a flag saying the result has been produced, and a second call returns the buffer without rebuilding it. That flag protects the elements, which is why the elements are not duplicated above. It does not reach the brackets or the brace, those being written by the derived classes after the parent has returned. Each aggregate now keeps the same kind of flag for the punctuation it writes itself, reset per group in `clear()` next to the flags already there. `JSON_OBJECTAGG` skips the closing brace when the result is already closed; `JSON_ARRAYAGG` returns what the parent gave it, already inside its brackets. Both flags are initialised in the copy constructor as well as in the first one, for the symmetry rather than for a state a reader could reach: the first row of every group - the copy `WITH ROLLUP` evaluates included - goes through `Item_sum::reset_and_add()`, which calls `clear()`, and both `clear()` bodies set the flag before anything looks at it. Given `m_closed(true)` in both copy constructors, which is the value that would spoil the answer if it were ever read, every recorded answer stays where it is. New test `func_json_agg_reread`: asked once, twice and three times; every group of a `GROUP BY`; a group of one row, where no separator is ever written; the `WITH ROLLUP` copy; and a group with no rows, which is a NULL and stays one however often it is asked for. The rollup section asks through `HAVING` like the rest of the file, because the condition is what makes it a test of the copy rather than of the copy being made. With no condition each level is asked exactly once and is closed once whether the flag is kept or not, and those statements answer `{"x":"x", "y":"y"}` and `["x","y"]` at every level against a server carrying neither guard. With `HAVING v LIKE '%'` on them that server stops at the first level, on the self-check that says a value marked valid has to read as a document. The two-condition form asks a third time, and the unconditional pair stays at the end, where it is the statement that asking once must not move either. Both defects date from the commit that added the two aggregates, `ba8e5e689c8` (2019-10-14), first released in 10.5.0. No recorded result in the suite moves, no test there having asked for either aggregate more than once in a statement, which is why neither has shown up. --- mysql-test/main/func_json_agg_reread.result | 123 ++++++++++++++++++++ mysql-test/main/func_json_agg_reread.test | 84 +++++++++++++ sql/item_jsonfunc.cc | 15 ++- sql/item_jsonfunc.h | 21 +++- 4 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 mysql-test/main/func_json_agg_reread.result create mode 100644 mysql-test/main/func_json_agg_reread.test diff --git a/mysql-test/main/func_json_agg_reread.result b/mysql-test/main/func_json_agg_reread.result new file mode 100644 index 0000000000000..37bdf79392a36 --- /dev/null +++ b/mysql-test/main/func_json_agg_reread.result @@ -0,0 +1,123 @@ +# +# Asking for the result of a JSON aggregate more than once. +# +# A group is built up across three calls: clear() at its start, add() +# once per row, and val_str() when the result is wanted. The text +# accumulates in a buffer belonging to the item, there being nowhere +# else to keep it between rows, so anything val_str() writes into that +# buffer stays written. +# +# Nothing says the result is wanted only once. HAVING on the alias is +# the shortest statement that wants it twice; the number of times +# follows the number of conditions. +# +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1, 'x'), (1, 'y'), (2, 'z'), (2, 'w'); +# +# 1. Asked once. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +v +{"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +["x","y","z","w"] +# +# 2. Asked twice. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%'; +v +{"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%'; +v +["x","y","z","w"] +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS valid FROM t1 HAVING valid IS NOT NULL; +valid +1 +# +# 3. Asked three times. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '{%'; +v +{"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '[%'; +v +["x","y","z","w"] +# +# 4. Every group, not only the first. +# +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; +g v +1 ["x","y"] +2 ["z","w"] +# +# 5. A group of one row, where no separator is ever written. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; +v +{"x":"x"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; +v +["x"] +# +# 6. A copy of the aggregate, which WITH ROLLUP makes without fixing +# the item again. +# +# The condition is what makes this section a test of anything: a +# rollup level asked for once is closed once whether the flag is +# kept or not, and every other section here reaches the second +# asking through HAVING. The extra level is produced by a copy of +# the item, so it is that copy's own flag being reset per group +# that has to hold. +# +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%'; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +NULL {"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%'; +g v +1 ["x","y"] +2 ["z","w"] +NULL ["x","y","z","w"] +# two conditions, so each level is asked for a third time +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%' AND v LIKE '{%'; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +NULL {"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%' AND v LIKE '[%'; +g v +1 ["x","y"] +2 ["z","w"] +NULL ["x","y","z","w"] +# and the levels without a condition, which must not move either +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +NULL {"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP; +g v +1 ["x","y"] +2 ["z","w"] +NULL ["x","y","z","w"] +# +# 7. No rows, which is a NULL and stays one however often it is asked +# for. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; +v +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; +v +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_reread.test b/mysql-test/main/func_json_agg_reread.test new file mode 100644 index 0000000000000..ac50057eea6d9 --- /dev/null +++ b/mysql-test/main/func_json_agg_reread.test @@ -0,0 +1,84 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Asking for the result of a JSON aggregate more than once. +--echo # +--echo # A group is built up across three calls: clear() at its start, add() +--echo # once per row, and val_str() when the result is wanted. The text +--echo # accumulates in a buffer belonging to the item, there being nowhere +--echo # else to keep it between rows, so anything val_str() writes into that +--echo # buffer stays written. +--echo # +--echo # Nothing says the result is wanted only once. HAVING on the alias is +--echo # the shortest statement that wants it twice; the number of times +--echo # follows the number of conditions. +--echo # + +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1, 'x'), (1, 'y'), (2, 'z'), (2, 'w'); + +--echo # +--echo # 1. Asked once. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; + +--echo # +--echo # 2. Asked twice. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%'; +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS valid FROM t1 HAVING valid IS NOT NULL; + +--echo # +--echo # 3. Asked three times. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '{%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '[%'; + +--echo # +--echo # 4. Every group, not only the first. +--echo # +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; + +--echo # +--echo # 5. A group of one row, where no separator is ever written. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; + +--echo # +--echo # 6. A copy of the aggregate, which WITH ROLLUP makes without fixing +--echo # the item again. +--echo # +--echo # The condition is what makes this section a test of anything: a +--echo # rollup level asked for once is closed once whether the flag is +--echo # kept or not, and every other section here reaches the second +--echo # asking through HAVING. The extra level is produced by a copy of +--echo # the item, so it is that copy's own flag being reset per group +--echo # that has to hold. +--echo # +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%'; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%'; +--echo # two conditions, so each level is asked for a third time +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%' AND v LIKE '{%'; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%' AND v LIKE '[%'; +--echo # and the levels without a condition, which must not move either +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP; + +--echo # +--echo # 7. No rows, which is a NULL and stays one however often it is asked +--echo # for. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; + +DROP TABLE t1; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index b6b37f8467c19..795b40f04c2a5 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -4896,6 +4896,7 @@ static const char json_arrayagg_name[]= "json_arrayagg"; void Item_func_json_arrayagg::clear() { m_bad_element= false; + m_closed= false; Item_func_group_concat::clear(); } @@ -5007,6 +5008,13 @@ String* Item_func_json_arrayagg::val_str(String *str) return NULL; } + /* + The brackets are already round what the parent returned, this + group having been asked for once before. + */ + if (m_closed) + return str; + /* The brackets are put on last, so a buffer that will not take them leaves behind something that reads as a value of its own rather @@ -5018,6 +5026,7 @@ String* Item_func_json_arrayagg::val_str(String *str) s.swap(*str); if (str->append(s) || str->append(']')) goto bad_result; + m_closed= true; } return str; @@ -5029,7 +5038,7 @@ String* Item_func_json_arrayagg::val_str(String *str) Item_func_json_objectagg:: Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item) - :Item_sum(thd, item), m_bad_pair(false) + :Item_sum(thd, item), m_bad_pair(false), m_closed(false) { quick_group= FALSE; result.set_charset(collation.collation); @@ -5112,6 +5121,7 @@ void Item_func_json_objectagg::clear() that lost one is refused whole rather than returned short. */ m_bad_pair= false; + m_closed= false; if (result.append('{')) m_bad_pair= true; } @@ -5180,11 +5190,12 @@ String* Item_func_json_objectagg::val_str(String* str) if (null_value) return 0; - if (m_bad_pair || result.append('}')) + if (m_bad_pair || (!m_closed && result.append('}'))) { null_value= 1; return 0; } + m_closed= true; return &result; } diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index c84471efa08ff..8c8440faf4697 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -784,6 +784,14 @@ class Item_func_json_arrayagg : public Item_func_group_concat clear(). */ bool m_bad_element; + /* + The brackets have been put on already. Nothing says how often the + result of a group is asked for, and the buffer they go into belongs + to this item and outlives the asking, so putting them on once per + call would put on one pair per call. Reset for each group by + clear(). + */ + bool m_closed; public: String m_tmp_json; /* Used in get_str_from_*.. */ Item_func_json_arrayagg(THD *thd, Name_resolution_context *context_arg, @@ -792,7 +800,7 @@ class Item_func_json_arrayagg : public Item_func_group_concat bool limit_clause, Item *row_limit, Item *offset_limit): Item_func_group_concat(thd, context_arg, is_distinct, is_select, is_order, is_separator, limit_clause, row_limit, offset_limit), - m_bad_element(false) + m_bad_element(false), m_closed(false) { } /* @@ -803,7 +811,7 @@ class Item_func_json_arrayagg : public Item_func_group_concat not JSON. */ Item_func_json_arrayagg(THD *thd, Item_func_json_arrayagg *item) : - Item_func_group_concat(thd, item), m_bad_element(false) + Item_func_group_concat(thd, item), m_bad_element(false), m_closed(false) { m_tmp_json.set_charset(collation.collation); } @@ -840,6 +848,13 @@ class Item_func_json_objectagg : public Item_sum clear(). */ bool m_bad_pair; + /* + The closing brace has been written already. Nothing says how often + the result of a group is asked for, and the buffer it goes into is + this item's own and outlives the asking, so writing it once per call + would write one brace per call. Reset for each group by clear(). + */ + bool m_closed; public: /* The opening brace is not written here. This runs while the @@ -849,7 +864,7 @@ class Item_func_json_objectagg : public Item_sum once per group and once the width is known. */ Item_func_json_objectagg(THD *thd, Item *key, Item *value) : - Item_sum(thd, key, value), m_bad_pair(false) + Item_sum(thd, key, value), m_bad_pair(false), m_closed(false) { quick_group= FALSE; } From a3c97b61afd74ea7dcd45ebe0fd9961b8006e878 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Wed, 12 Aug 2026 01:46:53 -0400 Subject: [PATCH 07/30] Compile Item_copy's assert-only member wherever the assert is `Item_copy::copied_in` says whether `copy()` has run, and twenty `DBUG_ASSERT`s across the copy classes read it before trusting the cached value. Nothing else reads it, and it is declared - and the three places that write it are guarded - `#ifndef DBUG_OFF`. That guard names the wrong set. `DBUG_ASSERT_AS_PRINTF` turns an assertion into a printed complaint with a stack trace instead of an abort, and it does so by setting `DBUG_OFF` while leaving the assertion's expression standing (`include/my_dbug.h`). So in that shape the twenty reads survive, the member does not, and the build stops on all twenty. The server otherwise supports the shape: `mysqld.cc` installs `mariadb_dbug_assert_failed` for it. `DBUG_ASSERT_EXISTS` is defined in both branches that leave an assertion's expression to be compiled, so it names exactly the set that reads this member. The five sites - the declaration, the clearing in `Item_copy::Item_copy()`, and the setting in `Item_copy_string::copy()`, `Item_copy_real::copy()` and `Item_copy_timestamp::copy()` - move to it together, since a member kept without the writes that maintain it would be read unset. No shipped binary changes: `DBUG_ASSERT_EXISTS` is defined in every build that defines `DBUG_OFF` alone, and a build defining neither is a debug build, which already had all five. --- sql/item.cc | 4 ++-- sql/item.h | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sql/item.cc b/sql/item.cc index b3fbf6b72e876..a0046ec8c7f95 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -5351,7 +5351,7 @@ void Item_copy_string::copy() if (res && res != &str_value) str_value.copy(*res); null_value=item->null_value; -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS copied_in= 1; #endif } @@ -5386,7 +5386,7 @@ void Item_copy_real::copy() { cached_value= item->val_real(); null_value= item->null_value; -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS copied_in= 1; #endif } diff --git a/sql/item.h b/sql/item.h index e8ea7d205ae9b..5e8f0d710ba2e 100644 --- a/sql/item.h +++ b/sql/item.h @@ -6804,12 +6804,20 @@ class Item_copy :public Item, Type_std_attributes::set(item); name= item->name; set_handler(item->type_handler()); -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS copied_in= 0; #endif } -#ifndef DBUG_OFF + /* + Read by a DBUG_ASSERT and by nothing else, so it is compiled wherever + a DBUG_ASSERT is rather than wherever a debug build is. Those are not + the same set: DBUG_ASSERT_AS_PRINTF turns the assertion into a printed + complaint and leaves its expression standing, while setting DBUG_OFF, + so a guard written the other way round leaves that expression with + nothing to read and the build stops compiling. + */ +#ifdef DBUG_ASSERT_EXISTS bool copied_in; #endif @@ -6971,7 +6979,7 @@ class Item_copy_timestamp: public Item_copy null_value= tmp.is_null(); m_value= tmp.is_null() ? Timestamp_or_zero_datetime() : Timestamp_or_zero_datetime(tmp); -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS copied_in=1; #endif } From 6c14f4ec2e14a00d47835c4e3a5677364aa4cb07 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Tue, 4 Aug 2026 00:00:13 -0400 Subject: [PATCH 08/30] Let a value carry what is known about it, instead of finding out again Every JSON function that is handed a document reads it to find out whether it is one, and reads it again if it is spliced into something else. The function that produced it knew the answer and had nowhere to put it: `val_str()` returns a pointer, a length and a character set, and there is no room in that for anything else. This adds the room. Nothing reads it yet - no scan is skipped here and no answer moves - so what this commit is, on its own, is a set of promises and a way of catching one that is not kept. That way of catching is two functions read by a `DBUG_ASSERT` and by nothing else, so they are compiled wherever a `DBUG_ASSERT` is rather than wherever a debug build is. Those are not the same set: `DBUG_ASSERT_AS_PRINTF` turns the assertion into a printed complaint and leaves its expression standing, while setting `DBUG_OFF`, so a guard written the other way round leaves that expression with nothing to call and the build stops compiling. **The two questions** `Item::is_valid_json()` asks whether the value just returned is a document. `Item::is_nice_json()` asks whether it is formatted the way `json_nice()` writes one in its loose form, which is the formatting these functions produce. Both are about the last evaluation and stand until the item is evaluated again. `Item::is_valid_json_static()` asks instead whether *every* evaluation gives back a document or nothing at all - a property of the class, so it can be asked before anything has run, which is what a caller working at create time needs. All three answer FALSE by default, and FALSE does not say the value is not a document. It says nothing attests to it, so a caller who needs to know must read it - which is what every caller does today. The asymmetry is the whole design: answering TRUE for a value that is not guaranteed is the one answer that must never be given, while answering FALSE for one that is costs a reading and nothing else. Where the cheap answer was not obviously safe, FALSE was chosen. **Catching a promise that is not kept** The marks appear in no result, so a broken one would go unnoticed until something acted on it. They are therefore not two bare bools but a `Json_result_marks`, whose setter is given the value being attested, and in a debug build reads it back: DBUG_ASSERT(!valid || json_value_reads_as_document(str)); DBUG_ASSERT(!nice || json_value_is_nice(str)); Whether a value reads as a document is the question `json_valid()` is, so the first of those asks it rather than asking it again in a loop of its own. The second runs `json_nice()` in its loose form and compares the two runs of bytes with the one function there is for comparing two. This turns every JSON statement in the existing suite into a detector for an overclaim, and it earned that during development: it caught a character set in which a document cannot be written at all, it caught `JSON_ARRAY` calling a value nicely written that was not a document, and it caught the path below that is written in a wide set and yet carries one part of itself in single bytes. Each was proven to fire by deliberately introducing a false claim and watching the server stop. The class lives in `sql/item.h` rather than with the JSON functions because a class that is not one of them can carry a value through - see `Item_func_conv_charset` below - and going through this member is what puts such a class under that check. A copy of it says nothing: a copy is a new item that has not been evaluated, and what it was copied from speaks about bytes that it still owns and may write over. **What each function says** Six editing functions - `JSON_INSERT`/`SET`/`REPLACE`, `JSON_REMOVE`, `JSON_ARRAY_APPEND`, `JSON_ARRAY_INSERT`, `JSON_MERGE` and `JSON_MERGE_PATCH` - answer the static question, their epilogue reading the whole result before returning it. So do `JSON_EXTRACT`, `JSON_QUERY`, `JSON_SEARCH` and `JSON_NORMALIZE`, each of which returns a span cut from something already read through. Of those, `JSON_NORMALIZE` is not nicely written - it emits the compact form - and neither is `JSON_QUERY`, which keeps the spacing it was given. `JSON_ARRAY`, `JSON_OBJECT` and the two aggregates build their result rather than reading it, so they answer per evaluation, from what they learned putting it together. A `Json_splice_marks` is carried through the splice helpers for that: it starts out saying both things and is only ever cleared, a conjunction over the values that went in, since an array is a document only if every element was. A value that did not parse, one holding a character that cannot be written, and one that came in by a route with no opinion about formatting each clear what they have to. Niceness is asked of the item rather than measured. An aggregate says nothing about a group cut to `group_concat_max_len`. The cut lands on whatever byte the limit falls on, so what stands between the brackets can be half an element - or, where the cut is just past an opening quote and that quote is written back to close the string, an element no row of the group ever held, which reads as data and is not any. Nor does it say the group is nicely written: what goes between the elements is the separator inherited from `GROUP_CONCAT`, a comma with nothing after it where the loose formatting puts a space as well, and there is no asking for another one - the grammar takes no `SEPARATOR` in this production and writes that comma itself. `JSON_FORMAT` forwards: its `val_str()` answers `(true, fmt == LOOSE)`, and its `val_json()` answers exactly what its argument answers. Everything else says nothing, `JSON_KEYS` and `JSON_QUOTE` included. **Values that arrive by some other route** `Item_func_conv_charset` is not a JSON function but carries one's result through. A conversion that substituted nothing and stopped on nothing kept the sequence of characters it was given, and JSON is a grammar over characters, so a document that went in comes out one, written the same way. It therefore passes the argument's answer on. Conversion to `binary` is refused: it is not a conversion but a relabelling, keeping the bytes and making each one a character of its own, so a document two bytes to the character comes out as twice as many characters and is not one. The copier cannot report that, `binary` having no byte it rejects. `Item_direct_ref` forwards all three, its `val_str()` returning what the referenced item returned byte for byte. `Item_ref` does not: it reads through `str_result()`, which for a field is a copy sitting in a record buffer rather than the value the producing item made. `Item_aggregate_ref` forwards for a reason of its own, given at the site. `Item_direct_view_ref` declines on a row an outer join filled in with NULLs, where its `val_XXX` return without evaluating the referenced item at all and what that item can say is about some earlier row. **A document cannot be written in every character set** A function that writes its own punctuation writes it into the result's character set, and `String::append(char)` puts a raw byte where a character can be a single byte. In swe7 the bytes that encode brackets and braces elsewhere encode national letters, so what is written is not a document and cannot be read as one - which is what the server has always returned, and is not for this to change. What it is for is to keep that result from being taken for a document later, so `is_json_compatible_charset()` gates the claim wherever punctuation is written without the result being read back. It admits ucs2, utf16 and utf32, which are wide enough that `String::append()` converts and the bracket goes down as a bracket. That the set can encode a document is not the same as the value having been written in it, and `JSON_SEARCH` is where the two part company. It builds a path out of punctuation it writes, keys it copies from the document, and the index of an array step. The punctuation goes in through `String::append()`, which converts it, so in `ucs2` a bracket is `005B`; the index does not go that way, `String::append_ulonglong()` writing the digits as themselves whatever the set. One stray byte leaves every character after it read from the wrong place - a path 39 bytes long where every character is meant to be two or four. So the path says for itself whether it was written, and one that was not is passed on as the bytes it is rather than as a document. Both shapes in one statement, over a table: id spliced 1 0 -- the path carries an index 2 1 -- the path is keys all the way down Released 10.11 answers 0 to both, refusing the second wrongly; that half is this branch's. Two constants, `json_loose_comma` and `json_loose_colon`, are hoisted to file scope and used both by `json_nice()` and by the functions that write that formatting themselves, so a claim of niceness rests on a shared fact rather than on two pieces of code happening to agree. `JSON_SEARCH` writes the separator between several paths from that constant for the same reason: it claims the loose formatting over a result it never reads back, so the claim has to rest on the one fact that formatting is defined by. With the constant temporarily made comma-tab, `JSON_SEARCH('{"a":"x","b":"x"}', 'all', 'x')` answers its two paths separated by a comma and a tab; with the separator written out by hand instead, a debug build stops on the niceness half of the self-check. **Nothing moves** No existing recorded result changes, which is this commit's gate and is checkable with `git status`. The new `func_json_marks` proves the same for statements that did not exist before: it was recorded on this build, then `sql/` was restored to the previous commit, rebuilt, and re-recorded, and the two files are identical byte for byte. That test walks every function past the point where it makes a promise, in a build that reads the value back. Its sections are: building from good values; building from a column holding text that is not a document; the six editing functions; the reading functions; the two aggregates, including a group cut to fit the length limit, `WITH ROLLUP`, and a broken column; character sets, including swe7, sjis and the three wide ones; values arriving through a conversion, including the binary relabelling; values arriving through a reference, including an outer join over a merged view; and items that are copied, by condition pushdown and by copying a table definition that holds a JSON expression. The cut group recorded there is `["a","b",""]`, which is not half of anything: the limit fell just past an opening quote and the quote written back to close the string left a third element that no row of the group ever held - valid JSON, so nothing downstream has any reason to reject it, and one row of it invented. Warning 1260 says a row was cut, not that one was manufactured. The group built out of a broken column alongside it is complete too, and is not a document at all. `func_json_search_wide_charset` carries the path shapes: two, four and one byte to the character, an index path and a keys-only path, read back and spliced into `JSON_ARRAY` and `JSON_MERGE_PRESERVE`, and out of a table so no one statement settles another. --- mysql-test/main/func_json_marks.result | 377 ++++++++++++++ mysql-test/main/func_json_marks.test | 246 ++++++++++ .../main/func_json_search_wide_charset.result | 118 +++++ .../main/func_json_search_wide_charset.test | 91 ++++ sql/item.cc | 11 + sql/item.h | 123 ++++- sql/item_jsonfunc.cc | 459 ++++++++++++++++-- sql/item_jsonfunc.h | 87 +++- sql/item_strfunc.cc | 20 +- sql/item_strfunc.h | 26 + 10 files changed, 1497 insertions(+), 61 deletions(-) create mode 100644 mysql-test/main/func_json_marks.result create mode 100644 mysql-test/main/func_json_marks.test create mode 100644 mysql-test/main/func_json_search_wide_charset.result create mode 100644 mysql-test/main/func_json_search_wide_charset.test diff --git a/mysql-test/main/func_json_marks.result b/mysql-test/main/func_json_marks.result new file mode 100644 index 0000000000000..a645c3b5e517a --- /dev/null +++ b/mysql-test/main/func_json_marks.result @@ -0,0 +1,377 @@ +# +# 1. Building a document out of values answering is_valid +# +SELECT JSON_ARRAY() AS empty_array, +JSON_OBJECT() AS empty_object; +empty_array empty_object +[] {} +SELECT JSON_ARRAY(1, 'two', NULL, TRUE, 3.5) AS scalars; +scalars +[1, "two", null, true, 3.5] +SELECT JSON_OBJECT('a', 1, 'b', 'two', 'c', NULL) AS pairs; +pairs +{"a": 1, "b": "two", "c": null} +SELECT JSON_ARRAY(JSON_ARRAY(1, 2), JSON_OBJECT('a', 1)) AS nested; +nested +[[1, 2], {"a": 1}] +SELECT JSON_OBJECT('k', JSON_ARRAY(1, 2)) AS nested_in_object; +nested_in_object +{"k": [1, 2]} +SELECT JSON_ARRAY(JSON_SET('{"a": 1}', '$.b', 2)) AS edited_then_built; +edited_then_built +[{"a": 1, "b": 2}] +SELECT JSON_ARRAY(JSON_EXTRACT('{"a": [1, 2]}', '$.a')) AS extracted_then_built; +extracted_then_built +[[1, 2]] +# +# 2. Building a document out of values that cannot be +# +# A JSON column holds whatever was in it when the check was off, +# and nothing between the column and here reads it. The value +# goes in as it stands, as it always has, and the note is what +# says so. +# +CREATE TABLE tj (id INT, j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tj VALUES (1, '{"a": 1}'), (2, '{"a":1,'), (3, ''), (4, '{"a":1}'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 1)) AS from_good; +from_good +[{"a": 1}] +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 2)) AS from_broken; +from_broken +[{"a":1,] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 3)) AS from_empty; +from_empty +[] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT JSON_OBJECT('k', (SELECT j FROM tj WHERE id = 2)) AS from_broken_object; +from_broken_object +{"k": {"a":1,} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +# +# The compact one reads perfectly well and is simply not +# written the way this function writes. +# +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 4)) AS from_compact; +from_compact +[{"a":1}] +# +# 3. Editing a document +# +SELECT JSON_SET('{"a": 1}', '$.b', 2) AS set_done, +JSON_INSERT('{"a": 1}', '$.b', 2) AS insert_done, +JSON_REPLACE('{"a": 1}', '$.a', 2) AS replace_done; +set_done insert_done replace_done +{"a": 1, "b": 2} {"a": 1, "b": 2} {"a": 2} +SELECT JSON_REMOVE('{"a": 1, "b": 2}', '$.a') AS removed, +JSON_ARRAY_APPEND('[1]', '$', 2) AS appended, +JSON_ARRAY_INSERT('[1, 3]', '$[1]', 2) AS inserted; +removed appended inserted +{"b": 2} [1, 2] [1, 2, 3] +SELECT JSON_MERGE('[1]', '[2]') AS merged, +JSON_MERGE_PATCH('{"a": 1}', '{"b": 2}') AS patched; +merged patched +[1, 2] {"a": 1, "b": 2} +SELECT JSON_SET((SELECT j FROM tj WHERE id = 2), '$.b', 2) AS set_over_broken; +set_over_broken +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT JSON_SET('{"a": 1}', '$.b', (SELECT j FROM tj WHERE id = 2)) +AS set_with_broken; +set_with_broken +NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 21 +# +# 4. Reading a document out of another one +# +SELECT JSON_EXTRACT('{"a": [1, 2], "b": 3}', '$.a') AS extracted, +JSON_EXTRACT('{"a": 1, "b": 2}', '$.a', '$.b') AS extracted_two; +extracted extracted_two +[1, 2] [1, 2] +SELECT JSON_QUERY('{"a": [1,2]}', '$.a') AS queried; +queried +[1,2] +SELECT JSON_KEYS('{"a": 1, "b": 2}') AS keys_of; +keys_of +["a", "b"] +SELECT JSON_SEARCH('{"a": "x", "b": "x"}', 'all', 'x') AS searched_all, +JSON_SEARCH('{"a": "x"}', 'one', 'x') AS searched_one; +searched_all searched_one +["$.a", "$.b"] "$.a" +SELECT JSON_NORMALIZE('{"b": 1, "a": 2}') AS normalized; +normalized +{"a":2.0E0,"b":1.0E0} +SELECT JSON_COMPACT('{"a": [1, 2]}') AS compacted, +JSON_LOOSE('{"a":[1,2]}') AS loosened; +compacted loosened +{"a":[1,2]} {"a": [1, 2]} +SELECT JSON_DETAILED('{"a": 1}') AS detailed; +detailed +{ + "a": 1 +} +# +# JSON_VALID reads its argument through val_json(), where the +# format functions pass the value straight through without touching +# it, so nothing here is written out afresh. +# +SELECT JSON_VALID(JSON_LOOSE('{"a":1}')) AS loose_forwarded, +JSON_VALID(JSON_COMPACT('{"a": 1}')) AS compact_forwarded, +JSON_VALID(JSON_LOOSE((SELECT j FROM tj WHERE id = 2))) +AS broken_forwarded; +loose_forwarded compact_forwarded broken_forwarded +1 1 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# 5. Building a document a row at a time +# +CREATE TABLE tg (g INT, v INT, s VARCHAR(16)); +INSERT INTO tg VALUES (1, 1, 'a'), (1, 2, 'b'), (2, 3, 'c'), (2, 4, 'd'); +SELECT g, JSON_ARRAYAGG(v) AS arr, JSON_OBJECTAGG(s, v) AS obj +FROM tg GROUP BY g ORDER BY g; +g arr obj +1 [1,2] {"a":1, "b":2} +2 [3,4] {"c":3, "d":4} +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS agg_then_built FROM tg; +agg_then_built +[[1,2,3,4]] +SELECT g, JSON_ARRAYAGG(v) AS arr FROM tg GROUP BY g WITH ROLLUP; +g arr +1 [1,2] +2 [3,4] +NULL [1,2,3,4] +# +# A group cut to fit group_concat_max_len is cut at the byte +# the limit falls on. Here that lands just past an opening +# quote, and the quote written back to close the string leaves +# an element no row of the group held - a whole document, and +# one row of it invented. A group built out of a broken column +# is complete too, and is not a document at all. +# +SET @save_len = @@group_concat_max_len; +SET SESSION group_concat_max_len = 10; +SELECT JSON_ARRAYAGG(s) AS cut FROM tg; +cut +["a","b",""] +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = @save_len; +SELECT JSON_ARRAYAGG(j) AS agg_broken FROM tj; +agg_broken +[{"a": 1},{"a":1,,,{"a":1}] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +SELECT JSON_OBJECTAGG(id, j) AS objagg_broken FROM tj; +objagg_broken +{"1":{"a": 1}, "2":{"a":1,, "3":, "4":{"a":1}} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +# +# 6. Character sets +# +# In swe7 the bytes that encode brackets and braces elsewhere +# encode national letters, so a document cannot be written in it +# at all. The bytes returned are the same ones as always. +# +SELECT HEX(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7, +JSON_VALID(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7_valid; +array_in_swe7 array_in_swe7_valid +5B2261225D 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT HEX(JSON_ARRAYAGG(CONVERT(s USING swe7))) AS agg_in_swe7 FROM tg; +agg_in_swe7 +5B2261222C2262222C2263222C2264225D +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) AS array_in_ucs2, +JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) +AS array_in_ucs2_valid; +array_in_ucs2 array_in_ucs2_valid +005B002200E90022005D 1 +SELECT JSON_VALID(JSON_OBJECT('k', CONVERT(_utf8mb4 0xC3A8 USING utf16))) +AS object_in_utf16_valid; +object_in_utf16_valid +1 +SELECT HEX(JSON_ARRAY(_binary 0x61)) AS array_in_binary; +array_in_binary +5B2261225D +# +# A character set whose characters are never a single byte +# encodes the punctuation properly, so a document can be written +# in it and these results really are documents. sjis is not +# such a set: it stores ASCII a byte at a time and puts a yen +# sign where the backslash belongs. +# +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32)) ) AS array_in_utf32, +JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32))) +AS array_in_utf32_valid; +array_in_utf32 array_in_utf32_valid +0000005B00000022000000E9000000220000005D 1 +SELECT JSON_VALID(JSON_ARRAY(CONVERT(s USING ucs2))) AS array_of_col_ucs2 +FROM tg ORDER BY v; +array_of_col_ucs2 +1 +1 +1 +1 +SELECT JSON_VALID(JSON_OBJECT(CONVERT(s USING utf16), v)) AS object_in_utf16 +FROM tg ORDER BY v; +object_in_utf16 +1 +1 +1 +1 +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(s USING ucs2))) AS agg_in_ucs2 +FROM tg GROUP BY g ORDER BY g; +agg_in_ucs2 +1 +1 +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(s USING utf16), v)) AS objectagg_utf16 +FROM tg GROUP BY g ORDER BY g; +objectagg_utf16 +1 +1 +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) AS backslash_in_sjis, +JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) +AS backslash_in_sjis_valid; +backslash_in_sjis backslash_in_sjis_valid +5B22815F815F225D 1 +# +# 7. Values that arrive through a conversion +# +# A conversion that lost nothing kept the characters it was +# given; one that had to put a question mark in its place did +# not, and what it produced is not the value that went in. +# +SELECT JSON_VALID(CONVERT(JSON_OBJECT('a', 1) USING ucs2)) AS converted_valid; +converted_valid +1 +SELECT JSON_ARRAY(CONVERT(JSON_OBJECT('a', 1) USING utf8mb4)) AS converted_built; +converted_built +[{"a": 1}] +SELECT HEX(CONVERT(JSON_ARRAY(_utf8mb4 0xC3A9) USING swe7)) AS converted_lossy; +converted_lossy +3F2260223F +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x5B to 'swe7' +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), '$.b', +_utf8mb4 0xC3A9)) AS mixed_charsets, +JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), +'$.b', _utf8mb4 0xC3A9)) AS mixed_charsets_valid; +mixed_charsets mixed_charsets_valid +7B2261223A20312C202262223A2022E9227D 1 +# +# Conversion to binary keeps the bytes and relabels them, so a +# document written two bytes to the character comes out as twice +# as many characters, and is no longer one. Nothing is lost, so +# the copier has nothing to report about it. +# +SELECT HEX(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) +AS relabelled_bytes, +JSON_VALID(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) +AS relabelled_valid; +relabelled_bytes relabelled_valid +005B0031005D 0 +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +SELECT JSON_ARRAY(CONVERT(CONVERT(JSON_OBJECT('a', 1) USING ucs2) +USING binary)) AS built_from_relabelled; +built_from_relabelled +[{"a": 1}] +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +# +# 8. Values that arrive through a reference +# +SELECT JSON_ARRAY(x) AS through_derived +FROM (SELECT JSON_OBJECT('a', 1) AS x) d; +through_derived +[{"a": 1}] +SELECT JSON_ARRAY(x) AS through_broken_derived +FROM (SELECT j AS x FROM tj WHERE id = 2) d; +through_broken_derived +[{"a":1,] +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +CREATE VIEW v1 AS SELECT JSON_OBJECT('a', 1) AS x; +SELECT JSON_ARRAY(x) AS through_view FROM v1; +through_view +[{"a": 1}] +DROP VIEW v1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS through_aggregate_ref +FROM tg GROUP BY g ORDER BY g; +through_aggregate_ref +[[1,2]] +[[3,4]] +# +# A reference into a merged view on the inner side of an outer +# join returns NULL for a row the join filled in, without asking +# the item it refers to. The second row below is such a row and +# follows one that matched, so anything remembered from the +# first would still be there. +# +CREATE TABLE touter (id INT); +CREATE TABLE tinner (id INT); +INSERT INTO touter VALUES (1),(2); +INSERT INTO tinner VALUES (1); +CREATE VIEW vinner AS SELECT id, JSON_OBJECT('a', id) AS x FROM tinner; +SELECT touter.id, JSON_ARRAY(vinner.x) AS through_outer_join_view +FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +id through_outer_join_view +1 [{"a": 1}] +2 [null] +SELECT touter.id, JSON_VALID(vinner.x) AS through_outer_join_view_valid +FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +id through_outer_join_view_valid +1 1 +2 NULL +DROP VIEW vinner; +DROP TABLE tinner; +DROP TABLE touter; +DROP TABLE tg; +DROP TABLE tj; +# +# 9. Items that are copied +# +# Pushing a condition into a derived table copies the expression +# that produces the column, leaving two items in two places of +# the plan. A copy has produced no value of its own yet. +# +CREATE TABLE tc (a INT, b VARCHAR(32)); +INSERT INTO tc VALUES (1, 'x'), (2, 'y'); +SELECT * FROM (SELECT a, JSON_ARRAY(1, 2) AS j FROM tc) d WHERE d.j LIKE '[%'; +a j +1 [1, 2] +2 [1, 2] +SELECT * FROM (SELECT a, JSON_ARRAY(a, b) AS j FROM tc) d WHERE d.j LIKE '[%'; +a j +1 [1, "x"] +2 [2, "y"] +SELECT * FROM (SELECT a, JSON_OBJECT('k', a) AS j FROM tc) d +WHERE d.j = '{"k": 1}'; +a j +1 {"k": 1} +# +# A table definition holding a JSON expression is copied whole +# when the table is altered. +# +CREATE TABLE tv (a INT, j VARCHAR(64) AS (JSON_ARRAY(a)) VIRTUAL, +CHECK (JSON_VALID(JSON_OBJECT('a', a)))); +INSERT INTO tv (a) VALUES (1), (2); +ALTER TABLE tv ADD COLUMN c INT; +SELECT * FROM tv ORDER BY a; +a j c +1 [1] NULL +2 [2] NULL +DROP TABLE tv; +DROP TABLE tc; diff --git a/mysql-test/main/func_json_marks.test b/mysql-test/main/func_json_marks.test new file mode 100644 index 0000000000000..b4398c8fdc6d4 --- /dev/null +++ b/mysql-test/main/func_json_marks.test @@ -0,0 +1,246 @@ +# +# What a JSON function is able to say about the value it has just +# produced is recorded on the function and appears in no result, so +# there is nothing in this file for a .result to assert about it +# directly. What the file is for is to walk every one of those +# functions past the point where it says something, in a build that +# reads the value back and stops the server if the two disagree. +# +# The recorded output is the other half of it, and the stricter half: +# none of it may move. Nothing here is a new answer - every statement +# below is answered exactly as it was before any of this was written. +# +# Values that are not ASCII are written as hex literals so that this +# file does not depend on its own encoding. The ones used below are +# C3A9 = e with an acute accent, and C3A8 = e with a grave accent. +# + +--echo # +--echo # 1. Building a document out of values answering is_valid +--echo # +SELECT JSON_ARRAY() AS empty_array, + JSON_OBJECT() AS empty_object; +SELECT JSON_ARRAY(1, 'two', NULL, TRUE, 3.5) AS scalars; +SELECT JSON_OBJECT('a', 1, 'b', 'two', 'c', NULL) AS pairs; +SELECT JSON_ARRAY(JSON_ARRAY(1, 2), JSON_OBJECT('a', 1)) AS nested; +SELECT JSON_OBJECT('k', JSON_ARRAY(1, 2)) AS nested_in_object; +SELECT JSON_ARRAY(JSON_SET('{"a": 1}', '$.b', 2)) AS edited_then_built; +SELECT JSON_ARRAY(JSON_EXTRACT('{"a": [1, 2]}', '$.a')) AS extracted_then_built; + +--echo # +--echo # 2. Building a document out of values that cannot be +--echo # +--echo # A JSON column holds whatever was in it when the check was off, +--echo # and nothing between the column and here reads it. The value +--echo # goes in as it stands, as it always has, and the note is what +--echo # says so. +--echo # +CREATE TABLE tj (id INT, j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tj VALUES (1, '{"a": 1}'), (2, '{"a":1,'), (3, ''), (4, '{"a":1}'); +SET SESSION check_constraint_checks = ON; + +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 1)) AS from_good; +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 2)) AS from_broken; +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 3)) AS from_empty; +SELECT JSON_OBJECT('k', (SELECT j FROM tj WHERE id = 2)) AS from_broken_object; + +--echo # +--echo # The compact one reads perfectly well and is simply not +--echo # written the way this function writes. +--echo # +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 4)) AS from_compact; + +--echo # +--echo # 3. Editing a document +--echo # +SELECT JSON_SET('{"a": 1}', '$.b', 2) AS set_done, + JSON_INSERT('{"a": 1}', '$.b', 2) AS insert_done, + JSON_REPLACE('{"a": 1}', '$.a', 2) AS replace_done; +SELECT JSON_REMOVE('{"a": 1, "b": 2}', '$.a') AS removed, + JSON_ARRAY_APPEND('[1]', '$', 2) AS appended, + JSON_ARRAY_INSERT('[1, 3]', '$[1]', 2) AS inserted; +SELECT JSON_MERGE('[1]', '[2]') AS merged, + JSON_MERGE_PATCH('{"a": 1}', '{"b": 2}') AS patched; +SELECT JSON_SET((SELECT j FROM tj WHERE id = 2), '$.b', 2) AS set_over_broken; +SELECT JSON_SET('{"a": 1}', '$.b', (SELECT j FROM tj WHERE id = 2)) + AS set_with_broken; + +--echo # +--echo # 4. Reading a document out of another one +--echo # +SELECT JSON_EXTRACT('{"a": [1, 2], "b": 3}', '$.a') AS extracted, + JSON_EXTRACT('{"a": 1, "b": 2}', '$.a', '$.b') AS extracted_two; +SELECT JSON_QUERY('{"a": [1,2]}', '$.a') AS queried; +SELECT JSON_KEYS('{"a": 1, "b": 2}') AS keys_of; +SELECT JSON_SEARCH('{"a": "x", "b": "x"}', 'all', 'x') AS searched_all, + JSON_SEARCH('{"a": "x"}', 'one', 'x') AS searched_one; +SELECT JSON_NORMALIZE('{"b": 1, "a": 2}') AS normalized; +SELECT JSON_COMPACT('{"a": [1, 2]}') AS compacted, + JSON_LOOSE('{"a":[1,2]}') AS loosened; +SELECT JSON_DETAILED('{"a": 1}') AS detailed; + +--echo # +--echo # JSON_VALID reads its argument through val_json(), where the +--echo # format functions pass the value straight through without touching +--echo # it, so nothing here is written out afresh. +--echo # +SELECT JSON_VALID(JSON_LOOSE('{"a":1}')) AS loose_forwarded, + JSON_VALID(JSON_COMPACT('{"a": 1}')) AS compact_forwarded, + JSON_VALID(JSON_LOOSE((SELECT j FROM tj WHERE id = 2))) + AS broken_forwarded; + +--echo # +--echo # 5. Building a document a row at a time +--echo # +CREATE TABLE tg (g INT, v INT, s VARCHAR(16)); +INSERT INTO tg VALUES (1, 1, 'a'), (1, 2, 'b'), (2, 3, 'c'), (2, 4, 'd'); +SELECT g, JSON_ARRAYAGG(v) AS arr, JSON_OBJECTAGG(s, v) AS obj + FROM tg GROUP BY g ORDER BY g; +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS agg_then_built FROM tg; +SELECT g, JSON_ARRAYAGG(v) AS arr FROM tg GROUP BY g WITH ROLLUP; + +--echo # +--echo # A group cut to fit group_concat_max_len is cut at the byte +--echo # the limit falls on. Here that lands just past an opening +--echo # quote, and the quote written back to close the string leaves +--echo # an element no row of the group held - a whole document, and +--echo # one row of it invented. A group built out of a broken column +--echo # is complete too, and is not a document at all. +--echo # +SET @save_len = @@group_concat_max_len; +SET SESSION group_concat_max_len = 10; +SELECT JSON_ARRAYAGG(s) AS cut FROM tg; +SET SESSION group_concat_max_len = @save_len; +SELECT JSON_ARRAYAGG(j) AS agg_broken FROM tj; +SELECT JSON_OBJECTAGG(id, j) AS objagg_broken FROM tj; + +--echo # +--echo # 6. Character sets +--echo # +--echo # In swe7 the bytes that encode brackets and braces elsewhere +--echo # encode national letters, so a document cannot be written in it +--echo # at all. The bytes returned are the same ones as always. +--echo # +SELECT HEX(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7, + JSON_VALID(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7_valid; +SELECT HEX(JSON_ARRAYAGG(CONVERT(s USING swe7))) AS agg_in_swe7 FROM tg; +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) AS array_in_ucs2, + JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) + AS array_in_ucs2_valid; +SELECT JSON_VALID(JSON_OBJECT('k', CONVERT(_utf8mb4 0xC3A8 USING utf16))) + AS object_in_utf16_valid; +SELECT HEX(JSON_ARRAY(_binary 0x61)) AS array_in_binary; + +--echo # +--echo # A character set whose characters are never a single byte +--echo # encodes the punctuation properly, so a document can be written +--echo # in it and these results really are documents. sjis is not +--echo # such a set: it stores ASCII a byte at a time and puts a yen +--echo # sign where the backslash belongs. +--echo # +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32)) ) AS array_in_utf32, + JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32))) + AS array_in_utf32_valid; +SELECT JSON_VALID(JSON_ARRAY(CONVERT(s USING ucs2))) AS array_of_col_ucs2 + FROM tg ORDER BY v; +SELECT JSON_VALID(JSON_OBJECT(CONVERT(s USING utf16), v)) AS object_in_utf16 + FROM tg ORDER BY v; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(s USING ucs2))) AS agg_in_ucs2 + FROM tg GROUP BY g ORDER BY g; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(s USING utf16), v)) AS objectagg_utf16 + FROM tg GROUP BY g ORDER BY g; +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) AS backslash_in_sjis, + JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) + AS backslash_in_sjis_valid; + +--echo # +--echo # 7. Values that arrive through a conversion +--echo # +--echo # A conversion that lost nothing kept the characters it was +--echo # given; one that had to put a question mark in its place did +--echo # not, and what it produced is not the value that went in. +--echo # +SELECT JSON_VALID(CONVERT(JSON_OBJECT('a', 1) USING ucs2)) AS converted_valid; +SELECT JSON_ARRAY(CONVERT(JSON_OBJECT('a', 1) USING utf8mb4)) AS converted_built; +SELECT HEX(CONVERT(JSON_ARRAY(_utf8mb4 0xC3A9) USING swe7)) AS converted_lossy; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), '$.b', + _utf8mb4 0xC3A9)) AS mixed_charsets, + JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), + '$.b', _utf8mb4 0xC3A9)) AS mixed_charsets_valid; + +--echo # +--echo # Conversion to binary keeps the bytes and relabels them, so a +--echo # document written two bytes to the character comes out as twice +--echo # as many characters, and is no longer one. Nothing is lost, so +--echo # the copier has nothing to report about it. +--echo # +SELECT HEX(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) + AS relabelled_bytes, + JSON_VALID(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) + AS relabelled_valid; +SELECT JSON_ARRAY(CONVERT(CONVERT(JSON_OBJECT('a', 1) USING ucs2) + USING binary)) AS built_from_relabelled; + +--echo # +--echo # 8. Values that arrive through a reference +--echo # +SELECT JSON_ARRAY(x) AS through_derived + FROM (SELECT JSON_OBJECT('a', 1) AS x) d; +SELECT JSON_ARRAY(x) AS through_broken_derived + FROM (SELECT j AS x FROM tj WHERE id = 2) d; +CREATE VIEW v1 AS SELECT JSON_OBJECT('a', 1) AS x; +SELECT JSON_ARRAY(x) AS through_view FROM v1; +DROP VIEW v1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS through_aggregate_ref + FROM tg GROUP BY g ORDER BY g; + +--echo # +--echo # A reference into a merged view on the inner side of an outer +--echo # join returns NULL for a row the join filled in, without asking +--echo # the item it refers to. The second row below is such a row and +--echo # follows one that matched, so anything remembered from the +--echo # first would still be there. +--echo # +CREATE TABLE touter (id INT); +CREATE TABLE tinner (id INT); +INSERT INTO touter VALUES (1),(2); +INSERT INTO tinner VALUES (1); +CREATE VIEW vinner AS SELECT id, JSON_OBJECT('a', id) AS x FROM tinner; +SELECT touter.id, JSON_ARRAY(vinner.x) AS through_outer_join_view + FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +SELECT touter.id, JSON_VALID(vinner.x) AS through_outer_join_view_valid + FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +DROP VIEW vinner; +DROP TABLE tinner; +DROP TABLE touter; + +DROP TABLE tg; +DROP TABLE tj; + +--echo # +--echo # 9. Items that are copied +--echo # +--echo # Pushing a condition into a derived table copies the expression +--echo # that produces the column, leaving two items in two places of +--echo # the plan. A copy has produced no value of its own yet. +--echo # +CREATE TABLE tc (a INT, b VARCHAR(32)); +INSERT INTO tc VALUES (1, 'x'), (2, 'y'); +SELECT * FROM (SELECT a, JSON_ARRAY(1, 2) AS j FROM tc) d WHERE d.j LIKE '[%'; +SELECT * FROM (SELECT a, JSON_ARRAY(a, b) AS j FROM tc) d WHERE d.j LIKE '[%'; +SELECT * FROM (SELECT a, JSON_OBJECT('k', a) AS j FROM tc) d + WHERE d.j = '{"k": 1}'; + +--echo # +--echo # A table definition holding a JSON expression is copied whole +--echo # when the table is altered. +--echo # +CREATE TABLE tv (a INT, j VARCHAR(64) AS (JSON_ARRAY(a)) VIRTUAL, + CHECK (JSON_VALID(JSON_OBJECT('a', a)))); +INSERT INTO tv (a) VALUES (1), (2); +ALTER TABLE tv ADD COLUMN c INT; +SELECT * FROM tv ORDER BY a; + +DROP TABLE tv; +DROP TABLE tc; diff --git a/mysql-test/main/func_json_search_wide_charset.result b/mysql-test/main/func_json_search_wide_charset.result new file mode 100644 index 0000000000000..cf04cc5c6a0af --- /dev/null +++ b/mysql-test/main/func_json_search_wide_charset.result @@ -0,0 +1,118 @@ +SET NAMES utf8mb4; +# +# A match under an array index, so the path carries a number. +# The number goes in as one byte per digit whatever the set, and +# the bytes below show it standing among characters of two and of +# four. +# +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'all', +CONVERT('V' USING ucs2))) AS ucs2_all; +ucs2_all +005B00220024005B30005D005B30005D0022002C002000220024005B31005D005B30005D0022005D +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'one', +CONVERT('V' USING ucs2))) AS ucs2_one; +ucs2_one +00220024005B30005D005B30005D0022 +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf16), 'all', +CONVERT('V' USING utf16))) AS utf16_all; +utf16_all +005B00220024005B30005D005B30005D0022002C002000220024005B31005D005B30005D0022005D +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf32), 'all', +CONVERT('V' USING utf32))) AS utf32_all; +utf32_all +0000005B00000022000000240000005B300000005D0000005B300000005D000000220000002C0000002000000022000000240000005B310000005D0000005B300000005D000000220000005D +# +# Reading such a path back, and splicing it into something being +# built. Four bytes to the character is the case that shows it: +# the path does not read, and asking is what finds that out. +# +SET @u = CONVERT('[["V"],["V"]]' USING utf32); +SET @m = CONVERT('V' USING utf32); +SELECT JSON_VALID(JSON_SEARCH(@u, 'all', @m)) AS utf32_answer_reads; +utf32_answer_reads +0 +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 16 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@u, 'all', @m))) AS utf32_spliced; +utf32_spliced +0 +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 +Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 20 +SELECT JSON_TYPE(JSON_SEARCH(@u, 'all', @m)) AS utf32_type; +utf32_type +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_type' at position 16 +# +# The same at two bytes to the character. +# +SET @d = CONVERT('[["V"],["V"]]' USING ucs2); +SET @n = CONVERT('V' USING ucs2); +SELECT JSON_VALID(JSON_SEARCH(@d, 'all', @n)) AS ucs2_answer_reads; +ucs2_answer_reads +1 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@d, 'all', @n))) AS ucs2_spliced; +ucs2_spliced +1 +SELECT HEX(JSON_MERGE_PRESERVE(CONVERT('[0]' USING ucs2), +JSON_SEARCH(@d, 'all', @n))) AS ucs2_merged; +ucs2_merged +005B0030002C002000220024005B30005D005B30005D0022002C002000220024005B31005D005B30005D0022005D +# +# A path with no array step in it has no number in it and is +# encoded all the way through, so it reads and it splices. The +# two shapes are told apart by what was written into them. +# +SET @o = CONVERT('{"a":{"b":"V"}}' USING ucs2); +SELECT HEX(JSON_SEARCH(@o, 'one', @n)) AS keys_only; +keys_only +00220024002E0061002E00620022 +SELECT JSON_VALID(JSON_SEARCH(@o, 'one', @n)) AS keys_only_reads; +keys_only_reads +1 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@o, 'one', @n))) AS keys_only_spliced; +keys_only_spliced +1 +# +# A character set that does write a character in one byte is not +# this case at all, and none of it moves. +# +SELECT JSON_SEARCH('[["V"],["V"]]', 'all', 'V') AS utf8_all; +utf8_all +["$[0][0]", "$[1][0]"] +SELECT JSON_VALID(JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_reads; +utf8_reads +1 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH('[["V"],["V"]]', 'all', 'V'))) AS utf8_spliced; +utf8_spliced +1 +SELECT JSON_MERGE_PRESERVE('[0]', JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_merged; +utf8_merged +[0, "$[0][0]", "$[1][0]"] +SELECT JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), 'all', +CONVERT('V' USING latin1)) AS latin1_all; +latin1_all +["$[0][0]", "$[1][0]"] +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), +'all', +CONVERT('V' USING latin1)))) AS latin1_spliced; +latin1_spliced +1 +# +# Out of a table, so no one statement settles another, and with +# both shapes of path in the same statement. +# +CREATE TABLE t (id INT, d BLOB, n BLOB); +INSERT INTO t VALUES (1, CONVERT('[["V"],["V"]]' USING utf32), CONVERT('V' USING utf32)), +(2, CONVERT('{"a":{"b":"V"}}' USING utf32), CONVERT('V' USING utf32)); +SELECT id, JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT(d USING utf32), 'all', +CONVERT(n USING utf32)))) AS spliced +FROM t ORDER BY id; +id spliced +1 0 +2 1 +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 +Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 20 +DROP TABLE t; diff --git a/mysql-test/main/func_json_search_wide_charset.test b/mysql-test/main/func_json_search_wide_charset.test new file mode 100644 index 0000000000000..3805cd98c2797 --- /dev/null +++ b/mysql-test/main/func_json_search_wide_charset.test @@ -0,0 +1,91 @@ +# +# The path JSON_SEARCH returns, in a character set that does not +# write a character in one byte. +# +# A path is built out of punctuation the function writes, keys it copies +# from the document, and the index of an array step. The punctuation and +# the keys arrive in the character set of the result. The index does +# not: it is written as the digits themselves, so in ucs2, utf16 or +# utf32 it is the one part of the path not formatted the way the rest of +# it is, and every character after it is read from the wrong place. +# +# What is pinned here is that nothing is claimed on that path's behalf. +# Whether such a path can be read at all is settled differently by +# different character sets, and by asking it is settled at all - what +# must not happen is the answer being taken on trust instead. +# + +SET NAMES utf8mb4; + +--echo # +--echo # A match under an array index, so the path carries a number. +--echo # The number goes in as one byte per digit whatever the set, and +--echo # the bytes below show it standing among characters of two and of +--echo # four. +--echo # +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'all', + CONVERT('V' USING ucs2))) AS ucs2_all; +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'one', + CONVERT('V' USING ucs2))) AS ucs2_one; +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf16), 'all', + CONVERT('V' USING utf16))) AS utf16_all; +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf32), 'all', + CONVERT('V' USING utf32))) AS utf32_all; + +--echo # +--echo # Reading such a path back, and splicing it into something being +--echo # built. Four bytes to the character is the case that shows it: +--echo # the path does not read, and asking is what finds that out. +--echo # +SET @u = CONVERT('[["V"],["V"]]' USING utf32); +SET @m = CONVERT('V' USING utf32); +SELECT JSON_VALID(JSON_SEARCH(@u, 'all', @m)) AS utf32_answer_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@u, 'all', @m))) AS utf32_spliced; +SELECT JSON_TYPE(JSON_SEARCH(@u, 'all', @m)) AS utf32_type; + +--echo # +--echo # The same at two bytes to the character. +--echo # +SET @d = CONVERT('[["V"],["V"]]' USING ucs2); +SET @n = CONVERT('V' USING ucs2); +SELECT JSON_VALID(JSON_SEARCH(@d, 'all', @n)) AS ucs2_answer_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@d, 'all', @n))) AS ucs2_spliced; +SELECT HEX(JSON_MERGE_PRESERVE(CONVERT('[0]' USING ucs2), + JSON_SEARCH(@d, 'all', @n))) AS ucs2_merged; + +--echo # +--echo # A path with no array step in it has no number in it and is +--echo # encoded all the way through, so it reads and it splices. The +--echo # two shapes are told apart by what was written into them. +--echo # +SET @o = CONVERT('{"a":{"b":"V"}}' USING ucs2); +SELECT HEX(JSON_SEARCH(@o, 'one', @n)) AS keys_only; +SELECT JSON_VALID(JSON_SEARCH(@o, 'one', @n)) AS keys_only_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@o, 'one', @n))) AS keys_only_spliced; + +--echo # +--echo # A character set that does write a character in one byte is not +--echo # this case at all, and none of it moves. +--echo # +SELECT JSON_SEARCH('[["V"],["V"]]', 'all', 'V') AS utf8_all; +SELECT JSON_VALID(JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH('[["V"],["V"]]', 'all', 'V'))) AS utf8_spliced; +SELECT JSON_MERGE_PRESERVE('[0]', JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_merged; + +SELECT JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), 'all', + CONVERT('V' USING latin1)) AS latin1_all; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), + 'all', + CONVERT('V' USING latin1)))) AS latin1_spliced; + +--echo # +--echo # Out of a table, so no one statement settles another, and with +--echo # both shapes of path in the same statement. +--echo # +CREATE TABLE t (id INT, d BLOB, n BLOB); +INSERT INTO t VALUES (1, CONVERT('[["V"],["V"]]' USING utf32), CONVERT('V' USING utf32)), + (2, CONVERT('{"a":{"b":"V"}}' USING utf32), CONVERT('V' USING utf32)); +SELECT id, JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT(d USING utf32), 'all', + CONVERT(n USING utf32)))) AS spliced + FROM t ORDER BY id; +DROP TABLE t; diff --git a/sql/item.cc b/sql/item.cc index a0046ec8c7f95..b2b58d5e76898 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -2281,6 +2281,17 @@ class Item_aggregate_ref : public Item_ref Item_ident::print(str, query_type); } Ref_Type ref_type() override final { return AGGREGATE_REF; } + + /* + What this stands in front of is an aggregate, which keeps no result + field of its own, so reading through it reaches the aggregate's own + val_str() and the value arrives unaltered. Whatever the aggregate + can say about it therefore still holds here. + */ + bool is_valid_json() const override { return (*ref)->is_valid_json(); } + bool is_nice_json() const override { return (*ref)->is_nice_json(); } + bool is_valid_json_static() const override + { return (*ref)->is_valid_json_static(); } protected: Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } diff --git a/sql/item.h b/sql/item.h index 5e8f0d710ba2e..85d86fc8c1c51 100644 --- a/sql/item.h +++ b/sql/item.h @@ -861,6 +861,53 @@ static inline item_with_t operator~(const item_with_t a) } +/* + What a JSON function says about the value it has just returned: + is_valid, that the value is a document, and is_nice, that it is + formatted the way json_nice() writes one in its loose form. + + Both start out saying nothing, and a function says something only + where it hands a result back, so every other way out leaves them as + they were. Saying nothing is always safe - a caller that is told + nothing reads the value, which is what every caller does today. + + The two are kept together in one member rather than as two bools of + their own so that a class picking up one picks up the other, and so + that there is a single place to say what copying and setting them + mean, however many classes come to hold one. + + It lives here, and not with the JSON functions, because a class that + is not one of them can still carry a value through - see + Item_func_conv_charset - and going through this member is what puts + such a class under the debug check below. +*/ +class Json_result_marks +{ + bool m_valid; + bool m_nice; +public: + Json_result_marks(): m_valid(false), m_nice(false) {} + /* + A copy is a new item that has not been evaluated, so it has produced + no value and there is nothing for it to say. What the item it was + copied from can say is about bytes that item still owns and is free + to write over. + */ + Json_result_marks(const Json_result_marks &) + : m_valid(false), m_nice(false) {} + bool valid() const { return m_valid; } + bool nice() const { return m_nice; } + void clear() { m_valid= m_nice= false; } + /* + 'str' is the value 'valid' and 'nice' are being set for. A debug + build reads it back and stops if it is not what they say it is. + Nothing else ever would: the marks appear in no result, so a claim + that was not true would go unnoticed until something acted on it. + */ + void set(const String *str, bool valid, bool nice); +}; + + class Item :public Value_source, public Type_all_attributes { @@ -1708,6 +1755,43 @@ class Item :public Value_source, String *val_str(String *str, String *converter, CHARSET_INFO *to); virtual String *val_json(String *str) { return val_str(str); } + + /* + Whether the value returned by the LAST evaluation of this item is + guaranteed to be a JSON document, written in the character set the + value says it is written in. + + The question is put to the item rather than to the value because a + value has nowhere to keep the answer: val_str() returns bytes, a + length and a character set, and that is all there is room for. So a + caller evaluates the item and asks it, in that order, and the answer + stands until the item is evaluated again. + + FALSE does not say the value is not a document. It says only that + nothing here attests to it, and a caller who needs to know must read + the value and find out - which is what every caller does today. + Answering TRUE for a value that is not guaranteed is the one answer + that must never be given; answering FALSE for one that is costs a + reading and nothing else. + */ + virtual bool is_valid_json() const { return false; } + + /* + Whether that same value is formatted the way json_nice() writes a + document in its LOOSE form, which is the formatting the JSON functions + return today. Same rules as above, and the same one-sided cost: + a document said not to be in that form is simply written out again. + */ + virtual bool is_nice_json() const { return false; } + + /* + Whether EVERY evaluation of this item returns a document or + nothing at all. A property of the class, so it can be asked before + anything has been evaluated - which is what a caller running at + create time needs, the two answers above having no meaning there. + */ + virtual bool is_valid_json_static() const { return false; } + /* Return decimal representation of item with fixed point. @@ -6203,6 +6287,19 @@ class Item_direct_ref :public Item_ref longlong val_time_packed(THD *) override; Ref_Type ref_type() override { return DIRECT_REF; } + /* + val_str() here returns what the referenced item returned, byte + for byte, so whatever could be said about that value can still be + said about this one. Item_ref cannot say the same: it reads through + str_result(), which for a field is the copy sitting in a record + buffer rather than the value the producing item made, and nothing + here knows what happened to it on the way. + */ + bool is_valid_json() const override { return (*ref)->is_valid_json(); } + bool is_nice_json() const override { return (*ref)->is_nice_json(); } + bool is_valid_json_static() const override + { return (*ref)->is_valid_json_static(); } + /* Should be called if ref is changed */ inline void ref_changed() { @@ -6401,8 +6498,7 @@ class Item_direct_view_ref :public Item_direct_ref bool check_null_ref() { - DBUG_ASSERT(null_ref_table); - if (null_ref_table != NO_NULL_TABLE && null_ref_table->null_row) + if (null_row_ref()) { null_value= 1; return TRUE; @@ -6410,6 +6506,16 @@ class Item_direct_view_ref :public Item_direct_ref return FALSE; } + /* + The same question without the answer being recorded, for callers that + only want to know and are not evaluating anything. + */ + bool null_row_ref() const + { + DBUG_ASSERT(null_ref_table); + return null_ref_table != NO_NULL_TABLE && null_ref_table->null_row; + } + public: Item_direct_view_ref(THD *thd, Name_resolution_context *context_arg, Item **item, @@ -6469,6 +6575,19 @@ class Item_direct_view_ref :public Item_direct_ref Item *in_subq_field_transformer_for_where(THD *thd, uchar *arg) override; Item *in_subq_field_transformer_for_having(THD *thd, uchar *arg) override; + /* + On a row the outer join filled in with NULLs the val_XXX below return + without evaluating the referenced item at all, so what that item can + say is about whatever row it was last asked about. Say nothing here + instead. The row that was just produced is the one being asked about, + which is why this reads null_row_ref() rather than remembering + anything: it is answering about the same row the val_XXX did. + */ + bool is_valid_json() const override + { return !null_row_ref() && Item_direct_ref::is_valid_json(); } + bool is_nice_json() const override + { return !null_row_ref() && Item_direct_ref::is_nice_json(); } + void save_val(Field *to) override { if (check_null_ref()) diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 795b40f04c2a5..82e12f43eff62 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -353,11 +353,59 @@ int json_path_compare(const json_path_t *a, const json_path_t *b, } +/* + How a document is punctuated in the loose form - this much after a + comma, and this much after a key. + + Named here because two different pieces of code have to agree on it. + json_nice() writes it when it reads a document back, and the functions + that BUILD a document write it themselves and never read anything + back; if those two ever drifted apart, a document said to be in the + loose form would not be in it. The compact and detailed forms are + shorter and are taken as the first characters of the same two strings, + which is what the lengths below select. +*/ +static const char json_loose_comma[]= ", "; +static const char json_loose_colon[]= "\": "; + + +/* + Whether a document can be written in this character set at all. + + A document is punctuated with characters that most character sets + encode the way ASCII does, and a few do not: swe7 puts national letters + where the brackets, the braces and the backslash belong, so the bytes + that make an array anywhere else make a word there. A function that + writes an array out in such a character set has written something that + is not a document and cannot be read as one - which is what the server + has always done with it, and is not for this to change. What it is + for is to keep that result from being taken for a document later. + + Only the functions that write their own punctuation have to ask. The + ones that read their whole result back afterwards read it in the + character set it is written in, so a document they accept is one that + reads there, and asking would tell them nothing new. + + What decides it is how narrow a character is, not whether the set is + ASCII-compatible. MY_CS_NONASCII marks two unrelated things: sets that + put other characters at the ASCII code points, and sets that are simply + too wide to hold ASCII a byte at a time. Only the first is a problem. + String::append(char) converts for a set whose characters are never one + byte, so ucs2 gets a real bracket, encoded as 005B; where a character + can be a single byte the byte goes in as it stands, and in swe7 that + byte is a letter. +*/ +static inline bool is_json_compatible_charset(CHARSET_INFO *cs) +{ + return cs->mbminlen > 1 || !(cs->state & MY_CS_NONASCII); +} + + static int json_nice(json_engine_t *je, String *nice_js, Item_func_json_format::formats mode, int tab_size=4) { int depth= 0; - static const char *comma= ", ", *colon= "\": "; + static const char *comma= json_loose_comma, *colon= json_loose_colon; uint comma_len, colon_len; int first_value= 1; int value_size = 0; @@ -518,6 +566,76 @@ static int json_nice(json_engine_t *je, String *nice_js, } +/* + The two below are read by DBUG_ASSERT and by nothing else, so they are + here wherever a DBUG_ASSERT is compiled rather than wherever a debug + build is. Those are not the same set: DBUG_ASSERT_AS_PRINTF turns the + assertion into a printed complaint and leaves its expression standing, + while setting DBUG_OFF, so a guard written the other way round leaves + those expressions with nothing to call and that build stops compiling. +*/ +#ifdef DBUG_ASSERT_EXISTS +/* + Reads a value the way whoever receives it will read it: in the + character set the value says it is written in, from one end of it to + the other. +*/ +static bool json_value_reads_as_document(const String *str) +{ + /* + Which is the question json_valid() is, so it is asked rather than + asked again here. + */ + return json_valid(str->ptr(), str->length(), str->charset()) != 0; +} + + +/* + Writes the value out again in the loose form and asks whether that + changed anything. A value already written that way comes back byte + for byte; one written any other way does not. +*/ +static bool json_value_is_nice(const String *str) +{ + json_engine_t je; + StringBuffer nice; + + json_scan_start(&je, str->charset(), (const uchar *) str->ptr(), + (const uchar *) str->ptr() + str->length()); + if (json_nice(&je, &nice, Item_func_json_format::LOOSE)) + return false; + + return !stringcmp(&nice, str); +} +#endif + + +/* + A mark is a promise about a run of bytes, and the bytes are right + here, so a debug build keeps the promise by reading them. + + Nothing else ever will. The marks appear in no result and change no + output, so one of them being wrong shows up nowhere at all until + something starts acting on it - and by then the function that made the + promise is a long way from the code that believed it. +*/ +void Json_result_marks::set(const String *str, bool valid, bool nice) +{ + /* + The loose form is a formatting of a document, so there is no such + thing as a value formatted that way which is not one. Said before the + two below because it holds whatever the bytes turn out to be, and + because a caller that arrives here with the pair the wrong way round + has lost track of which question it was answering. + */ + DBUG_ASSERT(!nice || valid); + DBUG_ASSERT(!valid || json_value_reads_as_document(str)); + DBUG_ASSERT(!nice || json_value_is_nice(str)); + m_valid= valid; + m_nice= nice; +} + + #define report_json_error(js, je, n_param) \ report_json_error_ex(js->ptr(), je, func_name(), n_param, \ Sql_condition::WARN_LEVEL_WARN) @@ -1355,6 +1473,8 @@ String *Item_func_json_extract::read_json(String *str, uint has_negative_path= 0; THD *thd; + m_marks.clear(); + if ((null_value= args[0]->null_value)) return 0; @@ -1486,6 +1606,9 @@ String *Item_func_json_extract::read_json(String *str, if (json_nice(&je, &tmp_js, Item_func_json_format::LOOSE)) goto error; + /* The reading back just above is what sets both marks. */ + m_marks.set(&tmp_js, true, true); + return_ok: return &tmp_js; @@ -2134,6 +2257,31 @@ bool is_json_type(const Item *item) } +/* + What splicing values into a document has left the document able to say + about itself. Starts out saying both and is only ever cleared: one + value that cannot be attested to is enough, however many others went + in cleanly. + + Named for what holds rather than for what went wrong, and named after + the questions an item answers, so that a value's own answer and what + became of it here read the same way round. A document is what the + the loose form is a form OF a document, so nothing clears the second + without clearing the first. + + A caller that reads the whole of its own result back afterwards learns + nothing from these and passes NULL instead - the reading back is a + stronger answer than anything collected on the way. That is every + function which EDITS a document today; when the reading back goes, + each of them will want a set of these of its own. +*/ +struct Json_splice_marks +{ + bool is_valid; /* everything that went in read as a document */ + bool is_nice; /* everything that went in was written the loose way */ +}; + + /* Appends a value whose type is JSON, which is spliced into the result as it stands instead of being quoted as a string. Any complete JSON @@ -2169,9 +2317,13 @@ bool is_json_type(const Item *item) 'depth' is how many structures the value will sit inside once it has been spliced. Its own nesting adds to that, and it is the total that the scanner's limit applies to. + + 'marks' is where the parse goes, the bytes having gone in whatever it + found. See Json_splice_marks. */ static int append_json_typed_value(String *str, const String *sv, uint depth, - const char *fname, int n_param) + const char *fname, int n_param, + Json_splice_marks *marks) { StringBuffer cnv; const char *ptr= sv->ptr(); @@ -2247,8 +2399,12 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, same thing at the same level for the same reason. */ if (je.s.error) + { + if (marks) + marks->is_valid= marks->is_nice= false; report_json_error_ex(ptr, &je, fname, n_param, Sql_condition::WARN_LEVEL_NOTE); + } return append_simple(str, ptr, length); } @@ -2307,7 +2463,8 @@ __attribute__((warn_unused_result)) out as a string complains through the bad-character note. */ static int append_json_value(String *str, Item *item, String *tmp_val, - uint depth, const char *fname, int n_param) + uint depth, const char *fname, int n_param, + Json_splice_marks *marks) { if (item->type_handler()->is_bool_type()) { @@ -2333,13 +2490,34 @@ static int append_json_value(String *str, Item *item, String *tmp_val, } { String *sv= item->val_json(tmp_val); + int rc; + if (item->null_value) goto append_null; if (is_json_type(item)) - return append_json_typed_value(str, sv, depth, fname, n_param); + { + /* + Spliced as it stands, so the document being built is written the + loose way only if this value was. Nothing is measured here to + find that out - the item is asked, and one that says nothing is + taken at its word. + */ + if (marks && !item->is_nice_json()) + marks->is_nice= false; + return append_json_typed_value(str, sv, depth, fname, n_param, marks); + } - return append_escaped_value(str, sv, item->result_type() == STRING_RESULT, - fname, n_param); + rc= append_escaped_value(str, sv, item->result_type() == STRING_RESULT, + fname, n_param); + /* + A character that could not be written leaves the value half + written and the quote around it unclosed. A caller that gives up + here does not care; one that carries on is building something that + no longer reads as a document. + */ + if (marks && rc == JSON_APPEND_BAD_CHR) + marks->is_valid= marks->is_nice= false; + return rc; } append_null: @@ -2351,7 +2529,7 @@ static int append_json_value(String *str, Item *item, String *tmp_val, __attribute__((warn_unused_result)) static int append_json_value_from_field(String *str, Item *i, Field *f, const uchar *key, size_t offset, String *tmp_val, - uint depth, const char *fname, int n_param) + uint depth, const char *fname, int n_param, Json_splice_marks *marks) { if (i->type_handler()->is_bool_type()) { @@ -2377,13 +2555,27 @@ static int append_json_value_from_field(String *str, } { String *sv= f->val_str(tmp_val, key + offset); + int rc; + if (f->is_null_in_record(key)) goto append_null; if (is_json_type(i)) - return append_json_typed_value(str, sv, depth, fname, n_param); + { + /* + The bytes come out of a record rather than off an item, and a + record says nothing about how what is in it was written, so the + document being built cannot be said to be written the loose way. + */ + if (marks) + marks->is_nice= false; + return append_json_typed_value(str, sv, depth, fname, n_param, marks); + } - return append_escaped_value(str, sv, i->result_type() == STRING_RESULT, - fname, n_param); + rc= append_escaped_value(str, sv, i->result_type() == STRING_RESULT, + fname, n_param); + if (marks && rc == JSON_APPEND_BAD_CHR) + marks->is_valid= marks->is_nice= false; + return rc; } append_null: @@ -2422,10 +2614,13 @@ static json_append_result append_json_keyname(String *str, Item *item, return rc; } - return str->append("\": ", 3) ? JSON_APPEND_OOM : JSON_APPEND_OK; + return str->append(STRING_WITH_LEN(json_loose_colon)) ? + JSON_APPEND_OOM : JSON_APPEND_OK; append_null: - return str->append("\"\": ", 4) ? JSON_APPEND_OOM : JSON_APPEND_OK; + return (str->append('"') || + str->append(STRING_WITH_LEN(json_loose_colon))) ? + JSON_APPEND_OOM : JSON_APPEND_OK; } @@ -2478,20 +2673,23 @@ String *Item_func_json_array::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; + Json_splice_marks marks= {true, true}; + + m_marks.clear(); str->length(0); str->set_charset(collation.collation); if (str->append('[') || ((arg_count > 0) && - append_json_value(str, args[0], &tmp_val, 1, func_name(), 0))) + append_json_value(str, args[0], &tmp_val, 1, func_name(), 0, &marks))) goto err_return; for (n_arg=1; n_arg < arg_count; n_arg++) { - if (str->append(", ", 2) || + if (str->append(STRING_WITH_LEN(json_loose_comma)) || append_json_value(str, args[n_arg], &tmp_val, 1, func_name(), - (int) n_arg)) + (int) n_arg, &marks)) goto err_return; } @@ -2502,7 +2700,23 @@ String *Item_func_json_array::val_str(String *str) result_limit= current_thd->variables.max_allowed_packet; if (str->length() <= result_limit) + { + /* + Nothing was read back, so what can be said about the array is only + what was learned putting it together: the brackets and separators + are written here in the loose form, the values that were + quoted came out of json_escape() and are documents on their own, + and the values that went in as they stand were read as they went. + + All of which holds only where the brackets written above are + brackets - see is_json_compatible_charset(). + */ + bool is_json_compatible= is_json_compatible_charset(str->charset()); + + m_marks.set(str, is_json_compatible && marks.is_valid, + is_json_compatible && marks.is_nice); return str; + } push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_ALLOWED_PACKET_OVERFLOWED, @@ -2549,6 +2763,7 @@ String *Item_func_json_array_append::val_str(String *str) THD *thd; DBUG_ASSERT(fixed()); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; @@ -2612,7 +2827,7 @@ String *Item_func_json_array_append::val_str(String *str) if (n_items) str->append(", ", 2); if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1)) + (int) n_arg + 1, NULL)) goto return_null; /* Out of memory. */ if (str->reserve(str_rest_len, 1024)) @@ -2640,7 +2855,7 @@ String *Item_func_json_array_append::val_str(String *str) str->append((const char *) c_from, c_to - c_from) || str->append(", ", 2) || append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1) || + (int) n_arg + 1, NULL) || str->append(']') || str->append((const char *) je.s.c_str, js->end() - (const char *) je.s.c_str)) @@ -2667,6 +2882,8 @@ String *Item_func_json_array_append::val_str(String *str) if (json_nice(&je, str, Item_func_json_format::LOOSE)) goto js_error; + /* The reading back just above is what sets both marks. */ + m_marks.set(str, true, true); return str; js_error: @@ -2686,6 +2903,7 @@ String *Item_func_json_array_insert::val_str(String *str) THD *thd; DBUG_ASSERT(fixed()); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; @@ -2798,7 +3016,7 @@ String *Item_func_json_array_insert::val_str(String *str) goto return_null; /* Out of memory. */ } if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1)) + (int) n_arg + 1, NULL)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ @@ -2838,7 +3056,7 @@ String *Item_func_json_array_insert::val_str(String *str) goto return_null; /* Out of memory. */ } if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1)) + (int) n_arg + 1, NULL)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ @@ -2872,6 +3090,8 @@ String *Item_func_json_array_insert::val_str(String *str) if (json_nice(&je, str, Item_func_json_format::LOOSE)) goto js_error; + /* The reading back just above is what sets both marks. */ + m_marks.set(str, true, true); return str; js_error: @@ -2887,6 +3107,9 @@ String *Item_func_json_object::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; + Json_splice_marks marks= {true, true}; + + m_marks.clear(); str->length(0); str->set_charset(collation.collation); @@ -2894,16 +3117,16 @@ String *Item_func_json_object::val_str(String *str) if (str->append('{') || (arg_count > 0 && (append_json_keyname(str, args[0], &tmp_val, func_name(), 0) || - append_json_value(str, args[1], &tmp_val, 1, func_name(), 1)))) + append_json_value(str, args[1], &tmp_val, 1, func_name(), 1, &marks)))) goto err_return; for (n_arg=2; n_arg < arg_count; n_arg+=2) { - if (str->append(", ", 2) || + if (str->append(STRING_WITH_LEN(json_loose_comma)) || append_json_keyname(str, args[n_arg], &tmp_val, func_name(), (int) n_arg) || append_json_value(str, args[n_arg+1], &tmp_val, 1, func_name(), - (int) n_arg + 1)) + (int) n_arg + 1, &marks)) goto err_return; } @@ -2914,7 +3137,14 @@ String *Item_func_json_object::val_str(String *str) result_limit= current_thd->variables.max_allowed_packet; if (str->length() <= result_limit) + { + /* As in the sister constructor above. */ + bool is_json_compatible= is_json_compatible_charset(str->charset()); + + m_marks.set(str, is_json_compatible && marks.is_valid, + is_json_compatible && marks.is_nice); return str; + } push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_ALLOWED_PACKET_OVERFLOWED, @@ -3145,6 +3375,8 @@ String *Item_func_json_merge::val_str(String *str) uint n_arg; THD *thd; + m_marks.clear(); + if (args[0]->null_value) goto null_return; @@ -3194,6 +3426,8 @@ String *Item_func_json_merge::val_str(String *str) goto error_return; null_value= 0; + /* The reading back just above is what sets both marks. */ + m_marks.set(str, true, true); return str; error_return: @@ -3458,6 +3692,8 @@ String *Item_func_json_merge_patch::val_str(String *str) bool empty_result, merge_to_null; THD *thd= current_thd; + m_marks.clear(); + JSON_DO_PAUSE_EXECUTION(thd, 0.0002); /* To report errors properly if some JSON is invalid. */ @@ -3529,6 +3765,8 @@ String *Item_func_json_merge_patch::val_str(String *str) goto error_return; null_value= 0; + /* The reading back just above is what sets both marks. */ + m_marks.set(str, true, true); return str; error_return: @@ -3802,6 +4040,7 @@ String *Item_func_json_insert::val_str(String *str) THD *thd; DBUG_ASSERT(fixed()); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; @@ -3909,7 +4148,7 @@ String *Item_func_json_insert::val_str(String *str) (append_simple(str, v_from, je.s.c_str - v_from) || str->append(", ", 2))) || append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1) || + (int) n_arg + 1, NULL) || (do_array_autowrap && str->append(']')) || append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) goto js_error; /* Out of memory. */ @@ -3952,7 +4191,7 @@ String *Item_func_json_insert::val_str(String *str) if (append_simple(str, js->ptr(), v_to - js->ptr()) || (n_item > 0 && str->append(", ", 2)) || append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1) || + (int) n_arg + 1, NULL) || append_simple(str, v_to, js->end() - v_to)) goto js_error; /* Out of memory. */ } @@ -4007,7 +4246,7 @@ String *Item_func_json_insert::val_str(String *str) if (str->append("\":", 2) || append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1) || + (int) n_arg + 1, NULL) || append_simple(str, v_to, js->end() - v_to)) goto js_error; /* Out of memory. */ } @@ -4032,7 +4271,7 @@ String *Item_func_json_insert::val_str(String *str) if (append_simple(str, js->ptr(), v_to - js->ptr()) || append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1) || + (int) n_arg + 1, NULL) || append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) goto js_error; /* Out of memory. */ continue_point: @@ -4057,6 +4296,8 @@ String *Item_func_json_insert::val_str(String *str) if (json_nice(&je, str, Item_func_json_format::LOOSE)) goto js_error; + /* The reading back just above is what sets both marks. */ + m_marks.set(str, true, true); return str; js_error: @@ -4087,6 +4328,7 @@ String *Item_func_json_remove::val_str(String *str) THD *thd; DBUG_ASSERT(fixed()); + m_marks.clear(); if (args[0]->null_value) goto null_return; @@ -4269,6 +4511,8 @@ String *Item_func_json_remove::val_str(String *str) goto js_error; null_value= 0; + /* The reading back just above is what sets both marks. */ + m_marks.set(str, true, true); return str; js_error: @@ -4524,7 +4768,8 @@ int Item_func_json_search::compare_json_value_wild(json_engine_t *je, } -static int append_json_path(String *str, const json_path_t *p) +static int append_json_path(String *str, const json_path_t *p, + bool *path_bytes_well_formed) { const json_path_step_t *c; @@ -4541,6 +4786,17 @@ static int append_json_path(String *str, const json_path_t *p) } else /*JSON_PATH_ARRAY*/ { + /* + The brackets go in through String::append(), which converts them, + so a set that writes no character in one byte gets a bracket of + its own width. The number between them does not go that way: the + digits are written as themselves, whatever the set. It is the + one part of a path not formatted the way the rest of it is, and one + stray byte is enough to leave every character after it reading + from the wrong place. + */ + if (str->charset()->mbminlen > 1) + *path_bytes_well_formed= false; if (str->append('[') || str->append_ulonglong(c->n_item) || @@ -4562,6 +4818,9 @@ String *Item_func_json_search::val_str(String *str) uint n_arg; int array_sizes[JSON_DEPTH_LIMIT]; uint has_negative_path= 0; + bool path_bytes_well_formed= true; + + m_marks.clear(); if (args[0]->null_value || args[2]->null_value) goto null_return; @@ -4622,10 +4881,11 @@ String *Item_func_json_search::val_str(String *str) if (n_path_found == 2) { if (str->append('[') || - append_json_path(str, &sav_path)) + append_json_path(str, &sav_path, &path_bytes_well_formed)) goto js_error; } - if (str->append(", ", 2) || append_json_path(str, &p)) + if (str->append(STRING_WITH_LEN(json_loose_comma)) || + append_json_path(str, &p, &path_bytes_well_formed)) goto js_error; } if (mode_one) @@ -4642,7 +4902,7 @@ String *Item_func_json_search::val_str(String *str) goto null_return; if (n_path_found == 1) { - if (append_json_path(str, &sav_path)) + if (append_json_path(str, &sav_path, &path_bytes_well_formed)) goto js_error; } else @@ -4651,6 +4911,26 @@ String *Item_func_json_search::val_str(String *str) goto js_error; } + /* + A path is written out as a JSON string, and the keys that go into it + are copied from the document with whatever escaping they were + written with there - a key holding a quote was already written with + that quote escaped, or the document would not have read. Several + paths are put in an array with the loose formatting between them. + + None of that is worth anything if the bytes do not read at all, and + a path carrying an array index in a set that writes no character in + one byte does not - see the number written into it. A set that can + write answers only for what went through it, so the path says for + itself whether it was written, and an unencoded one is passed on as + the bytes it is rather than as a document. + */ + { + bool result_is_document= is_json_compatible_charset(str->charset()) && + path_bytes_well_formed; + + m_marks.set(str, result_is_document, result_is_document); + } null_value= 0; return str; @@ -4711,6 +4991,8 @@ String *Item_func_json_format::val_str(String *str) int tab_size= 4; THD *thd; + m_marks.clear(); + if ((null_value= args[0]->null_value)) return 0; @@ -4745,15 +5027,28 @@ String *Item_func_json_format::val_str(String *str) return 0; } + /* + Read back and written out again, so is_valid holds whatever the + argument was - but only json_loose() writes the loose form, the + other two formats being the whole point of the other two names. + */ + m_marks.set(str, true, fmt == LOOSE); return str; } +/* + Nothing is read and nothing is written: the argument's value is passed + straight on. So whatever the argument was able to say about it is + still true of it here, and this function says neither more nor less. +*/ String *Item_func_json_format::val_json(String *str) { String *js= args[0]->val_json(&tmp_js); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; + m_marks.set(js, args[0]->is_valid_json(), args[0]->is_nice_json()); return js; } @@ -4897,6 +5192,7 @@ void Item_func_json_arrayagg::clear() { m_bad_element= false; m_closed= false; + m_elements_valid= true; Item_func_group_concat::clear(); } @@ -4921,9 +5217,14 @@ void Item_func_json_arrayagg::clear() String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) { int rc; + Json_splice_marks marks= {true, true}; m_tmp_json.length(0); - if ((rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0))) + rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0, &marks); + if (!marks.is_valid) + m_elements_valid= false; + + if (rc) { if (rc == JSON_APPEND_OOM) m_bad_element= true; @@ -4937,11 +5238,16 @@ String *Item_func_json_arrayagg::get_str_from_field(Item *i,Field *f, String *tmp, const uchar *key, size_t offset) { int rc; + Json_splice_marks marks= {true, true}; m_tmp_json.length(0); - if ((rc= append_json_value_from_field(&m_tmp_json, i, f, key, offset, tmp, - 1, json_arrayagg_name, 0))) + rc= append_json_value_from_field(&m_tmp_json, i, f, key, offset, tmp, + 1, json_arrayagg_name, 0, &marks); + if (!marks.is_valid) + m_elements_valid= false; + + if (rc) { if (rc == JSON_APPEND_OOM) m_bad_element= true; @@ -4978,6 +5284,8 @@ Item *Item_func_json_arrayagg::copy_or_same(THD* thd) String* Item_func_json_arrayagg::val_str(String *str) { + m_marks.clear(); + if ((str= Item_func_group_concat::val_str(str))) { String s; @@ -5009,24 +5317,46 @@ String* Item_func_json_arrayagg::val_str(String *str) } /* - The brackets are already round what the parent returned, this - group having been asked for once before. + Asked for a second time, the group is already inside its brackets + and there is nothing left to do to it. What can be said about it + is said either way: the answer is about the value, and the value + is the same one. */ - if (m_closed) - return str; + if (!m_closed) + { + /* + The brackets are put on last, so a buffer that will not take them + leaves behind something that reads as a value of its own rather + than as the array it is meant to be. + */ + if (s.append('[')) + goto bad_result; + + s.swap(*str); + if (str->append(s) || str->append(']')) + goto bad_result; + + m_closed= true; + } /* - The brackets are put on last, so a buffer that will not take them - leaves behind something that reads as a value of its own rather - than as the array it is meant to be. + A group that was cut to fit the length limit is cut at the byte + the limit falls on, so what stands between the brackets can be + half an element - or, where the cut lands just past an opening + quote and that quote is written back to close the string, an + element no row of the group ever held. One that carried + something not readable as JSON is a group with that in it still. + Either way the brackets are around something this cannot answer + for. + + Never said to be nicely written: what goes between the elements is + the separator this function inherits from GROUP_CONCAT, a comma + with nothing after it, where a nicely written document puts a + space there as well. There is no asking for another one - the + grammar takes no SEPARATOR here and writes that comma itself. */ - if (s.append('[')) - goto bad_result; - - s.swap(*str); - if (str->append(s) || str->append(']')) - goto bad_result; - m_closed= true; + m_marks.set(str, is_json_compatible_charset(str->charset()) && + m_elements_valid && !warning_for_row, false); } return str; @@ -5038,7 +5368,8 @@ String* Item_func_json_arrayagg::val_str(String *str) Item_func_json_objectagg:: Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item) - :Item_sum(thd, item), m_bad_pair(false), m_closed(false) + :Item_sum(thd, item), m_bad_pair(false), m_closed(false), + m_pairs_valid(true) { quick_group= FALSE; result.set_charset(collation.collation); @@ -5122,6 +5453,7 @@ void Item_func_json_objectagg::clear() */ m_bad_pair= false; m_closed= false; + m_pairs_valid= true; if (result.append('{')) m_bad_pair= true; } @@ -5130,6 +5462,7 @@ void Item_func_json_objectagg::clear() bool Item_func_json_objectagg::add() { StringBuffer buf; + Json_splice_marks marks= {true, true}; String *key; int rc; @@ -5160,14 +5493,22 @@ bool Item_func_json_objectagg::add() if (rc == JSON_APPEND_OOM) goto bad_pair; report_bad_chr_note(func_name(), 1); + /* + However much of the key could be written went in and the pair was + finished around it, so the object is complete and its keys are + not what was asked for. + */ + m_pairs_valid= false; } if (result.append(STRING_WITH_LEN("\":"))) goto bad_pair; buf.length(0); - if (append_json_value(&result, args[1], &buf, 1, func_name(), 1) == - JSON_APPEND_OOM) + rc= append_json_value(&result, args[1], &buf, 1, func_name(), 1, &marks); + if (!marks.is_valid) + m_pairs_valid= false; + if (rc == JSON_APPEND_OOM) goto bad_pair; return 0; @@ -5187,6 +5528,8 @@ bool Item_func_json_objectagg::add() String* Item_func_json_objectagg::val_str(String* str) { DBUG_ASSERT(fixed()); + m_marks.clear(); + if (null_value) return 0; @@ -5197,6 +5540,13 @@ String* Item_func_json_objectagg::val_str(String* str) } m_closed= true; + /* + Never said to be nicely written: a key is followed here by a colon + and the value straight after it, where the loose form puts a space + between the two. + */ + m_marks.set(&result, is_json_compatible_charset(result.charset()) && + m_pairs_valid, false); return &result; } @@ -5208,6 +5558,8 @@ String *Item_func_json_normalize::val_str(String *buf) THD *thd; String *raw_json= args[0]->val_str(&tmp); + m_marks.clear(); + DYNAMIC_STRING normalized_json; if (init_dynamic_string(&normalized_json, NULL, 0, 0)) { @@ -5233,6 +5585,13 @@ String *Item_func_json_normalize::val_str(String *buf) if (buf->append(normalized_json.str, normalized_json.length)) goto null_return; + /* + Written out afresh from a document that was read through in full, + and written in the character set this function declares, whatever + the argument arrived in. Not written the loose way, though: the + normal form puts nothing after a comma or a colon. + */ + m_marks.set(buf, true, false); goto end; null_return: diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 8c8440faf4697..bedb9b06bcc92 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -182,6 +182,8 @@ class Item_func_json_exists: public Item_bool_func class Item_json_func: public Item_str_func { +protected: + Json_result_marks m_marks; public: Item_json_func(THD *thd) :Item_str_func(thd) { } @@ -195,6 +197,8 @@ class Item_json_func: public Item_str_func { return Type_handler_json_common::json_type_handler(max_length); } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } }; @@ -243,15 +247,29 @@ class Item_func_json_query: public Item_json_func, bool fix_length_and_dec(THD *thd) override; String *val_str(String *to) override { + m_marks.clear(); null_value= Json_path_extractor::extract(to, args[0], args[1], collation.collation, func_name_cstring(), true); - return null_value ? NULL : to; + if (null_value) + return NULL; + /* + Not said to be nicely formatted: the piece is copied out with + whatever spacing the document it came from was written with. + */ + m_marks.set(to, true, false); + return to; } bool check_and_get_value(Json_engine_scan *je, String *res, int *error) override { return je->check_and_get_value_complex(res, error); } + /* + Returns a slice of the searched document, delimited by the two ends + of a value the scanner has just parsed. A fully parsed value is a + document in its own right. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -352,6 +370,12 @@ class Item_func_json_extract: public Item_json_str_multipath double val_real() override; my_decimal *val_decimal(my_decimal *) override; uint get_n_paths() const override { return arg_count - 1; } + /* + The values found are put together into a result of their own, and + that result is then read back through json_nice() before it is + passed on. Anything it cannot read comes back as NULL. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -453,6 +477,14 @@ class Item_func_json_array_append: public Item_json_str_multipath static LEX_CSTRING name= {STRING_WITH_LEN("json_array_append") }; return name; } + /* + Every function that EDITS a document reads the whole of what it has + written back through json_nice() before handing it over, and hands + back NULL for anything that will not read. That reading is what + makes is_valid true for the result, and it is the same in all six + of them. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -511,6 +543,8 @@ class Item_func_json_merge: public Item_func_json_array static LEX_CSTRING name= {STRING_WITH_LEN("json_merge_preserve") }; return name; } + /* Reads its result back, as the other editing functions do. */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -535,6 +569,11 @@ class Item_func_json_merge_patch: public Item_func_json_merge }; +/* + Reads its argument through in full and writes the document out again + in a normal form, in utf8mb4 whatever the argument arrived in. What + it writes is the compact formatting, so it is never in the loose form. +*/ class Item_func_json_normalize: public Item_json_func { public: @@ -547,6 +586,7 @@ class Item_func_json_normalize: public Item_json_func return name; } bool fix_length_and_dec(THD *thd) override; + bool is_valid_json_static() const override { return true; } Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } }; @@ -641,6 +681,8 @@ class Item_func_json_insert: public Item_json_str_multipath bool fix_length_and_dec(THD *thd) override; String *val_str(String *) override; uint get_n_paths() const override { return arg_count/2; } + /* Reads its result back, as the other editing functions do. */ + bool is_valid_json_static() const override { return true; } LEX_CSTRING func_name_cstring() const override { static LEX_CSTRING json_set= {STRING_WITH_LEN("json_set") }; @@ -671,6 +713,8 @@ class Item_func_json_remove: public Item_json_str_multipath static LEX_CSTRING name= {STRING_WITH_LEN("json_remove") }; return name; } + /* Reads its result back, as the other editing functions do. */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -725,6 +769,11 @@ class Item_func_json_search: public Item_json_str_multipath bool fix_length_and_dec(THD *thd) override; String *val_str(String *) override; uint get_n_paths() const override { return arg_count > 4 ? arg_count - 4 : 0; } + /* + A path is a JSON string, and every path this returns is built out + of pieces of a document that has just been read through. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -792,6 +841,18 @@ class Item_func_json_arrayagg : public Item_func_group_concat clear(). */ bool m_closed; + /* + Whether everything that went into this group leaves the array around + it answerable for. Cleared by a value that did not read as JSON, or + one holding a character that could not be written, which is dropped + and leaves the separator either side of it with nothing between + them. Reset for each group by clear(), the same as above, and kept + apart from it because they say different things - that one means the + group is missing a row, this one that the group is complete and + still not a document. + */ + bool m_elements_valid; + Json_result_marks m_marks; public: String m_tmp_json; /* Used in get_str_from_*.. */ Item_func_json_arrayagg(THD *thd, Name_resolution_context *context_arg, @@ -800,7 +861,7 @@ class Item_func_json_arrayagg : public Item_func_group_concat bool limit_clause, Item *row_limit, Item *offset_limit): Item_func_group_concat(thd, context_arg, is_distinct, is_select, is_order, is_separator, limit_clause, row_limit, offset_limit), - m_bad_element(false), m_closed(false) + m_bad_element(false), m_closed(false), m_elements_valid(true) { } /* @@ -811,7 +872,8 @@ class Item_func_json_arrayagg : public Item_func_group_concat not JSON. */ Item_func_json_arrayagg(THD *thd, Item_func_json_arrayagg *item) : - Item_func_group_concat(thd, item), m_bad_element(false), m_closed(false) + Item_func_group_concat(thd, item), m_bad_element(false), + m_closed(false), m_elements_valid(true) { m_tmp_json.set_charset(collation.collation); } @@ -819,6 +881,8 @@ class Item_func_json_arrayagg : public Item_func_group_concat { return Type_handler_json_common::json_type_handler_sum(this); } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } LEX_CSTRING func_name_cstring() const override { @@ -855,6 +919,18 @@ class Item_func_json_objectagg : public Item_sum would write one brace per call. Reset for each group by clear(). */ bool m_closed; + /* + Whether everything that went into this group leaves the object + around it answerable for. Cleared by a key or a value holding a + character that could not be written, which goes in as however much + of it fitted, or by a value that did not read as JSON. Reset for + each group by clear(), and kept apart from the flag above for the + same reason as in the sister aggregate - one means a pair is + missing, this one that the pairs are all there and still do not make + a document. + */ + bool m_pairs_valid; + Json_result_marks m_marks; public: /* The opening brace is not written here. This runs while the @@ -864,7 +940,8 @@ class Item_func_json_objectagg : public Item_sum once per group and once the width is known. */ Item_func_json_objectagg(THD *thd, Item *key, Item *value) : - Item_sum(thd, key, value), m_bad_pair(false), m_closed(false) + Item_sum(thd, key, value), m_bad_pair(false), m_closed(false), + m_pairs_valid(true) { quick_group= FALSE; } @@ -882,6 +959,8 @@ class Item_func_json_objectagg : public Item_sum { return Type_handler_json_common::json_type_handler_sum(this); } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } void clear() override; bool add() override; void reset_field() override { DBUG_ASSERT(0); } // not used diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index b4388f12e7f4c..c572d1d1547fd 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -3885,13 +3885,23 @@ String *Item_func_conv_charset::val_str(String *str) DBUG_ASSERT(fixed()); if (use_cached_value) return null_value ? 0 : &str_value; + m_marks.clear(); String *arg= args[0]->val_str(&tmp_value); String_copier_for_item copier(current_thd); - return ((null_value= args[0]->null_value || - copier.copy_with_warn(collation.collation, str, - arg->charset(), arg->ptr(), - arg->length(), arg->length()))) ? - 0 : str; + if ((null_value= args[0]->null_value || + copier.copy_with_warn(collation.collation, str, + arg->charset(), arg->ptr(), + arg->length(), arg->length()))) + return 0; + /* + Asked of the argument only now, after it has been evaluated: what it + answers is about the value it has just passed, which is the one + that was converted. + */ + if (!copier.most_important_error_pos() && + collation.collation != &my_charset_bin) + m_marks.set(str, args[0]->is_valid_json(), args[0]->is_nice_json()); + return str; } bool Item_func_conv_charset::fix_length_and_dec(THD *thd) diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index a0d8ea22136a7..fc7432a4f292f 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -1982,6 +1982,21 @@ class Item_func_conv_charset :public Item_str_func { bool use_cached_value; String tmp_value; + /* + What can be said about the converted value. A conversion that lost + nothing kept the sequence of characters it was given, and JSON is a + grammar over characters, so a document that went in comes out a + document and comes out written the same way. A conversion that + substituted a character, or stopped on a byte it could not read, has + changed the value into something nothing here has looked at. + + Conversion to "binary" is the exception, and is refused below. It + is not a conversion but a relabelling: the bytes are kept, and each + one becomes a character of its own, so a multi-byte document comes + out as several times as many characters as it had. The copier + cannot report that, binary having no byte that it rejects. + */ + Json_result_marks m_marks; public: bool safe; Item_func_conv_charset(THD *thd, Item *a, CHARSET_INFO *cs): @@ -2004,6 +2019,15 @@ class Item_func_conv_charset :public Item_str_func use_cached_value= 1; str_value.mark_as_const(); safe= (errors == 0); + /* + The bytes are settled here and never computed again, so what is + said about them has to be settled here too. Asking the argument + later would pair an answer about a value it has since gone on to + produce with the one frozen above. + */ + if (!null_value && safe && cs != &my_charset_bin) + m_marks.set(&str_value, args[0]->is_valid_json(), + args[0]->is_nice_json()); } else { @@ -2069,6 +2093,8 @@ class Item_func_conv_charset :public Item_str_func static LEX_CSTRING name= {STRING_WITH_LEN("convert") }; return name; } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } void print(String *str, enum_query_type query_type) override; int save_in_field(Field*, bool) override; From 6641089ecab8ed3b23e9091c02bf2cf4c87b0d6c Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Tue, 4 Aug 2026 01:28:11 -0400 Subject: [PATCH 09/30] MDEV-40615 Copy a value out of a document instead of writing it again In a character set that spends more than one byte on a character, `JSON_EXTRACT()` returned NULL for documents it had just read successfully, and complained about a character the document did not contain: SET collation_connection='utf16_bin'; SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a'); NULL Warning 4036 Character disallowed in JSON in argument 1 to function 'json_extract' at position 2 `JSON_ARRAY_APPEND()` did the same, but only when the path led to something that was not an array and had to be wrapped in one first. Appending to an array that was already there worked - the same function, in the same statement, in the same character set. **What was happening** `String::append(const char *, size_t)` converts what it is given from `latin1` whenever the string being appended to is one a character cannot fit into a single byte. That is right for text that really is `latin1`, and it is what puts a proper wide bracket into a `ucs2` result when a bracket is written. It is wrong for a run of bytes cut out of the document, because those bytes are already in the document's character set, which is the one the result is being built in. They were being encoded a second time. The single character `1` is `0031` in `utf16`. Its two bytes were read as the `latin1` characters U+0000 and U+0031 and written out again as `00000031` - a NUL, and then the digit. Both functions read their whole result back before returning it, met the NUL, and gave up. So what the caller saw was not the corruption but a complaint naming their own argument and a position inside it. The character it named had been put there by the server a few lines earlier, which is why this survived being looked at: MDEV-34143 rewrote the loop around the `JSON_EXTRACT` call while fixing a crash in `chop()` under exactly these character sets, carried the call across unchanged, and recorded the NULL as the expected answer. **What is done instead** The three places that copy a piece of document now say so, with `append_simple()`, which puts the bytes down as they stand. The punctuation these functions write themselves - brackets, commas - is still appended the converting way, because it is written rather than copied and does have to be converted. `JSON_ARRAY_APPEND`'s other arm has always copied its two pieces of document correctly, with `q_append()`; the two arms now agree. Nothing changes in a character set of one byte, where the conversion was a copy anyway. **Not covered here** `Item_func_json_object_to_array`, which does not exist on this branch, copies both the key and the value of every pair the same way and wants the same treatment where it exists. The recorded answer from MDEV-34143 moves from NULL to the value, and `func_json_wide_charset` covers reading and editing at two and at four bytes to the character, keeps a one-byte character set alongside as a control, and checks that a document which is genuinely malformed is still refused and still has the fault located in what the caller sent. --- mysql-test/main/func_json.result | 4 +- mysql-test/main/func_json_wide_charset.result | 74 +++++++++++++++++++ mysql-test/main/func_json_wide_charset.test | 67 +++++++++++++++++ sql/item_jsonfunc.cc | 24 +++++- 4 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 mysql-test/main/func_json_wide_charset.result create mode 100644 mysql-test/main/func_json_wide_charset.test diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index 448e5c9043db8..6020e407a7184 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -1743,9 +1743,7 @@ SET @save_collation_connection= @@collation_connection; SET collation_connection='utf16_bin'; SELECT JSON_EXTRACT('{"a": 1,"b": 2}','$.a'); JSON_EXTRACT('{"a": 1,"b": 2}','$.a') -NULL -Warnings: -Warning 4036 Character disallowed in JSON in argument 1 to function 'json_extract' at position 2 +1 SET @@collation_connection= @save_collation_connection; # End of 10.5 tests # diff --git a/mysql-test/main/func_json_wide_charset.result b/mysql-test/main/func_json_wide_charset.result new file mode 100644 index 0000000000000..43855f3154aad --- /dev/null +++ b/mysql-test/main/func_json_wide_charset.result @@ -0,0 +1,74 @@ +SET @save_collation_connection= @@collation_connection; +# +# Reading a value out. The result holds the value itself, in the +# width the character set gives it. +# +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a')) AS scalar_bytes; +scalar_value scalar_bytes +1 0031 +SELECT JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a') AS object_value, +HEX(JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a')) AS object_bytes; +object_value object_bytes +{"x": 1} 007B002200780022003A00200031007D +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b') AS several_paths, +HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b')) AS several_bytes; +several_paths several_bytes +[1, 2] 005B0031002C00200032005D +SELECT JSON_VALID(JSON_EXTRACT('{"a":{"x":1}}','$.a')) AS result_is_a_document; +result_is_a_document +1 +# +# Appending to an array, and to something that is not one and has +# to be wrapped in one first. Both arms copy the document. +# +SELECT JSON_ARRAY_APPEND('{"a":[1,2]}','$.a',3) AS target_is_array; +target_is_array +{"a": [1, 2, 3]} +SELECT JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS target_is_scalar; +target_is_scalar +{"a": [1, 3]} +SELECT JSON_ARRAY_APPEND('{"a":{"x":1}}','$.a',3) AS target_is_object; +target_is_object +{"a": [{"x": 1}, 3]} +SELECT JSON_VALID(JSON_ARRAY_APPEND('{"a":1}','$.a',3)) AS result_is_a_document; +result_is_a_document +1 +# +# The same at two bytes to the character, and at four. +# +SET collation_connection='ucs2_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +SET collation_connection='utf32_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +# +# A character set of one byte answers as it always did. +# +SET collation_connection='utf8mb4_general_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +SET collation_connection='latin1_swedish_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +# +# A document that is genuinely malformed is still refused, and the +# position named is a position in what the caller sent. +# +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":oops}','$.a') AS malformed; +malformed +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 24 +SET @@collation_connection= @save_collation_connection; diff --git a/mysql-test/main/func_json_wide_charset.test b/mysql-test/main/func_json_wide_charset.test new file mode 100644 index 0000000000000..d425143e3e1cf --- /dev/null +++ b/mysql-test/main/func_json_wide_charset.test @@ -0,0 +1,67 @@ +# +# Reading and editing a document written in a character set whose +# characters are more than one byte wide. +# +# A value taken out of the document is already in the character set the +# result is being built in, so it has to be copied rather than written. +# Writing it converts it a second time, which used to turn the value +# into something that could not be read back - and, because it was read +# back, into NULL and a complaint about a character the caller never +# sent. +# +--disable_service_connection + +SET @save_collation_connection= @@collation_connection; + +--echo # +--echo # Reading a value out. The result holds the value itself, in the +--echo # width the character set gives it. +--echo # +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a')) AS scalar_bytes; +SELECT JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a') AS object_value, + HEX(JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a')) AS object_bytes; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b') AS several_paths, + HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b')) AS several_bytes; +SELECT JSON_VALID(JSON_EXTRACT('{"a":{"x":1}}','$.a')) AS result_is_a_document; + +--echo # +--echo # Appending to an array, and to something that is not one and has +--echo # to be wrapped in one first. Both arms copy the document. +--echo # +SELECT JSON_ARRAY_APPEND('{"a":[1,2]}','$.a',3) AS target_is_array; +SELECT JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS target_is_scalar; +SELECT JSON_ARRAY_APPEND('{"a":{"x":1}}','$.a',3) AS target_is_object; +SELECT JSON_VALID(JSON_ARRAY_APPEND('{"a":1}','$.a',3)) AS result_is_a_document; + +--echo # +--echo # The same at two bytes to the character, and at four. +--echo # +SET collation_connection='ucs2_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +SET collation_connection='utf32_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +--echo # +--echo # A character set of one byte answers as it always did. +--echo # +SET collation_connection='utf8mb4_general_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +SET collation_connection='latin1_swedish_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +--echo # +--echo # A document that is genuinely malformed is still refused, and the +--echo # position named is a position in what the caller sent. +--echo # +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":oops}','$.a') AS malformed; + +SET @@collation_connection= @save_collation_connection; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 82e12f43eff62..8ec83af33a9c8 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -1566,7 +1566,14 @@ String *Item_func_json_extract::read_json(String *str, goto error; while(count_path) { - if (str->append((const char *) value, v_len)) + /* + The value is copied, not written: it came out of the document + and is already in the character set the result is being built + in, so it goes in as it stands. The separators around it are + written, and are converted on the way, which is why they are + appended a different way from this. + */ + if (append_simple(str, value, v_len)) goto error; count_path--; if (count_path) @@ -2851,14 +2858,23 @@ String *Item_func_json_array_append::val_str(String *str) else c_to= je.value_end; + /* + The brackets and the comma are written, and go through a String + that converts them; the value and the rest of the document are + copied, and must not. They are already in the character set + being built in - they came out of the document, which is what + the result is being made from - so converting them writes them + a second time. The arm above copies its two pieces of document + the same way, with q_append(). + */ if (str->append('[') || - str->append((const char *) c_from, c_to - c_from) || + append_simple(str, c_from, c_to - c_from) || str->append(", ", 2) || append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), (int) n_arg + 1, NULL) || str->append(']') || - str->append((const char *) je.s.c_str, - js->end() - (const char *) je.s.c_str)) + append_simple(str, je.s.c_str, + js->end() - (const char *) je.s.c_str)) goto return_null; /* Out of memory. */ } { From 5501ce4547ae5106bc9d162109dfa22cecc9c358 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Tue, 4 Aug 2026 13:29:42 -0400 Subject: [PATCH 10/30] MDEV-40625 Copy a document that is taken over, instead of pointing at it `JSON_MERGE_PATCH()` returned NULL, and blamed an argument holding no JSON text at all, for a statement with nothing wrong in it: SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}')); NULL Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 1 Argument 1 is the four characters `NULL`. Writing the same two documents out as literals instead of building them gave the answer they ask for, `{"aaaaaaaaaaaaaaaa": 1}`. **What was happening** When everything before it was NULL, the function takes a whole document over rather than merging into one - unless it is an object, which is merged. It took it over by pointing at the buffer it came in, and that buffer is the one every argument is read into. The next argument was then read into it, a buffer that has to grow to hold one is not the buffer it was, and the document taken over was read from where it used to be. It had been released by then. Which is why the formatting of the argument decided the answer. Whether an argument is read into the caller's buffer at all is a property of the function that produced it: `LOWER`, `UPPER`, `CONCAT` and `REVERSE` build an answer and write into it; `TRIM` and `SUBSTRING` return a piece of what they were given and leave it alone; a plain column reference and a literal never touch it. So the function was right or wrong according to how its arguments happened to be written, and every test in the tree writes them as literals. **What is done instead** The document is copied. A copy that cannot be made is a failure like any other and takes the failure path. `func_json_merge_patch_adopt` covers taking over a document of each kind that is not an object, arguments written both ways, room that has to be found rather than grown into, and further arguments after the one taken over. Two controls sit alongside: the same second argument reached without a take-over, and a second argument no longer than the first. --- .../main/func_json_merge_patch_adopt.result | 77 +++++++++++++++++++ .../main/func_json_merge_patch_adopt.test | 65 ++++++++++++++++ sql/item_jsonfunc.cc | 9 ++- 3 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 mysql-test/main/func_json_merge_patch_adopt.result create mode 100644 mysql-test/main/func_json_merge_patch_adopt.test diff --git a/mysql-test/main/func_json_merge_patch_adopt.result b/mysql-test/main/func_json_merge_patch_adopt.result new file mode 100644 index 0000000000000..334a1e5e5a77d --- /dev/null +++ b/mysql-test/main/func_json_merge_patch_adopt.result @@ -0,0 +1,77 @@ +# the answer must not depend on how the arguments are produced +SELECT JSON_MERGE_PATCH(NULL, '[1]', '{"aaaaaaaaaaaaaaaa":1}') AS literal; +literal +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS built; +built +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, CONCAT('[', '1]'), +CONCAT('{"aaaaaaaaaaaaaaaa"', ':1}')) AS concatenated; +concatenated +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, REVERSE(']1['), +REVERSE('}1:"aaaaaaaaaaaaaaaa"{')) AS reversed; +reversed +{"aaaaaaaaaaaaaaaa": 1} +# a taken-over document of every kind that is not an object, since +# an object is merged rather than taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_number; +was_number +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('"S"'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_string; +was_string +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('TRUE'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_true; +was_true +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2]'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_array; +was_array +{"aaaaaaaaaaaaaaaa": 1} +# room that has to be found rather than grown into +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), +LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}'))) +IS NULL AS big_jump_null; +big_jump_null +0 +SELECT JSON_LENGTH(JSON_MERGE_PATCH(NULL, LOWER('1'), +LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}')))) AS big_len; +big_len +1 +# more arguments after the one that was taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), +LOWER('{"BBBBBBBBBBBBBBBB":2}')) AS three; +three +{"aaaaaaaaaaaaaaaa": 1, "bbbbbbbbbbbbbbbb": 2} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), +LOWER('{"AAAAAAAAAAAAAAAA":9}')) AS three_same_key; +three_same_key +{"aaaaaaaaaaaaaaaa": 9} +# out of a table, so the arguments change from row to row +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES +(1, '[1]', '{"aaaaaaaaaaaaaaaa":1}'), +(2, '[2]', '{"bbbbbbbbbbbbbbbb":2}'); +SELECT id, JSON_MERGE_PATCH(NULL, a, b) AS plain FROM t ORDER BY id; +id plain +1 {"aaaaaaaaaaaaaaaa": 1} +2 {"bbbbbbbbbbbbbbbb": 2} +SELECT id, JSON_MERGE_PATCH(NULL, LOWER(a), LOWER(b)) AS built FROM t +ORDER BY id; +id built +1 {"aaaaaaaaaaaaaaaa": 1} +2 {"bbbbbbbbbbbbbbbb": 2} +DROP TABLE t; +# controls: the same second argument reached without a taken-over +# first one, and a second argument no longer than the first +SELECT JSON_MERGE_PATCH('{}', LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS no_adoption; +no_adoption +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2,3,4,5,6,7,8,9]'), +LOWER('{"A":1}')) AS shorter_after; +shorter_after +{"a": 1} diff --git a/mysql-test/main/func_json_merge_patch_adopt.test b/mysql-test/main/func_json_merge_patch_adopt.test new file mode 100644 index 0000000000000..0dde50f0136d7 --- /dev/null +++ b/mysql-test/main/func_json_merge_patch_adopt.test @@ -0,0 +1,65 @@ +# +# JSON_MERGE_PATCH taking a whole document over when everything before it +# was NULL. +# +# The document is taken over from the buffer the argument was read into, +# and the argument after it is read into that same buffer. An argument +# that needs more room than the one before leaves that buffer somewhere +# else, so a document taken over by being pointed at is gone by the time +# it comes to be read. +# +# Whether an argument is read into the caller's buffer at all depends on +# the function that produced it - one that returns a piece of its own +# argument does not, one that builds its answer does. So the cases below +# come in pairs which differ only in how the same value is produced, and +# both have to give the same answer. +# + +--echo # the answer must not depend on how the arguments are produced +SELECT JSON_MERGE_PATCH(NULL, '[1]', '{"aaaaaaaaaaaaaaaa":1}') AS literal; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS built; +SELECT JSON_MERGE_PATCH(NULL, CONCAT('[', '1]'), + CONCAT('{"aaaaaaaaaaaaaaaa"', ':1}')) AS concatenated; +SELECT JSON_MERGE_PATCH(NULL, REVERSE(']1['), + REVERSE('}1:"aaaaaaaaaaaaaaaa"{')) AS reversed; + +--echo # a taken-over document of every kind that is not an object, since +--echo # an object is merged rather than taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_number; +SELECT JSON_MERGE_PATCH(NULL, LOWER('"S"'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_string; +SELECT JSON_MERGE_PATCH(NULL, LOWER('TRUE'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_true; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2]'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_array; + +--echo # room that has to be found rather than grown into +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), + LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}'))) + IS NULL AS big_jump_null; +SELECT JSON_LENGTH(JSON_MERGE_PATCH(NULL, LOWER('1'), + LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}')))) AS big_len; + +--echo # more arguments after the one that was taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), + LOWER('{"BBBBBBBBBBBBBBBB":2}')) AS three; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), + LOWER('{"AAAAAAAAAAAAAAAA":9}')) AS three_same_key; + +--echo # out of a table, so the arguments change from row to row +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES + (1, '[1]', '{"aaaaaaaaaaaaaaaa":1}'), + (2, '[2]', '{"bbbbbbbbbbbbbbbb":2}'); +SELECT id, JSON_MERGE_PATCH(NULL, a, b) AS plain FROM t ORDER BY id; +SELECT id, JSON_MERGE_PATCH(NULL, LOWER(a), LOWER(b)) AS built FROM t + ORDER BY id; +DROP TABLE t; + +--echo # controls: the same second argument reached without a taken-over +--echo # first one, and a second argument no longer than the first +SELECT JSON_MERGE_PATCH('{}', LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS no_adoption; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2,3,4,5,6,7,8,9]'), + LOWER('{"A":1}')) AS shorter_after; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 8ec83af33a9c8..ce6673053b341 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -3737,7 +3737,14 @@ String *Item_func_json_merge_patch::val_str(String *str) goto cont_point; merge_to_null= false; - str->set(js2->ptr(), js2->length(), js2->charset()); + /* + Taken over by copying it rather than by pointing at it. The next + argument is read into the very buffer this one came in, and a + buffer that has to grow to hold it is not the buffer it was: what + was pointed at is gone by the time it comes to be read. + */ + if (str->copy(js2->ptr(), js2->length(), js2->charset())) + goto error_return; goto cont_point; } From 4510d82f68c4178aa2a5f8d51aed29fcb6ce0839 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Tue, 4 Aug 2026 14:40:19 -0400 Subject: [PATCH 11/30] MDEV-40628 Say when text stands after a merge argument's value `JSON_MERGE()` and `JSON_MERGE_PATCH()` took a document argument holding text that is no part of any document, and said nothing: SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}'); {"a": 1, "b": 2} `JSON_VALID()` returns 0 about those same characters, and `JSON_INSERT()`, `JSON_REMOVE()`, `JSON_EXTRACT()` and `JSON_TYPE()` all return NULL with a warning. So one column value was a document to two functions and not a document to the rest, and nothing said which of the two answers was being given. What stands there can be a whole second document, dropped just as quietly: SELECT JSON_MERGE_PATCH('{"a":1} {"zzz":9}', '{"b":2}'); {"a": 1, "b": 2} **Why these two and not the others** Both read a document argument only as far as its first value, and compose the answer out of that value alone. The other functions that edit a document copy the rest of it into the result untouched, so the text lands there and the reading back at the end meets it. These two never copy it, so there is nothing for that reading to meet: it reads what was composed, and what was composed holds only what was scanned. **What is done instead** The reading is carried on to the end of the argument, and what it finds is said. It is carried on from where the merging left off rather than started again, so an argument with nothing after its value pays one step and no more, and it is carried on over a copy of the engine, so that a reading done to find out whether to say something cannot become the reason anything else is said. It is not carried on at all where the argument has already answered for itself. Being a document is a statement about the whole of a value, so one with anything standing after it is not one: where `is_valid_json()` is true - the same question this code asks a few lines above, to decide whether to splice - the reading could only come back saying there is nothing there. The taken-over argument is the one that would otherwise pay a full walk of a whole document on every row, that walk being the only reading this path has ever made. Said before it is settled whether the argument goes into the answer at all. Every shape of argument merged onto SQL NULL is taken over, except an object, which contributes nothing and is dropped; being dropped makes what stands after it no more a document than it was, and the very same characters are spoken for in every other argument position. So the sentence stands above the early return that handles the object and both arms reach it, while the marks stay below it, they being about a document that was taken over and an object is not. A note, not a warning. A warning becomes an error under a strict mode and would take away an answer that has always been given back. `JSON_VALID()` already says the same thing the same way, and for the same reason. Only an argument can be named. After the first turn of the loop the document on the left is what the function itself composed, so the left is only spoken about while it is still argument 1. What comes of that is a position that is a real offset into the text the caller wrote, and it is the position every other function names for the same bytes: `JSON_EXTRACT` says 8 for `{"a":1 "b":2}`, and so does the merge. The block over the reporting helper says both of the things it is for: text standing after a value, and an argument the merging broke off inside, the merging composing out of what it had and handing the engine over still carrying the refusal. **What is deliberately left alone** The key loops still swallow an error their condition hides, so a broken argument does not fail the merge. A released server answers SELECT JSON_MERGE('{}','{"a":1 "b":2}'); {"a": 1} and failing the merge would answer NULL instead, which is the one transition nothing here may make. Measured against a released server, every answer and every mark given for a broken merge argument is already identical to it; the notes are the whole of the difference. `func_json_merge_trailing` covers text after the value of either argument and of both, three arguments with the middle one carrying it, a second whole document standing there, and the taken-over case in each of its two outcomes. The object arm has all its shapes - merged over, empty, followed by a whole document of its own, twice in one statement, and as the last word; without the sentence above the early return all five of those answer with no diagnostic at all, and with it each names its argument and its position, the two-argument one naming both 2 and 3. A section compares positions with `JSON_EXTRACT`, pinning that agreement rather than proving it, and pins the shapes that simply stop as well. Alongside them sit the controls that have to stay silent: trailing whitespace, well formed arguments, arguments that are not documents from their first character, and NULL. One more holds the note to being a note - the same statement inside a `STRICT_ALL_TABLES` insert, which says it and still stores the row. `func_json_columns` gains the two notes its row 12 has always had coming. --- mysql-test/main/func_json_columns.result | 2 + .../main/func_json_merge_trailing.result | 276 ++++++++++++++++++ mysql-test/main/func_json_merge_trailing.test | 150 ++++++++++ sql/item_jsonfunc.cc | 90 ++++++ 4 files changed, 518 insertions(+) create mode 100644 mysql-test/main/func_json_merge_trailing.result create mode 100644 mysql-test/main/func_json_merge_trailing.test diff --git a/mysql-test/main/func_json_columns.result b/mysql-test/main/func_json_columns.result index 770491e1f88f4..17ed0d5b50039 100644 --- a/mysql-test/main/func_json_columns.result +++ b/mysql-test/main/func_json_columns.result @@ -347,6 +347,7 @@ id v Warnings: Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 32 SELECT id, JSON_MERGE_PATCH(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; id v @@ -357,6 +358,7 @@ id v Warnings: Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_patch' Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_patch' at position 32 SELECT id, JSON_COMPACT(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v diff --git a/mysql-test/main/func_json_merge_trailing.result b/mysql-test/main/func_json_merge_trailing.result new file mode 100644 index 0000000000000..9a4aceca009cd --- /dev/null +++ b/mysql-test/main/func_json_merge_trailing.result @@ -0,0 +1,276 @@ +# --------------------------------------------------------------- +# JSON_MERGE_PATCH, ordinary merge - nothing is taken over here +# --------------------------------------------------------------- +# text after the value of argument 1 +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}') AS obj_lhs; +obj_lhs +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE_PATCH('[1,2] rubbish', '{"a":1}') AS arr_lhs; +arr_lhs +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 7 +# text after the value of argument 2 +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish') AS obj_rhs; +obj_rhs +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE_PATCH('{"a":1}', '[1,2] rubbish') AS arr_rhs; +arr_rhs +[1, 2] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +# both arguments, so both are named +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2} nonsense') AS both_args; +both_args +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# three arguments, the middle one carrying it +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish', '{"c":3}') AS middle; +middle +{"a": 1, "b": 2, "c": 3} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# what stands after the value is a whole document of its own, which +# goes nowhere +SELECT JSON_MERGE_PATCH('{"a":1} {"zzz":9}', '{"b":2}') AS second_doc; +second_doc +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +# --------------------------------------------------------------- +# JSON_MERGE_PATCH taking a document over, everything before NULL +# --------------------------------------------------------------- +# taken over and then merged over, so the answer never holds it +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish', '{"a":1}') AS adopt_merged; +adopt_merged +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +SELECT JSON_MERGE_PATCH(NULL, '1 rubbish', '{"a":1}') AS adopt_scalar; +adopt_scalar +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 3 +SELECT JSON_MERGE_PATCH(NULL, '"s" rubbish', '{"a":1}') AS adopt_string; +adopt_string +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 5 +SELECT JSON_MERGE_PATCH(NULL, '[1,2] [9]', '{"a":1}') AS adopt_second_doc; +adopt_second_doc +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +# taken over and left as the answer, which is refused as it always +# was - the warning that refuses it is not replaced by a note +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish') AS adopt_final; +adopt_final +NULL +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 7 +# an object is the one shape that is not taken over: merged onto +# SQL NULL it contributes nothing and is dropped. Being dropped is +# no reason to say less about it than about the shapes that are +# kept, the characters standing after it being the same characters +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '[3,4]') AS drop_obj; +drop_obj +[3, 4] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE_PATCH(NULL, '{} rubbish', '[3,4]') AS drop_empty_obj; +drop_empty_obj +[3, 4] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 4 +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} {"zzz":9}', '[3,4]') AS drop_obj_doc; +drop_obj_doc +[3, 4] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# two of them, so both are named and one saying it does not stand +# for the other +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '{"b":2} nonsense') +AS drop_two; +drop_two +NULL +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_merge_patch' at position 9 +# and where the dropped object is the last word, so the answer is +# SQL NULL and there is no document left to refuse +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish') AS drop_obj_final; +drop_obj_final +NULL +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# --------------------------------------------------------------- +# JSON_MERGE +# --------------------------------------------------------------- +SELECT JSON_MERGE('[1,2] rubbish', '{"a":1}') AS merge_lhs; +merge_lhs +[1, 2, {"a": 1}] +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 7 +SELECT JSON_MERGE('[1,2]', '{"a":1} rubbish') AS merge_rhs; +merge_rhs +[1, 2, {"a": 1}] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2} nonsense') AS merge_both; +merge_both +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE('[1]', '[2] rubbish', '[3]') AS merge_middle; +merge_middle +[1, 2, 3] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 5 +# --------------------------------------------------------------- +# controls - nothing here may say anything +# --------------------------------------------------------------- +# an object dropped onto SQL NULL with nothing standing after it +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}', '[3,4]') AS drop_obj_clean; +drop_obj_clean +[3, 4] +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS drop_obj_clean_final; +drop_obj_clean_final +NULL +# space after the value is not text after the value +SELECT JSON_MERGE_PATCH('{"a":1} ', '{"b":2}') AS trailing_space; +trailing_space +{"a": 1, "b": 2} +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} + ') AS trailing_ws; +trailing_ws +{"a": 1, "b": 2} +SELECT JSON_MERGE('[1] ', '[2] ') AS merge_trailing_space; +merge_trailing_space +[1, 2] +# ordinary well formed arguments +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS clean_patch; +clean_patch +{"a": 1, "b": 2} +SELECT JSON_MERGE('[1]', '[2]') AS clean_merge; +clean_merge +[1, 2] +# an argument that is not a document from its first character has +# always been refused, and still is, with a warning and not a note +SELECT JSON_MERGE_PATCH('rubbish', '{"b":2}') AS bad_lhs; +bad_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 1 +SELECT JSON_MERGE_PATCH('{"a":1}', 'rubbish') AS bad_rhs; +bad_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 1 +SELECT JSON_MERGE('rubbish', '[2]') AS merge_bad_lhs; +merge_bad_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 1 +# NULL arguments say nothing either +SELECT JSON_MERGE_PATCH(NULL, NULL) AS both_null; +both_null +NULL +SELECT JSON_MERGE('[1]', NULL) AS merge_null; +merge_null +NULL +# --------------------------------------------------------------- +# out of a table, so the arguments change from row to row and one +# row saying it does not settle what another row says +# --------------------------------------------------------------- +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES +(1, '{"a":1} rubbish', '{"b":2}'), +(2, '{"a":1}', '{"b":2} nonsense'), +(3, '{"a":1}', '{"b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +id v +1 {"a": 1, "b": 2} +2 {"a": 1, "b": 2} +3 {"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +id v +1 {"a": 1, "b": 2} +2 {"a": 1, "b": 2} +3 {"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +DROP TABLE t; +# --------------------------------------------------------------- +# where the trouble is said to be, which has to be where every +# other JSON function reading the same bytes says it is +# --------------------------------------------------------------- +# broken inside the value rather than after it, so the merging +# stops in the middle and composes out of what it had +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS extract_says; +extract_says +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS merge_says; +merge_says +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_says; +patch_says +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +# deeper in, so the position is further along +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS extract_deep; +extract_deep +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS merge_deep; +merge_deep +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 13 +# a document that simply stops, which has no position to give +SELECT JSON_EXTRACT('{"a":1,', '$') AS extract_stops; +extract_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_MERGE('{"a":1,', '{}') AS merge_stops_lhs; +merge_stops_lhs +{"a": 1} +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +SELECT JSON_MERGE('{}', '{"a":1,') AS merge_stops_rhs; +merge_stops_rhs +{"a": 1} +Warnings: +Note 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +# --------------------------------------------------------------- +# a note does not become an error where a warning would +# --------------------------------------------------------------- +CREATE TABLE t (v JSON); +SET @sm= @@sql_mode; +SET sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t VALUES (JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}')); +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +SET sql_mode=@sm; +SELECT * FROM t; +v +{"a": 1, "b": 2} +DROP TABLE t; diff --git a/mysql-test/main/func_json_merge_trailing.test b/mysql-test/main/func_json_merge_trailing.test new file mode 100644 index 0000000000000..8167faa5e2dcb --- /dev/null +++ b/mysql-test/main/func_json_merge_trailing.test @@ -0,0 +1,150 @@ +# +# JSON_MERGE and JSON_MERGE_PATCH read a document argument only as far as +# its first value. What they compose comes out of that value and nothing +# else, so text standing after it never reaches the answer and never +# stopped one being given. +# +# It is still not a document, and every other JSON function says so about +# the very same text. These two say so as well, with a note. A note and +# not a warning, because a warning becomes an error under a strict mode +# and would take away an answer that has always been given. +# +# So every case below has to keep the value it has always had. The note +# is the only thing that is new. +# + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH, ordinary merge - nothing is taken over here +--echo # --------------------------------------------------------------- + +--echo # text after the value of argument 1 +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}') AS obj_lhs; +SELECT JSON_MERGE_PATCH('[1,2] rubbish', '{"a":1}') AS arr_lhs; + +--echo # text after the value of argument 2 +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish') AS obj_rhs; +SELECT JSON_MERGE_PATCH('{"a":1}', '[1,2] rubbish') AS arr_rhs; + +--echo # both arguments, so both are named +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2} nonsense') AS both_args; + +--echo # three arguments, the middle one carrying it +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish', '{"c":3}') AS middle; + +--echo # what stands after the value is a whole document of its own, which +--echo # goes nowhere +SELECT JSON_MERGE_PATCH('{"a":1} {"zzz":9}', '{"b":2}') AS second_doc; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH taking a document over, everything before NULL +--echo # --------------------------------------------------------------- + +--echo # taken over and then merged over, so the answer never holds it +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish', '{"a":1}') AS adopt_merged; +SELECT JSON_MERGE_PATCH(NULL, '1 rubbish', '{"a":1}') AS adopt_scalar; +SELECT JSON_MERGE_PATCH(NULL, '"s" rubbish', '{"a":1}') AS adopt_string; +SELECT JSON_MERGE_PATCH(NULL, '[1,2] [9]', '{"a":1}') AS adopt_second_doc; + +--echo # taken over and left as the answer, which is refused as it always +--echo # was - the warning that refuses it is not replaced by a note +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish') AS adopt_final; + +--echo # an object is the one shape that is not taken over: merged onto +--echo # SQL NULL it contributes nothing and is dropped. Being dropped is +--echo # no reason to say less about it than about the shapes that are +--echo # kept, the characters standing after it being the same characters +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '[3,4]') AS drop_obj; +SELECT JSON_MERGE_PATCH(NULL, '{} rubbish', '[3,4]') AS drop_empty_obj; +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} {"zzz":9}', '[3,4]') AS drop_obj_doc; + +--echo # two of them, so both are named and one saying it does not stand +--echo # for the other +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '{"b":2} nonsense') + AS drop_two; + +--echo # and where the dropped object is the last word, so the answer is +--echo # SQL NULL and there is no document left to refuse +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish') AS drop_obj_final; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE +--echo # --------------------------------------------------------------- + +SELECT JSON_MERGE('[1,2] rubbish', '{"a":1}') AS merge_lhs; +SELECT JSON_MERGE('[1,2]', '{"a":1} rubbish') AS merge_rhs; +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2} nonsense') AS merge_both; +SELECT JSON_MERGE('[1]', '[2] rubbish', '[3]') AS merge_middle; + +--echo # --------------------------------------------------------------- +--echo # controls - nothing here may say anything +--echo # --------------------------------------------------------------- + +--echo # an object dropped onto SQL NULL with nothing standing after it +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}', '[3,4]') AS drop_obj_clean; +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS drop_obj_clean_final; + +--echo # space after the value is not text after the value +SELECT JSON_MERGE_PATCH('{"a":1} ', '{"b":2}') AS trailing_space; +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} + ') AS trailing_ws; +SELECT JSON_MERGE('[1] ', '[2] ') AS merge_trailing_space; + +--echo # ordinary well formed arguments +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS clean_patch; +SELECT JSON_MERGE('[1]', '[2]') AS clean_merge; + +--echo # an argument that is not a document from its first character has +--echo # always been refused, and still is, with a warning and not a note +SELECT JSON_MERGE_PATCH('rubbish', '{"b":2}') AS bad_lhs; +SELECT JSON_MERGE_PATCH('{"a":1}', 'rubbish') AS bad_rhs; +SELECT JSON_MERGE('rubbish', '[2]') AS merge_bad_lhs; + +--echo # NULL arguments say nothing either +SELECT JSON_MERGE_PATCH(NULL, NULL) AS both_null; +SELECT JSON_MERGE('[1]', NULL) AS merge_null; + +--echo # --------------------------------------------------------------- +--echo # out of a table, so the arguments change from row to row and one +--echo # row saying it does not settle what another row says +--echo # --------------------------------------------------------------- + +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES + (1, '{"a":1} rubbish', '{"b":2}'), + (2, '{"a":1}', '{"b":2} nonsense'), + (3, '{"a":1}', '{"b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +DROP TABLE t; + +--echo # --------------------------------------------------------------- +--echo # where the trouble is said to be, which has to be where every +--echo # other JSON function reading the same bytes says it is +--echo # --------------------------------------------------------------- + +--echo # broken inside the value rather than after it, so the merging +--echo # stops in the middle and composes out of what it had +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS extract_says; +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS merge_says; +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_says; + +--echo # deeper in, so the position is further along +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS extract_deep; +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS merge_deep; + +--echo # a document that simply stops, which has no position to give +SELECT JSON_EXTRACT('{"a":1,', '$') AS extract_stops; +SELECT JSON_MERGE('{"a":1,', '{}') AS merge_stops_lhs; +SELECT JSON_MERGE('{}', '{"a":1,') AS merge_stops_rhs; + +--echo # --------------------------------------------------------------- +--echo # a note does not become an error where a warning would +--echo # --------------------------------------------------------------- + +CREATE TABLE t (v JSON); +SET @sm= @@sql_mode; +SET sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t VALUES (JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}')); +SET sql_mode=@sm; +SELECT * FROM t; +DROP TABLE t; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index ce6673053b341..6dbad93abf7e8 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -3383,6 +3383,46 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) } +/* + Says what is wrong with a document argument that was read only as far + as its first value, when anything is. + + These two functions read a document argument no further than that. + What they compose comes out of that value and nothing else, so neither + text standing after it nor a break the reading never got as far as + reaches the answer, and neither has ever stopped an answer being + given. What is there is still not a document, and every other JSON + function says so about the very same characters. + + Saying so here takes nothing away, which is why it is a note. A + warning becomes an error under a strict mode, and that would take away + an answer that has always been given back. An argument the merging + broke off inside arrives here already refused, and is reported from + where it was refused: an engine that has been refused does not move + again, so carrying the reading on cannot relocate the complaint. + + The reading is carried on from where the merging left off rather than + started again, so an argument with nothing after its value pays one + step and no more. It is carried on over a copy, because a reading + done to find out whether to say something must not be the reason + anything else is said: the caller reports its own failures out of its + own engine, and this must leave that engine where it found it. +*/ +static void report_json_trailing_note(const json_engine_t *je, + const String *js, + const char *fname, int n_param) +{ + json_engine_t tail= *je; + + while (json_scan_next(&tail) == 0) + /* There is nothing left to compose from, only to read. */; + + if (tail.s.error) + report_json_error_ex(js->ptr(), &tail, fname, n_param, + Sql_condition::WARN_LEVEL_NOTE); +} + + String *Item_func_json_merge::val_str(String *str) { DBUG_ASSERT(fixed()); @@ -3419,6 +3459,16 @@ String *Item_func_json_merge::val_str(String *str) if (do_merge(str, &je1, &je2)) goto error_return; + /* + Both were read as far as their first value and no further. Only + the first turn of the loop has a document argument on the left: + after it, the left is what this function itself composed, and + nothing stands after that. + */ + if (n_arg == 1) + report_json_trailing_note(&je1, js1, func_name(), 0); + report_json_trailing_note(&je2, js2, func_name(), (int) n_arg); + { /* Swap str and js1. */ if (str == &tmp_js1) @@ -3733,6 +3783,35 @@ String *Item_func_json_merge_patch::val_str(String *str) { if (json_read_value(&je2)) goto error_return; + + /* + Said before it is settled whether the argument goes into the + answer at all. An object merged onto SQL NULL contributes + nothing and is dropped just below, and being dropped makes what + stands after it no more a document than it was; the very same + characters are spoken for in every other argument position. + + Only the first value was read, so getting to the end of it comes + first here, where the merging below does it for itself. A value + that cannot be got to the end of is one the merging will report + on in its own words; there is nothing to add to that. + + Not asked at all of a value already attested to. + Being a document is a statement about the whole of it - a value + with anything standing after it is not one - so the walk could + only ever come back saying what is already known, and it is the + walk this whole exercise is about not making. It is the one the + released server did not make either: it took the first value and + copied the rest in unread. + */ + if (!args[n_arg]->is_valid_json()) + { + json_engine_t adopted= je2; + + if (json_value_scalar(&adopted) || !json_skip_level(&adopted)) + report_json_trailing_note(&adopted, js2, func_name(), (int) n_arg); + } + if (je2.value_type == JSON_VALUE_OBJECT) goto cont_point; @@ -3745,6 +3824,7 @@ String *Item_func_json_merge_patch::val_str(String *str) */ if (str->copy(js2->ptr(), js2->length(), js2->charset())) goto error_return; + goto cont_point; } @@ -3759,6 +3839,16 @@ String *Item_func_json_merge_patch::val_str(String *str) if (do_merge_patch(str, &je1, &je2, &empty_result)) goto error_return; + /* + Both were read as far as their first value and no further. Only + the first turn of the loop has a document argument on the left: + after it, the left is what this function itself composed, or a + document taken over, which was spoken for where it was taken. + */ + if (n_arg == 1) + report_json_trailing_note(&je1, js1, func_name(), 0); + report_json_trailing_note(&je2, js2, func_name(), (int) n_arg); + if (empty_result) str->append(STRING_WITH_LEN("null")); From d7e048df0ffd4d592c48b5b7e8c52bedfd1ec1ae Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Tue, 4 Aug 2026 22:26:52 -0400 Subject: [PATCH 12/30] Return a document instead of reading it again for its formatting Seven JSON functions ended by reading their own answer back: `json_scan_start()` followed by `json_nice(&je, str, LOOSE)`. That epilogue does **two** jobs - it writes the answer the loose way, and it refuses it if what was composed does not read as a document. Neither job is free: the whole answer is walked a second time, every time. The second walk is only worth its cost when something about the composition is unknown. This drops it where nothing is: 1. **The document was a document.** Every function that edits one copies the parts it is not changing out of an argument that attested `is_valid`, at offsets its own walk computed rather than guessed. 2. **The document was already formatted that way.** The argument attested `is_nice`, and the punctuation written here was written to match, `colon_len` being settled before anything was composed. 3. **The punctuation can be written at all.** A character set that puts national letters at the ASCII code points writes a bracket that is not a bracket, so `is_json_compatible_charset()` sends those answers through the reading back exactly as before. Where any of the three fails, the epilogue stays where it was and the answer is byte for byte what it has always been. The same question settles a second reading on the way IN. Four functions that only report on a document parsed it through to the end to find out whether it was one, on top of the reading that answers what was actually asked: - `JSON_EXTRACT` walked the rest of the document after the last path it was given had been matched; - `JSON_CONTAINS_PATH` carried its walk on past an answer already settled, so that a fault further along could take the answer back; - `JSON_LENGTH` walked the remainder once the length had been counted; - `JSON_TYPE` walked the whole document before reading the type of the first value in it. Where the item has attested `is_valid`, that walk can only report what is already known, so it is not made. `JSON_TYPE` keeps the order it had - the document is checked before a type is worked out, so a type is never given for text that turns out not to be a document - and asks `json_valid_engine()` for it only where nothing has attested. What follows is not that walk repeated: `json_read_value()` stops at the bracket that opens a container, so for anything but a bare scalar it is one step rather than a second walk. Measured over two rows, `JSON_TYPE` of a column costs four readings, and of an edit of that column six rather than eight. The other three carry the parse on inside one reading rather than starting another, so no count moves for them, and `func_json_scan_count` records that their answers are the same either way instead. The three are one question and are asked in one place, rather than written out again at each of the five functions that ask them. What each of them then DOES with the answer stays where it is; that part is about the function and not about the argument. Patching a document asks the question too: the colon it writes is three characters where the answer is returned and two where it is read again, so the width follows from the answer rather than standing in for it. `Item` gained two per-evaluation questions, `is_valid_json()` and `is_nice_json()`, each true or false and asked after the argument has been worked out. They are separate because a value can be a document and still not be written the loose way, which is the ordinary case for anything cut out of a document that was written some other way. `Item_field` overrides neither, so a column is never `is_valid`: the bytes in a row are whatever was put there. `Json_splice_marks` carries the same two for the values spliced in, and a function returns its answer only when the document, the punctuation and every splice all say yes. Returning it is one method the six functions that do it call, rather than the same seven lines written out six times. What they differ in is where they report a buffer that would not grow - some have a document engine to complain through and some do not - so the failure is returned to them. The debug hook that stands in for that failure is one site as well, reached from whichever of the six is running. A function that claims the loose formatting over an answer it does not read back has to write that formatting from the fact it is defined by, and not from a literal that happens to agree. `JSON_EXTRACT` writes both of its separators from `json_loose_comma` for that reason - between two values, and between the repeats of a value that answered more than one of the paths asked for. Nothing is wrong until the two disagree, so it is measured that way: with the constant temporarily made comma-tab, both `JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b')` and the same document read through `'$.a'` twice answer with a comma and a tab between, where a hand-written separator stops a debug build on the niceness half of the self-check. The loose form is written in one place. Reading an answer back to settle its formatting walked it a second time through a copy of the same state machine, and the two were required to go on agreeing byte for byte with nothing making them do it: what says whether a value is formatted the loose way is a check that measures it by writing it through one of the two, so the day they drifted apart, the drift is what would have been approved. How deep a walk over a value went is wanted by every caller there is, so the walk takes somewhere to put that answer rather than a pointer that may be nothing - and the tests against nothing, made in the middle of the walk, go with it. Nothing is taken on faith. `Json_result_marks::set()` reads the answer back in a **debug** build and stops the server if `is_valid` was claimed over something that does not read as a document, or `is_nice` over something that is not formatted that way. It is the same reading, kept exactly where it is worth its cost and nowhere else. It found eleven defects while this was being written. That reading is held off the kill a test arranges partway through a walk. A debug build makes one walk more than a release build - the one that writes a value out again to find out whether it was already written that way - and a kill raised in THAT walk would be a debug build ending a statement a release build finishes. The same reading is held off the count of readings, and for the same reason. Where the reading back does still happen, what it reads is text this function composed, and a position it complains about is an offset into that text. So the text has to be composed the way a released server composed it, character for character, even in the parts the reading is about to write over. `JSON_REMOVE` is the one that turns on this. It writes its single comma at the width a released server writes it, and the space standing behind that comma is asked the same question: skipped either way, the text read back is one character shorter than a released server's and every position complained about in it moves. Nothing that comes out of the function moves either way - the reading back writes the answer afresh and drops the space itself, which is how a released server comes to return `[2]` while composing `[ 2]`. It shows only where that reading fails, which is where the position is all there is: SELECT JSON_REMOVE('[1, 2] x', '$[0]'); names position 6, which is the position a released server names. A composition can be deeper than a document is allowed to be even when both sides of it fit, so each site now accounts for the levels it adds: - reading a container **pushes** the scanner's stack, so `JSON_ARRAY_APPEND` and `JSON_ARRAY_INSERT` were counting the target twice, and refused documents that fit; - an array or object is **popped before** the state that says it ended is set, so `JSON_INSERT` and `JSON_SET` were counting one level too few at the two arms that reach the end without finding the path; - where a level is added over text that was copied rather than read - the bracket `JSON_EXTRACT` puts round more than one match, the array a merge wraps a conflict in, the array an autowrap keeps a value inside - how deep that text already went is not known here, and the answer is read back rather than guessed at. `append_json_value()` writes the depth down before it looks at the value, because the value is going there whatever it turns out to be, and it asks whether that depth is past the limit in the same place - in front of every arm rather than inside the one that reads. A value put where the limit has already been reached is past it whatever it holds, and only the arm that reads can say how much further down it goes; a value written out as a string, a bare word and a null never reach that arm at all. The flag saying the marks were given up over the depth is kept only where the assertion that reads it is kept, nothing else ever looking at it. That is not the same set of builds as the debug one: an assertion survives into a build with `DBUG_OFF` set where it was asked to print rather than to stop, and the expression it is given is compiled there. The marks get a constructor of their own with it, a member that is not in every build not being one a brace-initialiser can name. The one way to be handed a depth past the limit is a function that WRAPS: it puts a new array where a value already stood, and the value going in beside it is counted from inside that new array. Appending gets there because its path names the value and the array it adds needs no step of its own, so a path of the full 31 steps still leaves room to add a 32nd structure. The two that wrap while INSERTING name the place they insert at, which takes the step the path cannot spare, and the path refuses them first. So SET @w31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); SET @wp31= CONCAT('$', REPEAT('[0]', 31)); SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, 2); composes 32 structures and, the value having been quoted rather than read, would otherwise be passed as an answer nobody had read and marked as one. Every function that can be handed such a depth composes into an answer it passes only while these marks stand, so taking them away puts it back on the reading a released server always did - and that reading gives the answer that server gives, in the words it has always used. Punctuation written here arrives as ASCII and is converted into the set being built in. Bytes copied out of a document are already in it, and putting them through a converting append writes them a second time - which in a set that spends more than one byte on a character produces something that is not a document at all. `do_merge()` was doing that to both of its array spans, and `do_merge_patch()` was looking at a single byte to decide whether it was standing on a space, which it never is in `ucs2`, `utf16` or `utf32`. Both now copy, and a new `json_skip_space()` reads one character and asks what it is. A character is taken off a string through the one formatting the character set offers, where two other formats of the same call had crept in. A value spliced into a document being built in another character set is converted first, the result being read in the set it is built in. A conversion that lost nothing kept the characters it was given, and being a document is a property of characters, so what was answered about the value still holds of what came out of it. The three questions above are asked after that conversion rather than before, where they could only ever be attested a value that needed none. A conversion that put a question mark in place of a character it could not write has changed the value, so the bytes that arrived go in unchanged - as they always have - and are read. Measured, a document attested in one character set and spliced into another falls from three readings to two, and two such values from five to three. `Json_scans` counts every reading of a value. It is counted in `json_scan_start()`, which is the one door they all go through: counting at the callers would mean keeping a list of them, and a list like that is wrong the day something is added to it. The readings this file makes only to check what it claimed go through that door as well, so each is counted like every other and taken back off again - by `json_scan_start_unbilled()` where the reading starts here, and by the same two steps by hand where it is borrowed from `json_valid()`. Measured over two rows, editing a document passed on from another JSON function falls from six readings to four, and a chain of three edits from ten to six. Nothing here owns the document. A function takes raw pointers into it, works out the rest of what it was given, and reads it afterwards. What working those out can write is not this function's to know: the document can be a table's row, a routine's variable or another statement's user variable. That the bytes stay where they are is true today because of the shapes an expression can take, and not because anything makes it so, which is exactly the kind of thing that stops being true quietly. So a debug build holds a copy from before the rest is worked out and says whether the document still matches when it is read, comparing the two runs of bytes with the one function there is for comparing two. A release build keeps nothing and asks nothing, this being a check on the code rather than on the data. It costs a wrong answer nothing and it costs a right one nothing either. It is taken wherever a document outlives the working out of anything else, which is every function that takes one and something besides. Which argument does the working out does not matter: an argument is worked out no earlier for being a path than for being a value, so `JSON_REMOVE`, which has no value argument at all, holds its document across every path it is handed, and so does the one that says whether to match one key or all of them. The four that compose in place use what they wrote as the document for the next path, so what the document IS changes between one pass and the next, on purpose. Theirs is taken again each time round rather than once at the start. Thirteen files, of which `func_json_trusted` and `func_json_wide_compose` are the ones that matter most: everything else in the suite writes its documents as string literals, which attest `is_valid` false, so the epilogue still runs and the arm this commit adds is never taken. Their answers were recorded by running them against a released server, so what they hold is what that server says rather than what this one does. Where a released server is not a fair judge - the wide character sets, already put right below this - each answer is instead compared against the same composition in `utf8`, which recording cannot quietly agree with. `func_json_trusted` records the two documents that lose a space to that skip beside three that have none to lose: one written without spacing, one where what goes is not the first piece, and one where it is a key. All five are the positions a released 10.11 server gives. `func_json_splice_depth` carries the wrapping cases: refused at 32 on both ways, answering at 31 on both, the same refusal reached through a value that is a document of its own, and the two inserting functions stopped by the path length instead. `func_json_splice_oom` fails the writing of a spliced value, which until now no injection could reach. It also fails the writes this commit leaves no reading back to cover: passing a document that was composed in place, which is a copy and can run short of room like any other write; the comma between the values a reading function found and the copy of each value itself; the key a path names that the document did not have; and the colon that joins a key to its value, which is written in the formatting settled here. Every argument in the returning cases is built by a producer that attests to what it passes. A string literal attests is_valid false for its document, so a statement built out of literals never composes an answer at all and would say nothing about any of them - however the literal is reformatted first, reformatting being unable to attest to what nothing attested. `func_json_walk_oom` fails the writing of the loose formatting itself, which is the other half of the same injection. A mutator asks for the room the document it was given takes and a little over, so the loose form, which spends a character on every comma and every colon, has a fixed amount of room and runs out at a place that depends on what is being written. The room is made to run out by the walk itself, where the writing is, rather than by a wrapper standing in front of it: a wrapper is a second door, and a caller going through the other one would have had the one writing of a value that no injection could fail, a writing whose handling of failure nobody has checked. The cases sweep a key length and a run of pairs past that place, so the failure lands on each of the appends the walk makes in turn, and every sweep is run both as a bare statement and as an assignment, the two landing it in different appends. Wherever it lands the answer either reads back as a document or is refused; nothing half written is returned. `func_json_emit_killed` arms a kill partway through a walk, and what sorts its statements is whether a walk happens at all rather than which function was asked. An answer composed from a document nobody attested is read back before it is passed; that reading writes as it goes, so it is a walk like any other and a killed query is let go of in it - as it is on a released server, which makes the same reading over an engine armed to hear a kill. `func_json_search_typed` says that `JSON_SEARCH` returns a document whatever it was asked. Every path it produces is built out of pieces of a document it has just read through and the punctuation round them is its own, so a column filled from it is a column of documents and a later reader is spared finding that out. It has both answer shapes, the branches of a union agreeing about it, a list of values written out in the statement, a table the server built for itself, and a column the statement named. `func_json_unfinished` reads a path counted from the end of an array in a document that stops in the middle. Such a step is not known until the array it is counted in has been counted, so the document is read through twice, and the counting pass can run off the end of the text; what it finds there is carried out of the loop rather than left to the answering pass, which would otherwise answer about an array it never finished counting. Every unfinished document is recorded beside the same document finished, which is what its answer is read against. --- include/json_lib.h | 8 + mysql-test/main/func_json_aliasing.result | 227 ++ mysql-test/main/func_json_aliasing.test | 173 ++ mysql-test/main/func_json_emit_killed.result | 160 ++ mysql-test/main/func_json_emit_killed.test | 182 ++ mysql-test/main/func_json_key_span.result | 155 ++ mysql-test/main/func_json_key_span.test | 103 + mysql-test/main/func_json_nice_compose.result | 287 +++ mysql-test/main/func_json_nice_compose.test | 304 +++ .../main/func_json_punct_charset.result | 94 + mysql-test/main/func_json_punct_charset.test | 70 + mysql-test/main/func_json_scan_count.result | 378 ++++ mysql-test/main/func_json_scan_count.test | 434 ++++ mysql-test/main/func_json_search_typed.result | 87 + mysql-test/main/func_json_search_typed.test | 75 + mysql-test/main/func_json_splice_depth.result | 143 ++ mysql-test/main/func_json_splice_depth.test | 118 ++ mysql-test/main/func_json_splice_oom.result | 240 +++ mysql-test/main/func_json_splice_oom.test | 200 ++ mysql-test/main/func_json_trusted.result | 301 +++ mysql-test/main/func_json_trusted.test | 230 +++ mysql-test/main/func_json_unfinished.result | 76 + mysql-test/main/func_json_unfinished.test | 44 + mysql-test/main/func_json_walk_oom.result | 75 + mysql-test/main/func_json_walk_oom.test | 188 ++ mysql-test/main/func_json_wide_compose.result | 154 ++ mysql-test/main/func_json_wide_compose.test | 152 ++ sql/item_jsonfunc.cc | 1831 +++++++++++++++-- sql/item_jsonfunc.h | 70 +- sql/mysqld.cc | 3 + sql/sql_class.h | 16 + strings/json_lib.c | 27 + 32 files changed, 6433 insertions(+), 172 deletions(-) create mode 100644 mysql-test/main/func_json_aliasing.result create mode 100644 mysql-test/main/func_json_aliasing.test create mode 100644 mysql-test/main/func_json_emit_killed.result create mode 100644 mysql-test/main/func_json_emit_killed.test create mode 100644 mysql-test/main/func_json_key_span.result create mode 100644 mysql-test/main/func_json_key_span.test create mode 100644 mysql-test/main/func_json_nice_compose.result create mode 100644 mysql-test/main/func_json_nice_compose.test create mode 100644 mysql-test/main/func_json_punct_charset.result create mode 100644 mysql-test/main/func_json_punct_charset.test create mode 100644 mysql-test/main/func_json_scan_count.result create mode 100644 mysql-test/main/func_json_scan_count.test create mode 100644 mysql-test/main/func_json_search_typed.result create mode 100644 mysql-test/main/func_json_search_typed.test create mode 100644 mysql-test/main/func_json_splice_depth.result create mode 100644 mysql-test/main/func_json_splice_depth.test create mode 100644 mysql-test/main/func_json_splice_oom.result create mode 100644 mysql-test/main/func_json_splice_oom.test create mode 100644 mysql-test/main/func_json_trusted.result create mode 100644 mysql-test/main/func_json_trusted.test create mode 100644 mysql-test/main/func_json_unfinished.result create mode 100644 mysql-test/main/func_json_unfinished.test create mode 100644 mysql-test/main/func_json_walk_oom.result create mode 100644 mysql-test/main/func_json_walk_oom.test create mode 100644 mysql-test/main/func_json_wide_compose.result create mode 100644 mysql-test/main/func_json_wide_compose.test diff --git a/include/json_lib.h b/include/json_lib.h index 53d6548694853..91053ffaf2e9f 100644 --- a/include/json_lib.h +++ b/include/json_lib.h @@ -273,6 +273,14 @@ typedef struct st_json_engine_t } json_engine_t; +#ifndef DBUG_OFF +/* + Told about every reading of a value, when somebody has asked to be. + See json_scan_start() for why this is where the telling happens. +*/ +extern void (*json_scan_start_hook)(void); +#endif + int json_scan_start(json_engine_t *je, CHARSET_INFO *i_cs, const uchar *str, const uchar *end); int json_scan_next(json_engine_t *j); diff --git a/mysql-test/main/func_json_aliasing.result b/mysql-test/main/func_json_aliasing.result new file mode 100644 index 0000000000000..5d8e44683174f --- /dev/null +++ b/mysql-test/main/func_json_aliasing.result @@ -0,0 +1,227 @@ +CREATE TABLE t1 (id INT, js VARCHAR(64), v VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"A":1,"B":2}', 'X'), (2, '{"A":3,"B":4}', 'Y'); +# +# 1. Document and value both built into the buffer offered them +# +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v)) AS ins FROM t1; +ins +{"A": 1, "B": 2, "c": "XX"} +{"A": 3, "B": 4, "c": "YY"} +SELECT JSON_INSERT(LOWER(js), '$.c', LOWER(v)) AS ins_low FROM t1; +ins_low +{"a": 1, "b": 2, "c": "x"} +{"a": 3, "b": 4, "c": "y"} +SELECT JSON_INSERT(REVERSE(REVERSE(js)), '$.c', UPPER(v)) AS ins_rev FROM t1; +ins_rev +{"A": 1, "B": 2, "c": "X"} +{"A": 3, "B": 4, "c": "Y"} +SELECT JSON_SET(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS st FROM t1; +st +{"A": "XX", "B": 2} +{"A": "YY", "B": 4} +SELECT JSON_REPLACE(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS rep FROM t1; +rep +{"A": "XX", "B": 2} +{"A": "YY", "B": 4} +SELECT JSON_REMOVE(CONCAT(js, ''), '$.A') AS rem FROM t1; +rem +{"B": 2} +{"B": 4} +SELECT JSON_ARRAY_APPEND(CONCAT('[', js, ']'), '$', CONCAT(v, v)) AS app +FROM t1; +app +[{"A": 1, "B": 2}, "XX"] +[{"A": 3, "B": 4}, "YY"] +SELECT JSON_ARRAY_INSERT(CONCAT('[', js, ']'), '$[0]', CONCAT(v, v)) AS ari +FROM t1; +ari +["XX", {"A": 1, "B": 2}] +["YY", {"A": 3, "B": 4}] +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mrg FROM t1; +mrg +{"A": 1, "B": 2, "c": "X"} +{"A": 3, "B": 4, "c": "Y"} +SELECT JSON_MERGE_PATCH(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mpt +FROM t1; +mpt +{"A": 1, "B": 2, "c": "X"} +{"A": 3, "B": 4, "c": "Y"} +# +# 2. More than one path, so the document read from on the second +# pass is the one the first pass wrote +# +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), +'$.d', CONCAT(v, v, v)) AS two_paths FROM t1; +two_paths +{"A": 1, "B": 2, "c": "XX", "d": "XXX"} +{"A": 3, "B": 4, "c": "YY", "d": "YYY"} +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), +'$.zz.deeper', CONCAT(v, v), +'$.d', CONCAT(v, v, v)) AS three_paths FROM t1; +three_paths +{"A": 1, "B": 2, "c": "XX", "d": "XXX"} +{"A": 3, "B": 4, "c": "YY", "d": "YYY"} +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}'), +CONCAT('{"d":"', v, '"}')) AS mrg3 FROM t1; +mrg3 +{"A": 1, "B": 2, "c": "X", "d": "X"} +{"A": 3, "B": 4, "c": "Y", "d": "Y"} +# +# 3. A value that grows a great deal, so a buffer holding it has +# to be found somewhere else +# +# These are the only cases here big enough to make a buffer +# move, and so the only ones that reach what this file is +# about. How long the answer is cannot be what they ask: +# copying out of a buffer that has moved brings back the same +# number of bytes it would have brought back intact, so a +# length is exactly the measure that survives the fault. They +# ask instead whether the answer still reads as a document, +# whether the member nobody touched came through, and what +# stands at both ends of the value that grew. +# +SELECT LENGTH(g) AS grown, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail +FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', REPEAT(v, 5000)) AS g +FROM t1) AS x; +grown parses kept_a kept_b value_head value_tail +5025 1 1 2 XXXXXXXX XXXXXXXX +5025 1 3 4 YYYYYYYY YYYYYYYY +SELECT LENGTH(g) AS mgrown, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail +FROM (SELECT JSON_MERGE(CONCAT(js, ''), +CONCAT('{"c":"', REPEAT(v, 5000), '"}')) AS g +FROM t1) AS x; +mgrown parses kept_a kept_b value_head value_tail +5025 1 1 2 XXXXXXXX XXXXXXXX +5025 1 3 4 YYYYYYYY YYYYYYYY +# +# 4. A user variable holding the document, written to by the +# expression that works out the value +# +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 30))) AS uvar; +uvar +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +SET @d= '{"a":1111111111}'; +SELECT LENGTH(g) AS uvar_big, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.a') AS kept_a, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail +FROM (SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 100000))) AS g) AS x; +uvar_big parses kept_a value_head value_tail +100026 1 1111111111 zzzzzzzz zzzzzzzz +SET @d= '[1111111111]'; +SELECT JSON_ARRAY_APPEND(@d, '$', (@d := REPEAT('z', 30))) AS uvar_app; +uvar_app +[1111111111, "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"] +SET @d= '{"a":1111111111}'; +SELECT JSON_MERGE(@d, (@d := '{"b":2}')) AS uvar_mrg; +uvar_mrg +{"a": 1111111111, "b": 2} +# +# 5. A routine's own variable holding the document, with the +# value worked out by calling something +# +CREATE FUNCTION f_grow(n INT) RETURNS TEXT +BEGIN +RETURN REPEAT('z', n); +END| +CREATE PROCEDURE p_edit() +BEGIN +DECLARE d TEXT DEFAULT '{"a":1111111111}'; +SELECT JSON_INSERT(d, '$.b', f_grow(30)) AS sp_ins; +SELECT LENGTH(g) AS sp_ins_big, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.a') AS kept_a, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail +FROM (SELECT JSON_INSERT(d, '$.b', f_grow(50000)) AS g) AS x; +SELECT JSON_INSERT(CONCAT(d, ''), '$.b', f_grow(30)) AS sp_ins_concat; +SELECT JSON_MERGE(d, CONCAT('{"b":"', f_grow(30), '"}')) AS sp_mrg; +SELECT JSON_REMOVE(CONCAT(d, ''), '$.a') AS sp_rem; +END| +CALL p_edit(); +sp_ins +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +sp_ins_big parses kept_a value_head value_tail +50026 1 1111111111 zzzzzzzz zzzzzzzz +sp_ins_concat +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +sp_mrg +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +sp_rem +{} +DROP PROCEDURE p_edit; +DROP FUNCTION f_grow; +# +# 6. A column read straight out of the row, which is the one +# producer that returns the row itself +# +SELECT JSON_INSERT(js, '$.c', (SELECT MAX(v) FROM t1)) AS col_sub FROM t1; +col_sub +{"A": 1, "B": 2, "c": "Y"} +{"A": 3, "B": 4, "c": "Y"} +SELECT JSON_INSERT(js, '$.c', (SELECT COUNT(*) FROM t1)) AS col_cnt FROM t1; +col_cnt +{"A": 1, "B": 2, "c": 2} +{"A": 3, "B": 4, "c": 2} +# +# 7. A value that is itself a document +# +# Everything above passes a plain string, which is copied +# a character at a time with quotes put round it. A value +# that is already a document goes in another way: its bytes +# are noted down where they stand, and only then is the +# composing written into the buffer they may have to share. +# That is the one place in this work where a note of where +# something was outlives an append, so it is the one that +# belongs here. +# +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', +JSON_EXTRACT(CONCAT('{"q":[1,', v, ']}'), '$.q')) +AS typed_ins FROM t1; +typed_ins +{"A": 1, "B": 2, "c": null} +{"A": 3, "B": 4, "c": null} +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', +JSON_OBJECT('k', CONCAT(v, v))) AS typed_built FROM t1; +typed_built +{"A": 1, "B": 2, "c": {"k": "XX"}} +{"A": 3, "B": 4, "c": {"k": "YY"}} +SELECT JSON_MERGE(CONCAT(js, ''), +JSON_OBJECT('c', CONCAT(v, v))) AS typed_mrg FROM t1; +typed_mrg +{"A": 1, "B": 2, "c": "XX"} +{"A": 3, "B": 4, "c": "YY"} +# +# The same where the value is long enough to move a buffer +# while it is being written out. +# +SELECT LENGTH(g) AS typed_big, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.A') AS kept_a, +JSON_LENGTH(JSON_EXTRACT(g, '$.c')) AS value_members, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[0]')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[1]')), 8) AS value_tail +FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', +JSON_ARRAY(REPEAT(v, 5000), REPEAT(v, 5000))) AS g +FROM t1) AS x; +typed_big parses kept_a value_members value_head value_tail +10031 1 1 2 XXXXXXXX XXXXXXXX +10031 1 3 2 YYYYYYYY YYYYYYYY +# +# And where the document being edited is a user variable that +# the value expression writes to, with the value a document. +# +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', +JSON_ARRAY((@d := REPEAT('z', 30)), 2)) AS typed_uvar; +typed_uvar +{"a": 1111111111, "b": ["zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 2]} +DROP TABLE t1; diff --git a/mysql-test/main/func_json_aliasing.test b/mysql-test/main/func_json_aliasing.test new file mode 100644 index 0000000000000..521ac8ad902ce --- /dev/null +++ b/mysql-test/main/func_json_aliasing.test @@ -0,0 +1,173 @@ +# +# Working out a value while the document is still being read from. +# +# A function that edits a document walks it to find the place to edit +# and then copies the pieces either side of that place out of it, and +# in between it works out the value to put there. Working out a value +# is running whatever expression a caller wrote, and the document is +# not this function's to begin with - it can be a row, a routine's +# variable, another statement's user variable. Nothing stops such an +# expression writing where the document is. +# +# Whether an argument is worked out INTO the buffer that was offered to +# it is a property of the function that produces it, not of the value: +# CONCAT, LOWER, UPPER and REVERSE build into it, TRIM and SUBSTRING +# return a piece of what they were given, a bare column returns +# the row, and something written out by hand is not worked out at all. +# So a case built from written-out arguments exercises none of this, +# and every case below is built from producers that do write. +# + +CREATE TABLE t1 (id INT, js VARCHAR(64), v VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"A":1,"B":2}', 'X'), (2, '{"A":3,"B":4}', 'Y'); + +--echo # +--echo # 1. Document and value both built into the buffer offered them +--echo # +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v)) AS ins FROM t1; +SELECT JSON_INSERT(LOWER(js), '$.c', LOWER(v)) AS ins_low FROM t1; +SELECT JSON_INSERT(REVERSE(REVERSE(js)), '$.c', UPPER(v)) AS ins_rev FROM t1; +SELECT JSON_SET(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS st FROM t1; +SELECT JSON_REPLACE(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS rep FROM t1; +SELECT JSON_REMOVE(CONCAT(js, ''), '$.A') AS rem FROM t1; +SELECT JSON_ARRAY_APPEND(CONCAT('[', js, ']'), '$', CONCAT(v, v)) AS app + FROM t1; +SELECT JSON_ARRAY_INSERT(CONCAT('[', js, ']'), '$[0]', CONCAT(v, v)) AS ari + FROM t1; +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mrg FROM t1; +SELECT JSON_MERGE_PATCH(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mpt + FROM t1; + +--echo # +--echo # 2. More than one path, so the document read from on the second +--echo # pass is the one the first pass wrote +--echo # +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), + '$.d', CONCAT(v, v, v)) AS two_paths FROM t1; +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), + '$.zz.deeper', CONCAT(v, v), + '$.d', CONCAT(v, v, v)) AS three_paths FROM t1; +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}'), + CONCAT('{"d":"', v, '"}')) AS mrg3 FROM t1; + +--echo # +--echo # 3. A value that grows a great deal, so a buffer holding it has +--echo # to be found somewhere else +--echo # +--echo # These are the only cases here big enough to make a buffer +--echo # move, and so the only ones that reach what this file is +--echo # about. How long the answer is cannot be what they ask: +--echo # copying out of a buffer that has moved brings back the same +--echo # number of bytes it would have brought back intact, so a +--echo # length is exactly the measure that survives the fault. They +--echo # ask instead whether the answer still reads as a document, +--echo # whether the member nobody touched came through, and what +--echo # stands at both ends of the value that grew. +--echo # +SELECT LENGTH(g) AS grown, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail + FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', REPEAT(v, 5000)) AS g + FROM t1) AS x; +SELECT LENGTH(g) AS mgrown, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail + FROM (SELECT JSON_MERGE(CONCAT(js, ''), + CONCAT('{"c":"', REPEAT(v, 5000), '"}')) AS g + FROM t1) AS x; + +--echo # +--echo # 4. A user variable holding the document, written to by the +--echo # expression that works out the value +--echo # +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 30))) AS uvar; +SET @d= '{"a":1111111111}'; +SELECT LENGTH(g) AS uvar_big, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.a') AS kept_a, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail + FROM (SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 100000))) AS g) AS x; +SET @d= '[1111111111]'; +SELECT JSON_ARRAY_APPEND(@d, '$', (@d := REPEAT('z', 30))) AS uvar_app; +SET @d= '{"a":1111111111}'; +SELECT JSON_MERGE(@d, (@d := '{"b":2}')) AS uvar_mrg; + +--echo # +--echo # 5. A routine's own variable holding the document, with the +--echo # value worked out by calling something +--echo # +--delimiter | +CREATE FUNCTION f_grow(n INT) RETURNS TEXT +BEGIN + RETURN REPEAT('z', n); +END| +CREATE PROCEDURE p_edit() +BEGIN + DECLARE d TEXT DEFAULT '{"a":1111111111}'; + SELECT JSON_INSERT(d, '$.b', f_grow(30)) AS sp_ins; + SELECT LENGTH(g) AS sp_ins_big, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.a') AS kept_a, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail + FROM (SELECT JSON_INSERT(d, '$.b', f_grow(50000)) AS g) AS x; + SELECT JSON_INSERT(CONCAT(d, ''), '$.b', f_grow(30)) AS sp_ins_concat; + SELECT JSON_MERGE(d, CONCAT('{"b":"', f_grow(30), '"}')) AS sp_mrg; + SELECT JSON_REMOVE(CONCAT(d, ''), '$.a') AS sp_rem; +END| +--delimiter ; +CALL p_edit(); +DROP PROCEDURE p_edit; +DROP FUNCTION f_grow; + +--echo # +--echo # 6. A column read straight out of the row, which is the one +--echo # producer that returns the row itself +--echo # +SELECT JSON_INSERT(js, '$.c', (SELECT MAX(v) FROM t1)) AS col_sub FROM t1; +SELECT JSON_INSERT(js, '$.c', (SELECT COUNT(*) FROM t1)) AS col_cnt FROM t1; + +--echo # +--echo # 7. A value that is itself a document +--echo # +--echo # Everything above passes a plain string, which is copied +--echo # a character at a time with quotes put round it. A value +--echo # that is already a document goes in another way: its bytes +--echo # are noted down where they stand, and only then is the +--echo # composing written into the buffer they may have to share. +--echo # That is the one place in this work where a note of where +--echo # something was outlives an append, so it is the one that +--echo # belongs here. +--echo # +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', + JSON_EXTRACT(CONCAT('{"q":[1,', v, ']}'), '$.q')) + AS typed_ins FROM t1; +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', + JSON_OBJECT('k', CONCAT(v, v))) AS typed_built FROM t1; +SELECT JSON_MERGE(CONCAT(js, ''), + JSON_OBJECT('c', CONCAT(v, v))) AS typed_mrg FROM t1; + +--echo # +--echo # The same where the value is long enough to move a buffer +--echo # while it is being written out. +--echo # +SELECT LENGTH(g) AS typed_big, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.A') AS kept_a, + JSON_LENGTH(JSON_EXTRACT(g, '$.c')) AS value_members, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[0]')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[1]')), 8) AS value_tail + FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', + JSON_ARRAY(REPEAT(v, 5000), REPEAT(v, 5000))) AS g + FROM t1) AS x; + +--echo # +--echo # And where the document being edited is a user variable that +--echo # the value expression writes to, with the value a document. +--echo # +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', + JSON_ARRAY((@d := REPEAT('z', 30)), 2)) AS typed_uvar; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_emit_killed.result b/mysql-test/main/func_json_emit_killed.result new file mode 100644 index 0000000000000..c46cef55f3795 --- /dev/null +++ b/mysql-test/main/func_json_emit_killed.result @@ -0,0 +1,160 @@ +SET @old_debug= @@debug_dbug; +SET debug_dbug='+d,json_kill_while_emitting'; +# +# 1. Values written out during the reading +# +# An object and an array, each with more than one member, so +# the walk takes several steps and the kill lands inside it. +# +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS obj_value; +ERROR 70100: Query execution was interrupted +SELECT JSON_EXTRACT('{"a": [1, 2, 3]}', '$.a') AS array_value; +ERROR 70100: Query execution was interrupted +SELECT JSON_EXTRACT('{"a": {"b": [1, 2], "c": {"d": 3}}}', '$.a') AS nested; +ERROR 70100: Query execution was interrupted +# +# Written the same way when more than one path is asked for, +# and when the value is reached through a wildcard. +# +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.a', '$.b') AS two_paths; +ERROR 70100: Query execution was interrupted +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.*') AS wildcard; +ERROR 70100: Query execution was interrupted +# +# 2. Values that are copied rather than written out +# +# A scalar is punctuated with nothing, so it is taken across as +# it stands and there is no walk to be stopped in. These are +# answered, and that is what says the arming above reaches the +# walk and only the walk. +# +SELECT JSON_EXTRACT('{"a": 1}', '$.a') AS scalar_value; +scalar_value +1 +SELECT JSON_EXTRACT('{"a": "text"}', '$.a') AS scalar_string; +scalar_string +"text" +SELECT JSON_EXTRACT('{"a": null, "b": true}', '$.a', '$.b') AS two_scalars; +two_scalars +[null, true] +# Nothing matched, so nothing was written at all. +SELECT JSON_EXTRACT('{"a": 1}', '$.zz') AS no_match; +no_match +NULL +# +# 3. Answers that are read back after being composed +# +# These build an answer by copying the parts of the document +# they are not changing and putting the new part between them, +# which is no walk of their own. Then they read the whole of +# it back to settle its formatting, and that reading writes as it +# goes: it is the same walk section 1 is stopped in, reached +# from the other end. So the kill lands in it. +# +# The reading that finds the place to edit lets go of a killed +# query as well, a token at a time, and is covered by +# func_json_notembedded. +# +SELECT JSON_INSERT('{"a": {"b": 1, "c": 2}}', '$.d', 4) AS insert_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_REMOVE('{"a": {"b": 1, "c": 2}}', '$.a.b') AS remove_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_SET('{"a": [1, 2, 3]}', '$.a[0]', 9) AS set_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_REPLACE('{"a": [1, 2, 3]}', '$.a[0]', 9) AS replace_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_ARRAY_APPEND('{"a": [1, 2]}', '$.a', 3) AS append_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_ARRAY_INSERT('{"a": [1, 2]}', '$.a[0]', 9) AS array_insert_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_MERGE_PATCH('{"a": {"b": 1}}', '{"a": {"c": 2}}') AS merge_patch_done; +ERROR 70100: Query execution was interrupted +# +# The same with a document answering is_valid, so that the +# answer is passed as it is composed and there is no +# reading back at all. Answered - which is what says the +# ones above they match are stopped in the reading back and +# nowhere else, the composing being the same either way. +# +SELECT JSON_INSERT(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.d', 4) +AS insert_is_valid; +insert_is_valid +{"a": {"b": 1, "c": 2}, "d": 4} +SELECT JSON_REMOVE(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.a.b') +AS remove_is_valid; +remove_is_valid +{"a": {"c": 2}} +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', JSON_OBJECT('b', 1)), +JSON_OBJECT('a', JSON_OBJECT('c', 2))) +AS merge_patch_is_valid; +merge_patch_is_valid +{"a": {"b": 1, "c": 2}} +# +# 4. A value written out during the reading +# +# A value going into a document is written out again where the +# document it joins is is_valid and is_nice and the value is +# not, that being the one way the value's own formatting can +# still be put right. That writing IS a walk, so the kill +# below is raised inside it. +# +# What these show is that a kill arriving in the middle of that +# walk is carried out - the walk gives up, the bytes fall back +# to what they were, and the statement ends the way a killed +# statement ends. They do NOT show which reading noticed it: +# the flag stays raised once it is raised, and a statement that +# ignored it here would still end this way at the next step. +# How soon it is noticed is the part that cannot be written +# down as an expected result. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, '{"b":1,"c":2}'), (2, '[1,2,3]'); +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS value_walked FROM t1; +ERROR 70100: Query execution was interrupted +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', j) AS value_walked_set FROM t1; +ERROR 70100: Query execution was interrupted +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', j) AS value_walked_append +FROM t1; +ERROR 70100: Query execution was interrupted +# +# Three more, sorted by whether anything walks in them. +# +# A value already formatted the way the answer needs is taken +# across without being read at all, so there is no walk. It +# has to come from something that does not walk either - a +# document BUILT out of values, not one cut out of another, +# since cutting one out is itself a walk and would be what the +# kill landed in. +# +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_OBJECT('x', 1, 'y', 2)) +AS value_taken; +value_taken +{"a": 1, "d": {"x": 1, "y": 2}} +# +# A document nobody attested sends the whole answer through +# the reading back, and a value going into that is left as it +# arrived - so there is no walk of the value's own here. The +# reading back is one, though, so this is stopped where the +# statements in section 3 are. +# +SELECT JSON_INSERT('{"a": 1}', '$.d', j) AS value_left FROM t1; +ERROR 70100: Query execution was interrupted +# +# A scalar carries no punctuation, so it is copied like the +# scalars in section 2 and attests to the same reason. +# +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '11'), (2, '"text"'); +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS scalar_value FROM t2; +scalar_value +{"a": 1, "d": 11} +{"a": 1, "d": "text"} +DROP TABLE t1; +DROP TABLE t2; +# +# 5. A kill is let go of once the statement it was aimed at is over +# +SET debug_dbug= @old_debug; +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS after_disarming; +after_disarming +{"b": 1, "c": 2} diff --git a/mysql-test/main/func_json_emit_killed.test b/mysql-test/main/func_json_emit_killed.test new file mode 100644 index 0000000000000..e1c602348b425 --- /dev/null +++ b/mysql-test/main/func_json_emit_killed.test @@ -0,0 +1,182 @@ +# +# A killed query has to be let go of while a result is being written, +# not only before the writing starts. +# +# The existing coverage arms the kill at the top of each function, so +# what it reaches is the first reading step of all - a check that was +# already there and would still be there whatever the writing did. The +# writing itself has never been reached under a kill. These arm one +# partway through the walk that does the writing, so that the walk is +# where the killed statement is standing when it has to let go. +# +# What that shows is that a kill arriving in the middle of the writing +# is carried out at all: the walk gives up, nothing is left half +# written, and the statement ends the way a killed statement ends. It +# does NOT show which reading noticed it. Once the flag is raised it +# stays raised, so a walk that ignored it would still end this way at +# the next step, and no expected result can tell the two apart. How +# soon a kill is noticed is a matter of how long the statement runs, +# which is not something a result file can hold. +# +# What is sorted below is not which function was asked but whether a +# walk happens at all. A statement that writes a document out, that +# writes a value into one, or that reads its own answer back to settle +# its formatting, walks - and lets go. A statement that only copies +# bytes from one place to another has nothing to be stopped in, and +# answers. Which side a function falls on is decided by what it was +# handed, so the same function stands on both. +# + +--source include/have_debug.inc + +SET @old_debug= @@debug_dbug; +SET debug_dbug='+d,json_kill_while_emitting'; + +--echo # +--echo # 1. Values written out during the reading +--echo # +--echo # An object and an array, each with more than one member, so +--echo # the walk takes several steps and the kill lands inside it. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS obj_value; +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": [1, 2, 3]}', '$.a') AS array_value; +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": {"b": [1, 2], "c": {"d": 3}}}', '$.a') AS nested; + +--echo # +--echo # Written the same way when more than one path is asked for, +--echo # and when the value is reached through a wildcard. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.a', '$.b') AS two_paths; +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.*') AS wildcard; + +--echo # +--echo # 2. Values that are copied rather than written out +--echo # +--echo # A scalar is punctuated with nothing, so it is taken across as +--echo # it stands and there is no walk to be stopped in. These are +--echo # answered, and that is what says the arming above reaches the +--echo # walk and only the walk. +--echo # +SELECT JSON_EXTRACT('{"a": 1}', '$.a') AS scalar_value; +SELECT JSON_EXTRACT('{"a": "text"}', '$.a') AS scalar_string; +SELECT JSON_EXTRACT('{"a": null, "b": true}', '$.a', '$.b') AS two_scalars; +--echo # Nothing matched, so nothing was written at all. +SELECT JSON_EXTRACT('{"a": 1}', '$.zz') AS no_match; + +--echo # +--echo # 3. Answers that are read back after being composed +--echo # +--echo # These build an answer by copying the parts of the document +--echo # they are not changing and putting the new part between them, +--echo # which is no walk of their own. Then they read the whole of +--echo # it back to settle its formatting, and that reading writes as it +--echo # goes: it is the same walk section 1 is stopped in, reached +--echo # from the other end. So the kill lands in it. +--echo # +--echo # The reading that finds the place to edit lets go of a killed +--echo # query as well, a token at a time, and is covered by +--echo # func_json_notembedded. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_INSERT('{"a": {"b": 1, "c": 2}}', '$.d', 4) AS insert_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_REMOVE('{"a": {"b": 1, "c": 2}}', '$.a.b') AS remove_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_SET('{"a": [1, 2, 3]}', '$.a[0]', 9) AS set_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_REPLACE('{"a": [1, 2, 3]}', '$.a[0]', 9) AS replace_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_ARRAY_APPEND('{"a": [1, 2]}', '$.a', 3) AS append_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_ARRAY_INSERT('{"a": [1, 2]}', '$.a[0]', 9) AS array_insert_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_MERGE_PATCH('{"a": {"b": 1}}', '{"a": {"c": 2}}') AS merge_patch_done; + +--echo # +--echo # The same with a document answering is_valid, so that the +--echo # answer is passed as it is composed and there is no +--echo # reading back at all. Answered - which is what says the +--echo # ones above they match are stopped in the reading back and +--echo # nowhere else, the composing being the same either way. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.d', 4) + AS insert_is_valid; +SELECT JSON_REMOVE(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.a.b') + AS remove_is_valid; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', JSON_OBJECT('b', 1)), + JSON_OBJECT('a', JSON_OBJECT('c', 2))) + AS merge_patch_is_valid; + +--echo # +--echo # 4. A value written out during the reading +--echo # +--echo # A value going into a document is written out again where the +--echo # document it joins is is_valid and is_nice and the value is +--echo # not, that being the one way the value's own formatting can +--echo # still be put right. That writing IS a walk, so the kill +--echo # below is raised inside it. +--echo # +--echo # What these show is that a kill arriving in the middle of that +--echo # walk is carried out - the walk gives up, the bytes fall back +--echo # to what they were, and the statement ends the way a killed +--echo # statement ends. They do NOT show which reading noticed it: +--echo # the flag stays raised once it is raised, and a statement that +--echo # ignored it here would still end this way at the next step. +--echo # How soon it is noticed is the part that cannot be written +--echo # down as an expected result. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, '{"b":1,"c":2}'), (2, '[1,2,3]'); + +--error ER_QUERY_INTERRUPTED +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS value_walked FROM t1; +--error ER_QUERY_INTERRUPTED +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', j) AS value_walked_set FROM t1; +--error ER_QUERY_INTERRUPTED +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', j) AS value_walked_append + FROM t1; + +--echo # +--echo # Three more, sorted by whether anything walks in them. +--echo # +--echo # A value already formatted the way the answer needs is taken +--echo # across without being read at all, so there is no walk. It +--echo # has to come from something that does not walk either - a +--echo # document BUILT out of values, not one cut out of another, +--echo # since cutting one out is itself a walk and would be what the +--echo # kill landed in. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_OBJECT('x', 1, 'y', 2)) + AS value_taken; + +--echo # +--echo # A document nobody attested sends the whole answer through +--echo # the reading back, and a value going into that is left as it +--echo # arrived - so there is no walk of the value's own here. The +--echo # reading back is one, though, so this is stopped where the +--echo # statements in section 3 are. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_INSERT('{"a": 1}', '$.d', j) AS value_left FROM t1; + +--echo # +--echo # A scalar carries no punctuation, so it is copied like the +--echo # scalars in section 2 and attests to the same reason. +--echo # +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '11'), (2, '"text"'); +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS scalar_value FROM t2; + +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # 5. A kill is let go of once the statement it was aimed at is over +--echo # +SET debug_dbug= @old_debug; +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS after_disarming; diff --git a/mysql-test/main/func_json_key_span.result b/mysql-test/main/func_json_key_span.result new file mode 100644 index 0000000000000..3ae8b67cf23bb --- /dev/null +++ b/mysql-test/main/func_json_key_span.result @@ -0,0 +1,155 @@ +# +# 1. A plain key, and one holding a quote that makes nonsense of +# the document +# +SELECT JSON_INSERT('{}', '$.b', 1) AS plain; +plain +{"b": 1} +SELECT JSON_INSERT(JSON_OBJECT(), '$.b', 1) AS plain_is_valid; +plain_is_valid +{"b": 1} +SELECT JSON_INSERT('{}', '$.a"b', 1) AS broken; +broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 +SELECT JSON_INSERT(JSON_OBJECT(), '$.a"b', 1) AS broken_is_valid; +broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 +# +# 2. A key that closes its own member and opens another, which +# reads as a document saying something the caller did not +# write and is returned all the same +# +SELECT JSON_INSERT('{}', '$.a":1,"b', 1) AS smuggled; +smuggled +{"a": 1, "b": 1} +SELECT JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1) AS smuggled_is_valid; +smuggled_is_valid +{"a": 1, "b": 1} +# +# 3. A backslash begins an escape, so a key holding one may or +# may not still end where it looks like it ends +# +SELECT JSON_INSERT('{}', '$.a\\b', 1) AS backslash; +backslash +{"a\b": 1} +SELECT JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1) AS backslash_is_valid; +backslash_is_valid +{"a\b": 1} +# +# 4. The same keys in character sets that write a letter in more +# than one byte +# +SET @old_coll= @@collation_connection; +# All four of the keys above run again here, and the last two +# are the reason the section exists: a key that closes its own +# member, and a key holding a backslash, are where a reader +# stepping by bytes and one stepping by characters part +# company. Running only the plain and the quoted key would +# leave that untried. +SET @@collation_connection='ucs2_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS ucs2_plain; +ucs2_plain +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS ucs2_plain_is_valid; +ucs2_plain_is_valid +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS ucs2_broken; +ucs2_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS ucs2_broken_is_valid; +ucs2_broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS ucs2_smuggled; +ucs2_smuggled +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) +AS ucs2_smuggled_is_valid; +ucs2_smuggled_is_valid +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS ucs2_backslash; +ucs2_backslash +007B00220061005C00620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) +AS ucs2_backslash_is_valid; +ucs2_backslash_is_valid +007B00220061005C00620022003A00200031007D +SET @@collation_connection='utf16_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf16_plain; +utf16_plain +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf16_plain_is_valid; +utf16_plain_is_valid +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf16_broken; +utf16_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf16_broken_is_valid; +utf16_broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf16_smuggled; +utf16_smuggled +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) +AS utf16_smuggled_is_valid; +utf16_smuggled_is_valid +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf16_backslash; +utf16_backslash +007B00220061005C00620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) +AS utf16_backslash_is_valid; +utf16_backslash_is_valid +007B00220061005C00620022003A00200031007D +SET @@collation_connection='utf32_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf32_plain; +utf32_plain +0000007B0000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf32_plain_is_valid; +utf32_plain_is_valid +0000007B0000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf32_broken; +utf32_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 16 +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf32_broken_is_valid; +utf32_broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 16 +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf32_smuggled; +utf32_smuggled +0000007B0000002200000061000000220000003A00000020000000310000002C000000200000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) +AS utf32_smuggled_is_valid; +utf32_smuggled_is_valid +0000007B0000002200000061000000220000003A00000020000000310000002C000000200000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf32_backslash; +utf32_backslash +0000007B00000022000000610000005C00000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) +AS utf32_backslash_is_valid; +utf32_backslash_is_valid +0000007B00000022000000610000005C00000062000000220000003A00000020000000310000007D +SET @@collation_connection= @old_coll; +# +# 5. Removing a key reads the path the same way +# +SELECT JSON_REMOVE('{"a":1,"b":2}', '$.a') AS removed; +removed +{"b": 2} +SELECT JSON_REMOVE(JSON_INSERT('{"b":2}', '$.a', 1), '$.a') AS removed_is_valid; +removed_is_valid +{"b": 2} diff --git a/mysql-test/main/func_json_key_span.test b/mysql-test/main/func_json_key_span.test new file mode 100644 index 0000000000000..9e9c64446f675 --- /dev/null +++ b/mysql-test/main/func_json_key_span.test @@ -0,0 +1,103 @@ +# +# What a path's last step is allowed to be, when it becomes a key in +# the document being written. +# +# The step is copied into the result between two quotes. A step that +# holds a quote of its own therefore closes the key early, and what +# follows it lands in the document as punctuation rather than as part +# of the name. Sometimes that makes nonsense of the document and +# sometimes it makes a perfectly good one saying something else, and +# which of the two it is depends on the whole result and not on the +# step - so a step that reaches past its quotes is not turned down, it +# is only left to the reading back to judge, exactly as before. +# +# The cases below are here because the reading back is now skipped for +# a document answering is_valid and is_nice. Each key is put to such a +# document and to one answering is_valid false, and the two must give +# the same answer. +# +# The wide character sets are the second half of it. A key is written +# in characters and not in bytes: 'b' in ucs2 is the two bytes 00 62, +# and anything reading those one at a time finds a control character +# where there is none. +# + +--echo # +--echo # 1. A plain key, and one holding a quote that makes nonsense of +--echo # the document +--echo # +SELECT JSON_INSERT('{}', '$.b', 1) AS plain; +SELECT JSON_INSERT(JSON_OBJECT(), '$.b', 1) AS plain_is_valid; +SELECT JSON_INSERT('{}', '$.a"b', 1) AS broken; +SELECT JSON_INSERT(JSON_OBJECT(), '$.a"b', 1) AS broken_is_valid; + +--echo # +--echo # 2. A key that closes its own member and opens another, which +--echo # reads as a document saying something the caller did not +--echo # write and is returned all the same +--echo # +SELECT JSON_INSERT('{}', '$.a":1,"b', 1) AS smuggled; +SELECT JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1) AS smuggled_is_valid; + +--echo # +--echo # 3. A backslash begins an escape, so a key holding one may or +--echo # may not still end where it looks like it ends +--echo # +SELECT JSON_INSERT('{}', '$.a\\b', 1) AS backslash; +SELECT JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1) AS backslash_is_valid; + +--echo # +--echo # 4. The same keys in character sets that write a letter in more +--echo # than one byte +--echo # +SET @old_coll= @@collation_connection; + +--echo # All four of the keys above run again here, and the last two +--echo # are the reason the section exists: a key that closes its own +--echo # member, and a key holding a backslash, are where a reader +--echo # stepping by bytes and one stepping by characters part +--echo # company. Running only the plain and the quoted key would +--echo # leave that untried. +SET @@collation_connection='ucs2_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS ucs2_plain; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS ucs2_plain_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS ucs2_broken; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS ucs2_broken_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS ucs2_smuggled; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) + AS ucs2_smuggled_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS ucs2_backslash; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) + AS ucs2_backslash_is_valid; + +SET @@collation_connection='utf16_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf16_plain; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf16_plain_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf16_broken; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf16_broken_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf16_smuggled; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) + AS utf16_smuggled_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf16_backslash; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) + AS utf16_backslash_is_valid; + +SET @@collation_connection='utf32_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf32_plain; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf32_plain_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf32_broken; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf32_broken_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf32_smuggled; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) + AS utf32_smuggled_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf32_backslash; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) + AS utf32_backslash_is_valid; + +SET @@collation_connection= @old_coll; + +--echo # +--echo # 5. Removing a key reads the path the same way +--echo # +SELECT JSON_REMOVE('{"a":1,"b":2}', '$.a') AS removed; +SELECT JSON_REMOVE(JSON_INSERT('{"b":2}', '$.a', 1), '$.a') AS removed_is_valid; diff --git a/mysql-test/main/func_json_nice_compose.result b/mysql-test/main/func_json_nice_compose.result new file mode 100644 index 0000000000000..380eb863e2f8f --- /dev/null +++ b/mysql-test/main/func_json_nice_compose.result @@ -0,0 +1,287 @@ +CREATE TABLE docs (id INT, js VARCHAR(500)); +INSERT INTO docs VALUES +(1, '{"a":1,"b":2}'), +(2, '{ "a" : 1 , "b" : 2 }'), +(3, '{"a": 1, "b": 2}'), +(4, '{"a":[1,2,{"c":3}],"b":{"d":[4,5]}}'), +(5, '[1,2,3]'), +(6, '[]'), +(7, '{}'), +(8, '{"a":[],"b":{}}'), +(9, '{"a":1.50,"b":1e2,"c":-0.0}'), +(10, '{"a":"x\\ty","b":"\\u0062","c":"quote\\"inside"}'), +(11, '{"a":null,"b":true,"c":false}'), +(12, '{"a":1,"a":2}'), +(13, '{"a":{"b":{"c":{"d":{"e":[1,{"f":2}]}}}}}'), +(14, '[{"a":1},{"a":2},[3,[4,5]]]'), +(15, '{"a":[1],"z":[2]}'); +CREATE TABLE steps (id INT, p1 VARCHAR(50), v1 VARCHAR(50), +p2 VARCHAR(50), v2 VARCHAR(50)); +INSERT INTO steps VALUES +(1, '$.a', 'one', '$.b', 'two'), +(2, '$.zz', 'new', '$.yy', 'other'), +(3, '$.a', 'one', '$.zz', 'new'), +(4, '$.zz', 'new', '$.a', 'one'), +(5, '$[0]', 'one', '$[1]', 'two'), +(6, '$[0]', 'one', '$[9]', 'two'), +(7, '$.a[0]', 'one', '$.a[1]', 'two'), +(8, '$.a[9]', 'one', '$.b', 'two'), +(9, '$.nomatch', 'none', '$.a', 'one'), +(10, '$.a', 'one', '$.nomatch', 'none'), +(11, '$.nomatch', 'none', '$.other', 'none'), +(12, '$.a.b.c', 'deep', '$.a.b', 'less'), +(13, '$.b.d[1]', 'one', '$.a[2].c', 'two'), +(14, '$[2][1][0]','one', '$[0].a', 'two'), +(15, '$.a[0]', 'one', '$.z[0]', 'two'); +# +# JSON_INSERT +# +SELECT d.id AS doc, s.id AS step, +JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) +AS BINARY)); +doc step once_at_end each_pass +# +# JSON_SET +# +SELECT d.id AS doc, s.id AS step, +JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_REPLACE +# +SELECT d.id AS doc, s.id AS step, +JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) +AS BINARY)); +doc step once_at_end each_pass +# +# JSON_REMOVE +# +SELECT d.id AS doc, s.id AS step, +JSON_REMOVE(d.js, s.p1, s.p2) AS once_at_end, +JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> +CAST(JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_ARRAY_APPEND +# +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), s.p2, s.v2) +AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), +s.p2, s.v2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_ARRAY_INSERT +# +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) +AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), +s.p2, s.v2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_MERGE and JSON_MERGE_PATCH, which fold their documents from +# the left in the same way +# +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, +JSON_MERGE(a.js, b.js, c.js) AS once_at_end, +JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS each_pass +FROM docs a, docs b, docs c +WHERE NOT (CAST(JSON_MERGE(a.js, b.js, c.js) AS BINARY) <=> +CAST(JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS BINARY)); +doc1 doc2 doc3 once_at_end each_pass +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, +JSON_MERGE_PATCH(a.js, b.js, c.js) AS once_at_end, +JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) AS each_pass +FROM docs a, docs b, docs c +WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js, c.js) AS BINARY) <=> +CAST(JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) +AS BINARY)); +doc1 doc2 doc3 once_at_end each_pass +# +# A value that is itself a document, which is written out rather +# than copied, and the wrapping of a value that is not an array +# +SELECT d.id AS doc, +JSON_INSERT(d.js, '$.zz', JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) +AS once_at_end, +JSON_INSERT(JSON_INSERT(d.js, '$.zz', +JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) AS each_pass +FROM docs d +WHERE NOT (CAST(JSON_INSERT(d.js, '$.zz', +JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) +AS BINARY) <=> +CAST(JSON_INSERT(JSON_INSERT(d.js, '$.zz', +JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) +AS BINARY)); +doc once_at_end each_pass +SELECT d.id AS doc, +JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS once_at_end, +JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), '$.b', 'y') +AS each_pass +FROM docs d +WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS BINARY) <=> +CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), +'$.b', 'y') AS BINARY)); +doc once_at_end each_pass +# +# Three passes rather than two, so that a document written out in +# the loose form is read, edited and written out again +# +SELECT d.id AS doc, +JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS once_at_end, +JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), '$.q', 2), +'$.r', 3) AS each_pass +FROM docs d +WHERE NOT (CAST(JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS BINARY) <=> +CAST(JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), +'$.q', 2), '$.r', 3) AS BINARY)); +doc once_at_end each_pass +# +# Writing a document out twice gives what writing it out once gave +# +SELECT id FROM docs +WHERE NOT (CAST(JSON_LOOSE(js) AS BINARY) <=> +CAST(JSON_LOOSE(JSON_LOOSE(js)) AS BINARY)); +id +# +# Editing a document that was written out in the loose form before +# the first pass rather than after the last one gives the same +# answer. This is what lets a piece of a document be copied across +# as it stands: a piece of one written the loose way is written the +# loose way too. +# +SELECT d.id AS doc, s.id AS step, +JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_REMOVE(d.js, s.p1, s.p2) AS from_input, +JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> +CAST(JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT a.id AS doc1, b.id AS doc2, +JSON_MERGE(a.js, b.js) AS from_input, +JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose +FROM docs a, docs b +WHERE NOT (CAST(JSON_MERGE(a.js, b.js) AS BINARY) <=> +CAST(JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS BINARY)); +doc1 doc2 from_input from_loose +SELECT a.id AS doc1, b.id AS doc2, +JSON_MERGE_PATCH(a.js, b.js) AS from_input, +JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose +FROM docs a, docs b +WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js) AS BINARY) <=> +CAST(JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) +AS BINARY)); +doc1 doc2 from_input from_loose +# +# What was actually produced while all of the above stayed quiet +# +# Every statement in this file passes by returning nothing, and +# a document that is nothing matches another that is nothing. +# These count the answers, so that a run in which the editing +# returned NULL throughout can no longer look like a run in +# which it agreed with itself throughout. +# +SELECT COUNT(*) AS pairs, +COUNT(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2)) AS answered +FROM docs d, steps s; +pairs answered +225 225 +SELECT COUNT(*) AS pairs, +COUNT(JSON_REMOVE(d.js, s.p1, s.p2)) AS answered +FROM docs d, steps s; +pairs answered +225 225 +SELECT COUNT(*) AS pairs, +COUNT(JSON_MERGE(a.js, b.js)) AS answered +FROM docs a, docs b; +pairs answered +225 225 +SELECT COUNT(*) AS docs, +COUNT(JSON_LOOSE(js)) AS answered +FROM docs; +docs answered +15 15 +# +# And what two of them actually say, so that a change to both +# sides at once still has somewhere to show itself. +# +SELECT id, JSON_INSERT(js, '$.p', 1, '$.q', 2) AS inserted +FROM docs WHERE id IN (1, 3, 5) ORDER BY id; +id inserted +1 {"a": 1, "b": 2, "p": 1, "q": 2} +3 {"a": 1, "b": 2, "p": 1, "q": 2} +5 [1, 2, 3] +SELECT id, JSON_LOOSE(js) AS loosened +FROM docs WHERE id IN (1, 2, 4) ORDER BY id; +id loosened +1 {"a": 1, "b": 2} +2 {"a": 1, "b": 2} +4 {"a": [1, 2, {"c": 3}], "b": {"d": [4, 5]}} +DROP TABLE docs, steps; diff --git a/mysql-test/main/func_json_nice_compose.test b/mysql-test/main/func_json_nice_compose.test new file mode 100644 index 0000000000000..29f88baf9447c --- /dev/null +++ b/mysql-test/main/func_json_nice_compose.test @@ -0,0 +1,304 @@ +# +# Editing a document that has already been written out in the loose form +# gives the same bytes as editing the original and writing the result out +# once at the end. +# +# The functions that edit a document with more than one path work through +# the paths one at a time, each pass reading what the pass before it +# wrote. Whether those intermediate documents are left as they came out +# of the input, or written out in the loose form as they go, must not +# change the answer. +# +# Every statement below asks the same question twice: once with all the +# paths given to one call, so that the document is written out once at +# the end, and once with the calls nested, so that each pass reads a +# document already written out in the loose form. A row comes back only +# when the two disagree, so a passing run says nothing at all. +# +# The bytes are compared, not the characters, so that a difference in +# case or in trailing space cannot pass for equality. +# +# Saying nothing at all is not enough on its own. A document compares +# equal to itself even when both of them are nothing: NULL is neither +# more nor less than NULL, so a fault that emptied BOTH sides would +# leave every one of these statements as quiet as a healthy server does. +# The section at the end therefore counts what came back and writes down +# what one of them says, so that the quiet above only means something +# while an answer is still being produced. +# + +CREATE TABLE docs (id INT, js VARCHAR(500)); +INSERT INTO docs VALUES + (1, '{"a":1,"b":2}'), + (2, '{ "a" : 1 , "b" : 2 }'), + (3, '{"a": 1, "b": 2}'), + (4, '{"a":[1,2,{"c":3}],"b":{"d":[4,5]}}'), + (5, '[1,2,3]'), + (6, '[]'), + (7, '{}'), + (8, '{"a":[],"b":{}}'), + (9, '{"a":1.50,"b":1e2,"c":-0.0}'), + (10, '{"a":"x\\ty","b":"\\u0062","c":"quote\\"inside"}'), + (11, '{"a":null,"b":true,"c":false}'), + (12, '{"a":1,"a":2}'), + (13, '{"a":{"b":{"c":{"d":{"e":[1,{"f":2}]}}}}}'), + (14, '[{"a":1},{"a":2},[3,[4,5]]]'), + (15, '{"a":[1],"z":[2]}'); + +CREATE TABLE steps (id INT, p1 VARCHAR(50), v1 VARCHAR(50), + p2 VARCHAR(50), v2 VARCHAR(50)); +INSERT INTO steps VALUES + (1, '$.a', 'one', '$.b', 'two'), + (2, '$.zz', 'new', '$.yy', 'other'), + (3, '$.a', 'one', '$.zz', 'new'), + (4, '$.zz', 'new', '$.a', 'one'), + (5, '$[0]', 'one', '$[1]', 'two'), + (6, '$[0]', 'one', '$[9]', 'two'), + (7, '$.a[0]', 'one', '$.a[1]', 'two'), + (8, '$.a[9]', 'one', '$.b', 'two'), + (9, '$.nomatch', 'none', '$.a', 'one'), + (10, '$.a', 'one', '$.nomatch', 'none'), + (11, '$.nomatch', 'none', '$.other', 'none'), + (12, '$.a.b.c', 'deep', '$.a.b', 'less'), + (13, '$.b.d[1]', 'one', '$.a[2].c', 'two'), + (14, '$[2][1][0]','one', '$[0].a', 'two'), + (15, '$.a[0]', 'one', '$.z[0]', 'two'); + +--echo # +--echo # JSON_INSERT +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) + AS BINARY)); + +--echo # +--echo # JSON_SET +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS BINARY)); + +--echo # +--echo # JSON_REPLACE +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) + AS BINARY)); + +--echo # +--echo # JSON_REMOVE +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_REMOVE(d.js, s.p1, s.p2) AS once_at_end, + JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> + CAST(JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS BINARY)); + +--echo # +--echo # JSON_ARRAY_APPEND +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), s.p2, s.v2) + AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), + s.p2, s.v2) AS BINARY)); + +--echo # +--echo # JSON_ARRAY_INSERT +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) + AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), + s.p2, s.v2) AS BINARY)); + +--echo # +--echo # JSON_MERGE and JSON_MERGE_PATCH, which fold their documents from +--echo # the left in the same way +--echo # +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, + JSON_MERGE(a.js, b.js, c.js) AS once_at_end, + JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS each_pass + FROM docs a, docs b, docs c + WHERE NOT (CAST(JSON_MERGE(a.js, b.js, c.js) AS BINARY) <=> + CAST(JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS BINARY)); + +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, + JSON_MERGE_PATCH(a.js, b.js, c.js) AS once_at_end, + JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) AS each_pass + FROM docs a, docs b, docs c + WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js, c.js) AS BINARY) <=> + CAST(JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) + AS BINARY)); + +--echo # +--echo # A value that is itself a document, which is written out rather +--echo # than copied, and the wrapping of a value that is not an array +--echo # +SELECT d.id AS doc, + JSON_INSERT(d.js, '$.zz', JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) + AS once_at_end, + JSON_INSERT(JSON_INSERT(d.js, '$.zz', + JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) AS each_pass + FROM docs d + WHERE NOT (CAST(JSON_INSERT(d.js, '$.zz', + JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) + AS BINARY) <=> + CAST(JSON_INSERT(JSON_INSERT(d.js, '$.zz', + JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) + AS BINARY)); + +SELECT d.id AS doc, + JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS once_at_end, + JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), '$.b', 'y') + AS each_pass + FROM docs d + WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS BINARY) <=> + CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), + '$.b', 'y') AS BINARY)); + +--echo # +--echo # Three passes rather than two, so that a document written out in +--echo # the loose form is read, edited and written out again +--echo # +SELECT d.id AS doc, + JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS once_at_end, + JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), '$.q', 2), + '$.r', 3) AS each_pass + FROM docs d + WHERE NOT (CAST(JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS BINARY) <=> + CAST(JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), + '$.q', 2), '$.r', 3) AS BINARY)); + +--echo # +--echo # Writing a document out twice gives what writing it out once gave +--echo # +SELECT id FROM docs + WHERE NOT (CAST(JSON_LOOSE(js) AS BINARY) <=> + CAST(JSON_LOOSE(JSON_LOOSE(js)) AS BINARY)); + +--echo # +--echo # Editing a document that was written out in the loose form before +--echo # the first pass rather than after the last one gives the same +--echo # answer. This is what lets a piece of a document be copied across +--echo # as it stands: a piece of one written the loose way is written the +--echo # loose way too. +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_REMOVE(d.js, s.p1, s.p2) AS from_input, + JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> + CAST(JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT a.id AS doc1, b.id AS doc2, + JSON_MERGE(a.js, b.js) AS from_input, + JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose + FROM docs a, docs b + WHERE NOT (CAST(JSON_MERGE(a.js, b.js) AS BINARY) <=> + CAST(JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS BINARY)); + +SELECT a.id AS doc1, b.id AS doc2, + JSON_MERGE_PATCH(a.js, b.js) AS from_input, + JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose + FROM docs a, docs b + WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js) AS BINARY) <=> + CAST(JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) + AS BINARY)); + +--echo # +--echo # What was actually produced while all of the above stayed quiet +--echo # +--echo # Every statement in this file passes by returning nothing, and +--echo # a document that is nothing matches another that is nothing. +--echo # These count the answers, so that a run in which the editing +--echo # returned NULL throughout can no longer look like a run in +--echo # which it agreed with itself throughout. +--echo # +SELECT COUNT(*) AS pairs, + COUNT(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2)) AS answered + FROM docs d, steps s; +SELECT COUNT(*) AS pairs, + COUNT(JSON_REMOVE(d.js, s.p1, s.p2)) AS answered + FROM docs d, steps s; +SELECT COUNT(*) AS pairs, + COUNT(JSON_MERGE(a.js, b.js)) AS answered + FROM docs a, docs b; +SELECT COUNT(*) AS docs, + COUNT(JSON_LOOSE(js)) AS answered + FROM docs; + +--echo # +--echo # And what two of them actually say, so that a change to both +--echo # sides at once still has somewhere to show itself. +--echo # +SELECT id, JSON_INSERT(js, '$.p', 1, '$.q', 2) AS inserted + FROM docs WHERE id IN (1, 3, 5) ORDER BY id; +SELECT id, JSON_LOOSE(js) AS loosened + FROM docs WHERE id IN (1, 2, 4) ORDER BY id; + +DROP TABLE docs, steps; diff --git a/mysql-test/main/func_json_punct_charset.result b/mysql-test/main/func_json_punct_charset.result new file mode 100644 index 0000000000000..530ed55d5c1b9 --- /dev/null +++ b/mysql-test/main/func_json_punct_charset.result @@ -0,0 +1,94 @@ +# A scalar converts into swe7 and a container does not. +SELECT JSON_VALID(CONVERT('1' USING swe7)) AS scalar_reads; +scalar_reads +1 +SELECT HEX(CONVERT('[1,2]' USING swe7)) AS container_bytes; +container_bytes +3F312C323F +Warnings: +Warning 1977 Cannot convert 'latin1' character 0x5B to 'swe7' +# +# 1. Picking values out, which writes brackets when several of +# them can match and nothing at all when only one can +# +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$', '$')) AS two_paths; +two_paths +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 1 +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$[*]')) AS wildcard; +wildcard +NULL +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$')) AS one_path; +one_path +31 +# +# 2. A document in that set attested is_valid, so the +# functions below are asked their question with it and answer +# yes to every part of it but the formatting +# +SELECT HEX(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS is_valid; +is_valid +31 +SELECT JSON_VALID(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS still_doc; +still_doc +1 +# +# 3. The six that edit a document, each given that is_valid +# one. The two that have to write a container to answer +# return NULL; the other four return a scalar - the one they +# were given, except JSON_MERGE_PATCH, which returns the patch. +# +SELECT HEX(JSON_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$.b', 3)) AS ins; +ins +31 +SELECT HEX(JSON_REMOVE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$.b')) AS rem; +rem +31 +SELECT HEX(JSON_ARRAY_APPEND(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$', 3)) AS app; +app +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 1 +SELECT HEX(JSON_ARRAY_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$[0]', 3)) AS ari; +ari +31 +SELECT HEX(JSON_MERGE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) AS mrg; +mrg +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 1 +SELECT HEX(JSON_MERGE_PATCH(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) +AS mpt; +mpt +32 +# +# 4. The ones that have always asked before writing punctuation, +# the ones that write none, and the two that write their answer +# in utf8mb4 whatever they were given +# +SELECT HEX(JSON_SEARCH(CONVERT('1' USING swe7), 'all', CONVERT('1' USING swe7))) +AS srch; +srch +222422 +SELECT HEX(JSON_ARRAY(CONVERT('1' USING swe7), CONVERT('2' USING swe7))) AS arr; +arr +5B2231222C202232225D +SELECT HEX(JSON_OBJECT(CONVERT('a' USING swe7), CONVERT('1' USING swe7))) AS obj; +obj +7B2261223A202231227D +SELECT HEX(JSON_QUERY(CONVERT('1' USING swe7), '$')) AS qry; +qry +NULL +SELECT HEX(JSON_NORMALIZE(CONVERT('1' USING swe7))) AS nrm; +nrm +312E304530 +SELECT HEX(JSON_COMPACT(CONVERT('1' USING swe7))) AS cmp; +cmp +31 diff --git a/mysql-test/main/func_json_punct_charset.test b/mysql-test/main/func_json_punct_charset.test new file mode 100644 index 0000000000000..4b1114c614aba --- /dev/null +++ b/mysql-test/main/func_json_punct_charset.test @@ -0,0 +1,70 @@ +# +# What the JSON functions do in a character set that cannot encode the +# punctuation a document is written with. +# +# swe7 puts national letters at the code points the brackets and braces +# live at, so a container written there is a word rather than a +# container. A document can still be written in such a set, but only a +# scalar one - writing a container would take the brackets it does not +# have. So a function handed a document in such a set was handed a +# scalar, and a function that has to write a container to give its +# answer cannot give one there at all. +# +# The answers below are the ones the server has always given. They are +# here because the functions that used to find this out by reading the +# whole of their result back no longer always do, and what they hand +# back must not change for that. +# + +--echo # A scalar converts into swe7 and a container does not. +SELECT JSON_VALID(CONVERT('1' USING swe7)) AS scalar_reads; +SELECT HEX(CONVERT('[1,2]' USING swe7)) AS container_bytes; + +--echo # +--echo # 1. Picking values out, which writes brackets when several of +--echo # them can match and nothing at all when only one can +--echo # +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$', '$')) AS two_paths; +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$[*]')) AS wildcard; +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$')) AS one_path; + +--echo # +--echo # 2. A document in that set attested is_valid, so the +--echo # functions below are asked their question with it and answer +--echo # yes to every part of it but the formatting +--echo # +SELECT HEX(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS is_valid; +SELECT JSON_VALID(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS still_doc; + +--echo # +--echo # 3. The six that edit a document, each given that is_valid +--echo # one. The two that have to write a container to answer +--echo # return NULL; the other four return a scalar - the one they +--echo # were given, except JSON_MERGE_PATCH, which returns the patch. +--echo # +SELECT HEX(JSON_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$.b', 3)) AS ins; +SELECT HEX(JSON_REMOVE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$.b')) AS rem; +SELECT HEX(JSON_ARRAY_APPEND(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$', 3)) AS app; +SELECT HEX(JSON_ARRAY_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$[0]', 3)) AS ari; +SELECT HEX(JSON_MERGE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) AS mrg; +SELECT HEX(JSON_MERGE_PATCH(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) + AS mpt; + +--echo # +--echo # 4. The ones that have always asked before writing punctuation, +--echo # the ones that write none, and the two that write their answer +--echo # in utf8mb4 whatever they were given +--echo # +SELECT HEX(JSON_SEARCH(CONVERT('1' USING swe7), 'all', CONVERT('1' USING swe7))) + AS srch; +SELECT HEX(JSON_ARRAY(CONVERT('1' USING swe7), CONVERT('2' USING swe7))) AS arr; +SELECT HEX(JSON_OBJECT(CONVERT('a' USING swe7), CONVERT('1' USING swe7))) AS obj; +SELECT HEX(JSON_QUERY(CONVERT('1' USING swe7), '$')) AS qry; +SELECT HEX(JSON_NORMALIZE(CONVERT('1' USING swe7))) AS nrm; +SELECT HEX(JSON_COMPACT(CONVERT('1' USING swe7))) AS cmp; diff --git a/mysql-test/main/func_json_scan_count.result b/mysql-test/main/func_json_scan_count.result new file mode 100644 index 0000000000000..572565d4d07d7 --- /dev/null +++ b/mysql-test/main/func_json_scan_count.result @@ -0,0 +1,378 @@ +CREATE TABLE t1 (id INT, js VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); +CREATE TABLE t3 (id INT, js VARCHAR(200)); +INSERT INTO t3 VALUES (1, CONCAT('{"a":"', REPEAT('x', 80), '"}')), +(2, CONCAT('{"a":"', REPEAT('y', 80), '"}')); +# +# A function that edits a document reads it once to find the place +# to edit, and then reads its own result back to find out how to +# write it. Two rows, two readings each. +# +FLUSH STATUS; +SELECT JSON_INSERT(js, '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_SET(js, '$.a', 9) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_REPLACE(js, '$.a', 9) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_REMOVE(js, '$.a') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_ARRAY_APPEND(js, '$.a', 7) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_ARRAY_INSERT(js, '$.a[0]', 7) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# Merging reads both documents, and then reads the result. +# +FLUSH STATUS; +SELECT JSON_MERGE(js, '{"c":3}') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_MERGE_PATCH(js, '{"c":3}') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# A function that only reads pays for ONE reading, not two. The +# walk to the path and the writing out of what was found are the +# same reading carried on, so nothing is read twice; that is what +# this work did to reading, and the count below is where it says +# so. +# +FLUSH STATUS; +SELECT JSON_EXTRACT(js, '$.a') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A path that matches nothing costs the same one reading, there +# being nothing to write out and nothing to read back either way. +# It is here to say that finding something is what has stopped +# costing extra, not that finding nothing ever did. +# +FLUSH STATUS; +SELECT JSON_EXTRACT(js, '$.zz') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# Chained edits pay only for the innermost link. That one is +# given a document out of a column that attests is_valid false, +# so it reads its result back before returning it. Every link +# after it is handed a document answering is_valid, and +# attests to its own result the same way. +# +FLUSH STATUS; +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_REMOVE(JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4), '$.a') +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# The same for each of the others, each handed the is_valid +# result of the one before it. +# +FLUSH STATUS; +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b[0]', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# The two above put their value at a key the document already +# has, so nothing is put there and what they append to is the +# scalar that was there all along, which has to be wrapped in an +# array first. At a key the document does NOT have, the value +# goes in and the array they append to is that value. +# +FLUSH STATUS; +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c[0]', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +FLUSH STATUS; +SELECT JSON_MERGE_PATCH(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# A merge whose other argument attests is_valid false reads the +# answer back, every document that goes into it having a say in +# how it is written. A written-out literal is such a document: +# nothing has read it, so nothing can say how it is written. +# +FLUSH STATUS; +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +FLUSH STATUS; +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), '{"d":4}') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +# +# A document answering is_valid but NOT is_nice is read back like +# one answering is_valid false. +# Picking a piece out of a document returns the piece as it was +# formatted where it came from, so that is such a document; picking +# values out and putting them together writes them out afresh, so +# that one is not. The same edit is put to each below, and the +# difference between the two counts is the reading back. +# +FLUSH STATUS; +SELECT JSON_INSERT(JSON_QUERY(js, '$'), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# Asking for a document in a particular formatting reads nothing at +# all when what asked for it wanted a document rather than text. +# Both of these therefore cost what the edit alone costs, and +# neither the formatting asked for nor the reading it would have +# taken happens. +# +FLUSH STATUS; +SELECT JSON_INSERT(JSON_COMPACT(js), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_INSERT(JSON_LOOSE(js), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# A value spliced into a document being built is read too, to find +# out whether it is a document at all. A column that is not typed +# as JSON is written out as a string instead and is not read. +# +FLUSH STATUS; +SELECT JSON_ARRAY(js, js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# A column that IS typed as JSON is read, the type being a promise +# nothing on the way here has checked. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j, j) FROM t2; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# A value that something has already read and attested to is not +# read again. Two values, and neither of them costs a reading, +# so what is left is what the two of them cost to make. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$.a'), JSON_EXTRACT(js, '$.b')) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_OBJECT('x', JSON_EXTRACT(js, '$.a'), 'y', JSON_EXTRACT(js, '$.b')) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_INSERT(js, '$.c', 3), JSON_INSERT(js, '$.d', 4)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# The same where the value goes into a document being EDITED +# rather than built. +# +FLUSH STATUS; +SELECT JSON_INSERT(js, '$.c', JSON_EXTRACT(js, '$.a')) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', +JSON_INSERT(js, '$.e', 5)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +# +# What is skipped there is a reading that would have measured how +# deep the value goes, and how deep it ends up decides whether the +# result can be read at all. How long the value is answers that +# for a short one - it cannot nest deeper than half its length - +# and answers nothing for a long one, which is read as before. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$')) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# Asking whether a value is a document reads it once. There is +# nothing here left to take away. +# +FLUSH STATUS; +SELECT JSON_VALID(js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# Nothing read, nothing counted. +# +FLUSH STATUS; +SELECT id FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# A function that only reports on a document parses it through to +# check that it is one, and that parse is the whole of what it +# costs on a large document. Out of a column nothing has attested to +# anything, so it is made: once to check the document and once to +# read the value being reported on. +# +FLUSH STATUS; +SELECT JSON_TYPE(js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# The same over a document an item has attested is_valid. The +# editing costs what it always did, and the reporting costs one +# reading rather than two: there is nothing left to check. +# +FLUSH STATUS; +SELECT JSON_TYPE(JSON_INSERT(js, '$.c', 3)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# The answers are unchanged either way. JSON_LENGTH, +# JSON_EXTRACT and JSON_CONTAINS_PATH stop parsing early over an +# attested value too, which no count here can show - they carry +# the parse on inside one reading rather than starting another. +# +SELECT JSON_TYPE(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_type, +JSON_TYPE('{"a":1,"b":2}') AS plain_type; +attested_type plain_type +OBJECT OBJECT +SELECT JSON_LENGTH(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_length, +JSON_LENGTH('{"a":1,"b":2}') AS plain_length; +attested_length plain_length +2 2 +SELECT JSON_EXTRACT(JSON_INSERT('{"a":1}', '$.b', 2), '$.a') AS attested_x, +JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS plain_x; +attested_x plain_x +1 1 +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), +'one', '$.a') AS attested_cp, +JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a') AS plain_cp; +attested_cp plain_cp +1 1 +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), +'all', '$.a', '$.b') AS attested_cp_all, +JSON_CONTAINS_PATH('{"a":1,"b":2}', 'all', '$.a', '$.b') AS plain_cp_all; +attested_cp_all plain_cp_all +1 1 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +# +# A document attested in one character set, spliced into a +# result being built in another. It is converted first, because the +# result is read in the character set it is built in - and a +# conversion that lost nothing kept the characters it was given, so +# what was answered about the value still holds of what came out of +# it. One reading for the value that was built, none for putting it +# in. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', +CONVERT(JSON_OBJECT('a', 1) USING latin1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# Two of them, one reading each and none for either splice. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', +CONVERT(JSON_OBJECT('a', 1) USING latin1), +'$.q', +CONVERT(JSON_OBJECT('b', 2) USING latin1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +# +# The same value with no conversion in the way, which is what the +# two above now cost. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A conversion that could not encode a character put a question mark +# in its place, so the value that comes out is not the value that +# was attested. The bytes that arrived go in unchanged, as they +# always have, and they are read. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', +CONVERT(JSON_OBJECT('a', _utf8mb4 0xE4BDA0) USING latin1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 diff --git a/mysql-test/main/func_json_scan_count.test b/mysql-test/main/func_json_scan_count.test new file mode 100644 index 0000000000000..71163208b4259 --- /dev/null +++ b/mysql-test/main/func_json_scan_count.test @@ -0,0 +1,434 @@ +--source include/have_debug.inc + +# +# How many times the JSON functions read a value. +# +# These functions return the same answer however many times they read +# what they are working on, so no result anywhere says how much reading +# was done. Json_scans says it, and nothing else does: a reading that +# stops happening shows up here or nowhere. +# +# The counts below are therefore not a statement about anything a caller +# can see. They are a record of work, kept so that work taken away +# later has to be taken away on purpose and in the open, and so that +# work quietly added back shows up as a test that fails. +# +# A debug build also reads values back to check what the functions claim +# about them. That reading is not counted - it is the build's work and +# not the server's, and counting it would move these numbers for reasons +# no query is responsible for. +# + +CREATE TABLE t1 (id INT, js VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); + +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); + +CREATE TABLE t3 (id INT, js VARCHAR(200)); +INSERT INTO t3 VALUES (1, CONCAT('{"a":"', REPEAT('x', 80), '"}')), + (2, CONCAT('{"a":"', REPEAT('y', 80), '"}')); + +--echo # +--echo # A function that edits a document reads it once to find the place +--echo # to edit, and then reads its own result back to find out how to +--echo # write it. Two rows, two readings each. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(js, '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET(js, '$.a', 9) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_REPLACE(js, '$.a', 9) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_REMOVE(js, '$.a') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_APPEND(js, '$.a', 7) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_INSERT(js, '$.a[0]', 7) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Merging reads both documents, and then reads the result. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(js, '{"c":3}') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE_PATCH(js, '{"c":3}') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A function that only reads pays for ONE reading, not two. The +--echo # walk to the path and the writing out of what was found are the +--echo # same reading carried on, so nothing is read twice; that is what +--echo # this work did to reading, and the count below is where it says +--echo # so. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_EXTRACT(js, '$.a') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A path that matches nothing costs the same one reading, there +--echo # being nothing to write out and nothing to read back either way. +--echo # It is here to say that finding something is what has stopped +--echo # costing extra, not that finding nothing ever did. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_EXTRACT(js, '$.zz') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Chained edits pay only for the innermost link. That one is +--echo # given a document out of a column that attests is_valid false, +--echo # so it reads its result back before returning it. Every link +--echo # after it is handed a document answering is_valid, and +--echo # attests to its own result the same way. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_REMOVE(JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4), '$.a') + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same for each of the others, each handed the is_valid +--echo # result of the one before it. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b[0]', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The two above put their value at a key the document already +--echo # has, so nothing is put there and what they append to is the +--echo # scalar that was there all along, which has to be wrapped in an +--echo # array first. At a key the document does NOT have, the value +--echo # goes in and the array they append to is that value. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c[0]', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE_PATCH(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A merge whose other argument attests is_valid false reads the +--echo # answer back, every document that goes into it having a say in +--echo # how it is written. A written-out literal is such a document: +--echo # nothing has read it, so nothing can say how it is written. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), '{"d":4}') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A document answering is_valid but NOT is_nice is read back like +--echo # one answering is_valid false. +--echo # Picking a piece out of a document returns the piece as it was +--echo # formatted where it came from, so that is such a document; picking +--echo # values out and putting them together writes them out afresh, so +--echo # that one is not. The same edit is put to each below, and the +--echo # difference between the two counts is the reading back. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_QUERY(js, '$'), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Asking for a document in a particular formatting reads nothing at +--echo # all when what asked for it wanted a document rather than text. +--echo # Both of these therefore cost what the edit alone costs, and +--echo # neither the formatting asked for nor the reading it would have +--echo # taken happens. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_COMPACT(js), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_LOOSE(js), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A value spliced into a document being built is read too, to find +--echo # out whether it is a document at all. A column that is not typed +--echo # as JSON is written out as a string instead and is not read. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(js, js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A column that IS typed as JSON is read, the type being a promise +--echo # nothing on the way here has checked. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(j, j) FROM t2; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A value that something has already read and attested to is not +--echo # read again. Two values, and neither of them costs a reading, +--echo # so what is left is what the two of them cost to make. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$.a'), JSON_EXTRACT(js, '$.b')) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_OBJECT('x', JSON_EXTRACT(js, '$.a'), 'y', JSON_EXTRACT(js, '$.b')) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_INSERT(js, '$.c', 3), JSON_INSERT(js, '$.d', 4)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same where the value goes into a document being EDITED +--echo # rather than built. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(js, '$.c', JSON_EXTRACT(js, '$.a')) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', + JSON_INSERT(js, '$.e', 5)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # What is skipped there is a reading that would have measured how +--echo # deep the value goes, and how deep it ends up decides whether the +--echo # result can be read at all. How long the value is answers that +--echo # for a short one - it cannot nest deeper than half its length - +--echo # and answers nothing for a long one, which is read as before. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$')) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Asking whether a value is a document reads it once. There is +--echo # nothing here left to take away. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_VALID(js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Nothing read, nothing counted. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT id FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + + +--echo # +--echo # A function that only reports on a document parses it through to +--echo # check that it is one, and that parse is the whole of what it +--echo # costs on a large document. Out of a column nothing has attested to +--echo # anything, so it is made: once to check the document and once to +--echo # read the value being reported on. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_TYPE(js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same over a document an item has attested is_valid. The +--echo # editing costs what it always did, and the reporting costs one +--echo # reading rather than two: there is nothing left to check. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_TYPE(JSON_INSERT(js, '$.c', 3)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The answers are unchanged either way. JSON_LENGTH, +--echo # JSON_EXTRACT and JSON_CONTAINS_PATH stop parsing early over an +--echo # attested value too, which no count here can show - they carry +--echo # the parse on inside one reading rather than starting another. +--echo # +SELECT JSON_TYPE(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_type, + JSON_TYPE('{"a":1,"b":2}') AS plain_type; +SELECT JSON_LENGTH(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_length, + JSON_LENGTH('{"a":1,"b":2}') AS plain_length; +SELECT JSON_EXTRACT(JSON_INSERT('{"a":1}', '$.b', 2), '$.a') AS attested_x, + JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS plain_x; +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), + 'one', '$.a') AS attested_cp, + JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a') AS plain_cp; +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), + 'all', '$.a', '$.b') AS attested_cp_all, + JSON_CONTAINS_PATH('{"a":1,"b":2}', 'all', '$.a', '$.b') AS plain_cp_all; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; + +--echo # +--echo # A document attested in one character set, spliced into a +--echo # result being built in another. It is converted first, because the +--echo # result is read in the character set it is built in - and a +--echo # conversion that lost nothing kept the characters it was given, so +--echo # what was answered about the value still holds of what came out of +--echo # it. One reading for the value that was built, none for putting it +--echo # in. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', + CONVERT(JSON_OBJECT('a', 1) USING latin1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Two of them, one reading each and none for either splice. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', + CONVERT(JSON_OBJECT('a', 1) USING latin1), + '$.q', + CONVERT(JSON_OBJECT('b', 2) USING latin1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same value with no conversion in the way, which is what the +--echo # two above now cost. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', JSON_OBJECT('a', 1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A conversion that could not encode a character put a question mark +--echo # in its place, so the value that comes out is not the value that +--echo # was attested. The bytes that arrived go in unchanged, as they +--echo # always have, and they are read. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', + CONVERT(JSON_OBJECT('a', _utf8mb4 0xE4BDA0) USING latin1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; diff --git a/mysql-test/main/func_json_search_typed.result b/mysql-test/main/func_json_search_typed.result new file mode 100644 index 0000000000000..95b1a990974e0 --- /dev/null +++ b/mysql-test/main/func_json_search_typed.result @@ -0,0 +1,87 @@ +# +# JSON_SEARCH always returns a document. +# +# Every path it produces is built out of pieces of a document it +# has just read through, and the punctuation round them is its +# own. Several paths go inside brackets, which needs a character +# set that can encode a bracket; a document written in one that +# cannot is a scalar, there being no way to write a container in +# it, so it holds one value and one path comes back. +# +# So the answer is a document whatever was asked, and a column +# filled from this function is a column of documents. What that +# buys is the reading that a later reader of the column would +# otherwise have to do to find that out. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":"x","b":"y"}'), (2, '{"c":"x","d":"z"}'); +# +# 1. The two shapes of answer +# +SELECT id, JSON_SEARCH(j, 'one', 'x') AS one_path, +JSON_SEARCH(j, 'all', 'x') AS all_paths FROM t1 ORDER BY id; +id one_path all_paths +1 "$.a" "$.a" +2 "$.c" "$.c" +SELECT id, JSON_TYPE(JSON_SEARCH(j, 'one', 'x')) AS one_ty, +JSON_TYPE(JSON_SEARCH(j, 'all', '%')) AS all_ty +FROM t1 ORDER BY id; +id one_ty all_ty +1 STRING ARRAY +2 STRING ARRAY +# +# 2. Branches of a union agreeing about it +# +# A union settles what its column holds from what every branch +# says it always returns, and both of these say a document. +# +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS p +UNION +SELECT JSON_SEARCH('{"c":"z"}', 'one', 'z'); +p +"$.a" +"$.c" +SELECT JSON_TYPE(p) AS ty FROM +(SELECT JSON_SEARCH('{"a":"x"}', 'all', '%') AS p +UNION ALL +SELECT JSON_SEARCH('{"b":"y","c":"y"}', 'all', '%')) u; +ty +STRING +ARRAY +# +# 3. A list of values written out in the statement +# +VALUES (JSON_SEARCH('{"a":"x"}', 'one', 'x')), +(JSON_SEARCH('{"b":"y"}', 'one', 'y')); +JSON_SEARCH('{"a":"x"}', 'one', 'x') +"$.a" +"$.b" +# +# 4. A column of a table the server built for itself +# +SELECT p FROM (SELECT JSON_SEARCH(j, 'all', '%') AS p FROM t1) d +GROUP BY p ORDER BY p; +p +["$.a", "$.b"] +["$.c", "$.d"] +CREATE TABLE t2 AS SELECT id, JSON_SEARCH(j, 'all', '%') AS p FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t2 ORDER BY id; +id p ty +1 ["$.a", "$.b"] ARRAY +2 ["$.c", "$.d"] ARRAY +DROP TABLE t2; +# +# 5. A column of a table the statement named +# +# A path written into a column that only takes documents is +# accepted, which is the same claim put the other way round. +# +CREATE TABLE t3 (id INT, p VARCHAR(64) CHECK (JSON_VALID(p))) CHARSET utf8mb4; +INSERT INTO t3 SELECT id, JSON_SEARCH(j, 'all', '%') FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t3 ORDER BY id; +id p ty +1 ["$.a", "$.b"] ARRAY +2 ["$.c", "$.d"] ARRAY +DROP TABLE t3; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_search_typed.test b/mysql-test/main/func_json_search_typed.test new file mode 100644 index 0000000000000..9ea0a080c86f2 --- /dev/null +++ b/mysql-test/main/func_json_search_typed.test @@ -0,0 +1,75 @@ +--source include/have_debug.inc + +--echo # +--echo # JSON_SEARCH always returns a document. +--echo # +--echo # Every path it produces is built out of pieces of a document it +--echo # has just read through, and the punctuation round them is its +--echo # own. Several paths go inside brackets, which needs a character +--echo # set that can encode a bracket; a document written in one that +--echo # cannot is a scalar, there being no way to write a container in +--echo # it, so it holds one value and one path comes back. +--echo # +--echo # So the answer is a document whatever was asked, and a column +--echo # filled from this function is a column of documents. What that +--echo # buys is the reading that a later reader of the column would +--echo # otherwise have to do to find that out. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":"x","b":"y"}'), (2, '{"c":"x","d":"z"}'); + +--echo # +--echo # 1. The two shapes of answer +--echo # +SELECT id, JSON_SEARCH(j, 'one', 'x') AS one_path, + JSON_SEARCH(j, 'all', 'x') AS all_paths FROM t1 ORDER BY id; +SELECT id, JSON_TYPE(JSON_SEARCH(j, 'one', 'x')) AS one_ty, + JSON_TYPE(JSON_SEARCH(j, 'all', '%')) AS all_ty + FROM t1 ORDER BY id; + +--echo # +--echo # 2. Branches of a union agreeing about it +--echo # +--echo # A union settles what its column holds from what every branch +--echo # says it always returns, and both of these say a document. +--echo # +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS p +UNION +SELECT JSON_SEARCH('{"c":"z"}', 'one', 'z'); + +SELECT JSON_TYPE(p) AS ty FROM + (SELECT JSON_SEARCH('{"a":"x"}', 'all', '%') AS p + UNION ALL + SELECT JSON_SEARCH('{"b":"y","c":"y"}', 'all', '%')) u; + +--echo # +--echo # 3. A list of values written out in the statement +--echo # +VALUES (JSON_SEARCH('{"a":"x"}', 'one', 'x')), + (JSON_SEARCH('{"b":"y"}', 'one', 'y')); + +--echo # +--echo # 4. A column of a table the server built for itself +--echo # +SELECT p FROM (SELECT JSON_SEARCH(j, 'all', '%') AS p FROM t1) d + GROUP BY p ORDER BY p; + +CREATE TABLE t2 AS SELECT id, JSON_SEARCH(j, 'all', '%') AS p FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t2 ORDER BY id; +DROP TABLE t2; + +--echo # +--echo # 5. A column of a table the statement named +--echo # +--echo # A path written into a column that only takes documents is +--echo # accepted, which is the same claim put the other way round. +--echo # +CREATE TABLE t3 (id INT, p VARCHAR(64) CHECK (JSON_VALID(p))) CHARSET utf8mb4; +INSERT INTO t3 SELECT id, JSON_SEARCH(j, 'all', '%') FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t3 ORDER BY id; +DROP TABLE t3; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_splice_depth.result b/mysql-test/main/func_json_splice_depth.result new file mode 100644 index 0000000000000..83a4475ba1644 --- /dev/null +++ b/mysql-test/main/func_json_splice_depth.result @@ -0,0 +1,143 @@ +SET @d29= CONCAT(REPEAT('{"a":', 29), '1', REPEAT('}', 29)); +SET @d30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SET @p30= CONCAT('$', REPEAT('.a', 30)); +# +# 1. Two structures put 30 deep make 32 and do not read back; +# the same two put 29 deep make 31 and do +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, +JSON_EXTRACT('{"b":{"c":1}}', '$')) AS too_deep; +too_deep +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_replace' at position 15 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 187 +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, +JSON_EXTRACT('{"b":{"c":1}}', '$')) AS in_limit; +in_limit +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"b": {"c": 1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 2. A value of nothing but brackets, where how long it is says +# exactly how deep it goes and there is no room to be wrong by +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, +JSON_EXTRACT('[[1]]', '$')) AS br_limit; +br_limit +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": [[1]]}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, +JSON_EXTRACT('[[1]]', '$')) AS br_deep; +br_deep +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_replace' at position 5 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 182 +# +# 3. A scalar sits at whatever depth it is put and adds none of +# its own +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, +JSON_EXTRACT('[7]', '$[0]')) AS scalar_deep; +scalar_deep +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": 7}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 4. The same through a document answering is_valid false, which is +# read back in full and so is where the limit has always been +# noticed +# +SELECT JSON_REPLACE(@d30, @p30, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_deep; +raw_deep +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 157 +SELECT JSON_REPLACE(@d29, @p29, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_limit; +raw_limit +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"b": {"c": 1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 5. A value written out by hand is a string rather than a +# document, so it goes in quoted and takes one level whatever +# is written in it +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, '{"b":{"c":1}}') AS quoted; +quoted +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": "{\"b\":{\"c\":1}}"}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 6. A function that BUILDS a document puts the value one level +# inside the array or object it is making +# +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d30a= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_ARRAY(JSON_EXTRACT(@d31, '$')) AS built_deep; +built_deep +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT JSON_ARRAY(JSON_EXTRACT(@d30a, '$')) AS built_limit; +built_limit +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +# +# 10. Wrapping, where the new array is a structure the document +# being read never had. What was found is put inside it and +# the value goes in beside them both, so the value sits one +# level below where the path reached. +# +# A value written out as a string has nothing inside it, so +# where it sits is the whole of how deep the answer goes and +# there is nothing left to read to find that out. +# +SET @w31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @w30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @wp31= CONCAT('$', REPEAT('[0]', 31)); +SET @wp30= CONCAT('$', REPEAT('[0]', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, 2) AS wrap_deep; +wrap_deep +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +SELECT JSON_ARRAY_APPEND(@w31, @wp31, 2) AS wrap_deep_read_back; +wrap_deep_read_back +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +# +# One level shallower both answer, which is what says the two +# above are refused for their depth and not for their shape. +# +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w30, '$'), @wp30, 2) AS wrap_limit; +wrap_limit +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1, 2]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT JSON_ARRAY_APPEND(@w30, @wp30, 2) AS wrap_limit_read_back; +wrap_limit_read_back +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1, 2]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +# +# A value that is a document of its own is read, and reaches +# the same refusal through what the reading found rather than +# through where it was put. +# +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, +JSON_EXTRACT('2', '$')) AS wrap_deep_typed; +wrap_deep_typed +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_array_append' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +# +# The functions that wrap while INSERTING name the place they +# are inserting at, so reaching this far down takes a path one +# step longer than a path is allowed to be, and the path is +# what refuses them. Appending names the value instead and +# needs no step for the array it adds, which is why it is the +# one that gets here. +# +SELECT JSON_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) +AS insert_wrap_deep; +insert_wrap_deep +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_insert' at position 96 +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) +AS ainsert_wrap_deep; +ainsert_wrap_deep +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_array_insert' at position 96 diff --git a/mysql-test/main/func_json_splice_depth.test b/mysql-test/main/func_json_splice_depth.test new file mode 100644 index 0000000000000..ad4241e2d98ff --- /dev/null +++ b/mysql-test/main/func_json_splice_depth.test @@ -0,0 +1,118 @@ +# +# How deep a value put into a document is allowed to end up. +# +# A document can hold 32 structures inside one another and no more. +# A value spliced into one is written where the path reached, so how +# deep it ends up is how deep the path went plus how deep the value +# goes on its own, and it is the total the limit is about. +# +# The value's own depth is found by reading it. A value that something +# else has already read and attested to is not read again, so the +# reading that would have measured it is not done - except that it does +# not have to be, a value only nesting as deeply as it is long. The +# cases below are the ones that tell those two apart: a value of +# nothing but brackets is as deep as half its length and is where the +# reckoning has to be exact, and a scalar is no deep at all however +# long the document holding it. +# +# The answers are the ones the server has always given. +# + +SET @d29= CONCAT(REPEAT('{"a":', 29), '1', REPEAT('}', 29)); +SET @d30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SET @p30= CONCAT('$', REPEAT('.a', 30)); + +--echo # +--echo # 1. Two structures put 30 deep make 32 and do not read back; +--echo # the same two put 29 deep make 31 and do +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, + JSON_EXTRACT('{"b":{"c":1}}', '$')) AS too_deep; +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, + JSON_EXTRACT('{"b":{"c":1}}', '$')) AS in_limit; + +--echo # +--echo # 2. A value of nothing but brackets, where how long it is says +--echo # exactly how deep it goes and there is no room to be wrong by +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, + JSON_EXTRACT('[[1]]', '$')) AS br_limit; +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, + JSON_EXTRACT('[[1]]', '$')) AS br_deep; + +--echo # +--echo # 3. A scalar sits at whatever depth it is put and adds none of +--echo # its own +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, + JSON_EXTRACT('[7]', '$[0]')) AS scalar_deep; + +--echo # +--echo # 4. The same through a document answering is_valid false, which is +--echo # read back in full and so is where the limit has always been +--echo # noticed +--echo # +SELECT JSON_REPLACE(@d30, @p30, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_deep; +SELECT JSON_REPLACE(@d29, @p29, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_limit; + +--echo # +--echo # 5. A value written out by hand is a string rather than a +--echo # document, so it goes in quoted and takes one level whatever +--echo # is written in it +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, '{"b":{"c":1}}') AS quoted; + +--echo # +--echo # 6. A function that BUILDS a document puts the value one level +--echo # inside the array or object it is making +--echo # +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d30a= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_ARRAY(JSON_EXTRACT(@d31, '$')) AS built_deep; +SELECT JSON_ARRAY(JSON_EXTRACT(@d30a, '$')) AS built_limit; + +--echo # +--echo # 10. Wrapping, where the new array is a structure the document +--echo # being read never had. What was found is put inside it and +--echo # the value goes in beside them both, so the value sits one +--echo # level below where the path reached. +--echo # +--echo # A value written out as a string has nothing inside it, so +--echo # where it sits is the whole of how deep the answer goes and +--echo # there is nothing left to read to find that out. +--echo # +SET @w31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @w30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @wp31= CONCAT('$', REPEAT('[0]', 31)); +SET @wp30= CONCAT('$', REPEAT('[0]', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, 2) AS wrap_deep; +SELECT JSON_ARRAY_APPEND(@w31, @wp31, 2) AS wrap_deep_read_back; + +--echo # +--echo # One level shallower both answer, which is what says the two +--echo # above are refused for their depth and not for their shape. +--echo # +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w30, '$'), @wp30, 2) AS wrap_limit; +SELECT JSON_ARRAY_APPEND(@w30, @wp30, 2) AS wrap_limit_read_back; + +--echo # +--echo # A value that is a document of its own is read, and reaches +--echo # the same refusal through what the reading found rather than +--echo # through where it was put. +--echo # +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, + JSON_EXTRACT('2', '$')) AS wrap_deep_typed; + +--echo # +--echo # The functions that wrap while INSERTING name the place they +--echo # are inserting at, so reaching this far down takes a path one +--echo # step longer than a path is allowed to be, and the path is +--echo # what refuses them. Appending names the value instead and +--echo # needs no step for the array it adds, which is why it is the +--echo # one that gets here. +--echo # +SELECT JSON_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) + AS insert_wrap_deep; +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) + AS ainsert_wrap_deep; diff --git a/mysql-test/main/func_json_splice_oom.result b/mysql-test/main/func_json_splice_oom.result new file mode 100644 index 0000000000000..6fa48c9042549 --- /dev/null +++ b/mysql-test/main/func_json_splice_oom.result @@ -0,0 +1,240 @@ +# +# No room to grow the buffer while a value is being written out +# into the document it is joining. +# +# A value that is a document but is not written the loose way is +# written out again as it is spliced, whenever the document it +# joins is loose and would stop being so otherwise. That writing +# grows the buffer the same way every other writing here does, +# and can fail the same way. +# +# The failure has to be reachable from this writing and not only +# from the older ones. A writing that no injection can fail is a +# writing whose handling of failure nobody has ever run, and this +# file is what keeps that from happening quietly. +# +SET NAMES utf8mb4; +# +# 1. A value that is a document, and is not loose +# +# Read out of a document written the compact way, so it is +# formatted that way itself and has to be written out again when +# it joins one that is loose. +# +SET @compact = CONCAT('[', REPEAT('1,', 999), '1]'); +SET @small = '[1,2]'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) +AS spliced; +spliced +{"a": 1, "d": [1, 2]} +SELECT LENGTH(@compact) AS compact_len, +LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; +compact_len grew_by +2001 999 +# +# 2. The same splice with no room to grow +# +# This must not answer. If it does, the injection is not +# reaching the writing above and this file is testing nothing. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@compact, '$')) +AS spliced; +ERROR HY000: Out of memory (Needed 1288 bytes) +SET SESSION debug_dbug = DEFAULT; +# +# 3. The same statement once the room is there again +# +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', +JSON_QUERY(@compact, '$')), '')) +AS parses; +parses +1 +# +# 4. The other functions that splice a value the same way +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) +AS st; +SELECT JSON_REPLACE(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) +AS rep; +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', JSON_QUERY(@compact, '$')) +AS app; +SELECT JSON_ARRAY_INSERT(JSON_ARRAY(1), '$[0]', JSON_QUERY(@compact, '$')) +AS ari; +SET SESSION debug_dbug = DEFAULT; +# and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(JSON_OBJECT('a', 1), '$.a', +JSON_QUERY(@compact, '$')), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', +JSON_QUERY(@compact, '$')), '')) AS parses; +parses +1 +# +# 5. A value short enough that the room asked for is enough, +# which must be unaffected +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) +AS spliced; +spliced +{"a": 1, "d": [1, 2]} +SET SESSION debug_dbug = DEFAULT; +# +# 6. Several rows, so the failure is met more than once +# +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', +JSON_QUERY(j, '$')), '')) AS parses +FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', +JSON_QUERY(j, '$')), '')) AS parses +FROM t1 ORDER BY id; +id parses +1 1 +2 1 +3 1 +DROP TABLE t1; +# +# 7. No room to pass a document that was composed in place +# +# Where the item behind every document a function was given attests +# is_valid and is_nice, what it composed is the answer and is +# returned rather than read back. Returning it is a copy, and +# a copy is a write that can run short of room like any other. +# +# Every argument is built by a producer that attests to what +# it passes. A string literal attests is_valid false for +# its document, so a statement built out of literals +# never composes an answer at all and would say nothing here. +# +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) AS whole; +whole +{"a": [1, 2]} +SET SESSION debug_dbug = '+d,json_return_out_of_memory'; +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) +AS appended; +appended +NULL +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) +AS inserted; +inserted +NULL +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +added +NULL +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +removed +NULL +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS merged; +merged +NULL +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS patched; +patched +NULL +SET SESSION debug_dbug = DEFAULT; +# and the same six once the room is there again, which is what +# says the refusals above are about the copy and not about the +# statements +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) +AS appended; +appended +{"a": [1, 2]} +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) +AS inserted; +inserted +{"a": [2, 1]} +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +added +{"a": 1, "b": 2} +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +removed +{"a": 1} +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS merged; +merged +{"a": 1, "b": 2} +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS patched; +patched +{"a": 1, "b": 2} +# +# 8. No room while the values a path asked for are written out +# +# A reading function writes the values it found into the answer +# itself, with a comma between them where more than one was +# asked for, rather than composing a document and reading it +# back. Both writes are its own and both can run short. +# +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS whole, +JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS whole_one; +whole whole_one +[1, 2] 1 +SET SESSION debug_dbug = '+d,json_extract_comma_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_comma; +no_comma +NULL +# one value alone needs no comma, so this one is untouched +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS one_value; +one_value +1 +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_extract_scalar_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS no_value; +no_value +NULL +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_value_of_several; +no_value_of_several +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 9. No room for a key the document did not have +# +# A path naming a key that is not there has the key written +# into the document, which is a write the composing does +# rather than one a reading back would have done. +# +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS whole; +whole +{"a": 1, "b": 2} +SET SESSION debug_dbug = '+d,json_insert_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_key; +no_key +NULL +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_key_set; +no_key_set +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 10. The colon that joins a key to its value, and the key a path +# names that the document did not have +# +# Both are written in the formatting this commit settled, so +# both are failed here rather than where the key itself is. +# +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole, +JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole_added; +whole whole_added +{"k": 1, "l": 2} {"a": 1, "b": 2} +SET SESSION debug_dbug = '+d,json_keyname_colon_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_colon; +no_colon +NULL +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_insert_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_room_added_key; +no_room_added_key +NULL +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_room_added_key_set; +no_room_added_key_set +NULL +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_splice_oom.test b/mysql-test/main/func_json_splice_oom.test new file mode 100644 index 0000000000000..a72d7d0a1e727 --- /dev/null +++ b/mysql-test/main/func_json_splice_oom.test @@ -0,0 +1,200 @@ +--source include/have_debug.inc + +--echo # +--echo # No room to grow the buffer while a value is being written out +--echo # into the document it is joining. +--echo # +--echo # A value that is a document but is not written the loose way is +--echo # written out again as it is spliced, whenever the document it +--echo # joins is loose and would stop being so otherwise. That writing +--echo # grows the buffer the same way every other writing here does, +--echo # and can fail the same way. +--echo # +--echo # The failure has to be reachable from this writing and not only +--echo # from the older ones. A writing that no injection can fail is a +--echo # writing whose handling of failure nobody has ever run, and this +--echo # file is what keeps that from happening quietly. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A value that is a document, and is not loose +--echo # +--echo # Read out of a document written the compact way, so it is +--echo # formatted that way itself and has to be written out again when +--echo # it joins one that is loose. +--echo # +SET @compact = CONCAT('[', REPEAT('1,', 999), '1]'); +SET @small = '[1,2]'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) + AS spliced; +SELECT LENGTH(@compact) AS compact_len, + LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; + +--echo # +--echo # 2. The same splice with no room to grow +--echo # +--echo # This must not answer. If it does, the injection is not +--echo # reaching the writing above and this file is testing nothing. +--echo # +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 5 +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@compact, '$')) + AS spliced; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 3. The same statement once the room is there again +--echo # +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', + JSON_QUERY(@compact, '$')), '')) + AS parses; + +--echo # +--echo # 4. The other functions that splice a value the same way +--echo # +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) + AS st; +--error 0,5 +SELECT JSON_REPLACE(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) + AS rep; +--error 0,5 +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', JSON_QUERY(@compact, '$')) + AS app; +--error 0,5 +SELECT JSON_ARRAY_INSERT(JSON_ARRAY(1), '$[0]', JSON_QUERY(@compact, '$')) + AS ari; +SET SESSION debug_dbug = DEFAULT; + +--echo # and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(JSON_OBJECT('a', 1), '$.a', + JSON_QUERY(@compact, '$')), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', + JSON_QUERY(@compact, '$')), '')) AS parses; + +--echo # +--echo # 5. A value short enough that the room asked for is enough, +--echo # which must be unaffected +--echo # +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) + AS spliced; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 6. Several rows, so the failure is met more than once +--echo # +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', + JSON_QUERY(j, '$')), '')) AS parses + FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', + JSON_QUERY(j, '$')), '')) AS parses + FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 7. No room to pass a document that was composed in place +--echo # +--echo # Where the item behind every document a function was given attests +--echo # is_valid and is_nice, what it composed is the answer and is +--echo # returned rather than read back. Returning it is a copy, and +--echo # a copy is a write that can run short of room like any other. +--echo # +--echo # Every argument is built by a producer that attests to what +--echo # it passes. A string literal attests is_valid false for +--echo # its document, so a statement built out of literals +--echo # never composes an answer at all and would say nothing here. +--echo # +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) AS whole; + +SET SESSION debug_dbug = '+d,json_return_out_of_memory'; +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) + AS appended; +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) + AS inserted; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS merged; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS patched; +SET SESSION debug_dbug = DEFAULT; + +--echo # and the same six once the room is there again, which is what +--echo # says the refusals above are about the copy and not about the +--echo # statements +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) + AS appended; +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) + AS inserted; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS merged; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS patched; + +--echo # +--echo # 8. No room while the values a path asked for are written out +--echo # +--echo # A reading function writes the values it found into the answer +--echo # itself, with a comma between them where more than one was +--echo # asked for, rather than composing a document and reading it +--echo # back. Both writes are its own and both can run short. +--echo # +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS whole, + JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS whole_one; + +SET SESSION debug_dbug = '+d,json_extract_comma_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_comma; +--echo # one value alone needs no comma, so this one is untouched +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS one_value; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_extract_scalar_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS no_value; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_value_of_several; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 9. No room for a key the document did not have +--echo # +--echo # A path naming a key that is not there has the key written +--echo # into the document, which is a write the composing does +--echo # rather than one a reading back would have done. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS whole; + +SET SESSION debug_dbug = '+d,json_insert_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_key; +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_key_set; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 10. The colon that joins a key to its value, and the key a path +--echo # names that the document did not have +--echo # +--echo # Both are written in the formatting this commit settled, so +--echo # both are failed here rather than where the key itself is. +--echo # +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole, + JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole_added; + +SET SESSION debug_dbug = '+d,json_keyname_colon_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_colon; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_insert_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_room_added_key; +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_room_added_key_set; +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_trusted.result b/mysql-test/main/func_json_trusted.result new file mode 100644 index 0000000000000..efddccfc697c8 --- /dev/null +++ b/mysql-test/main/func_json_trusted.result @@ -0,0 +1,301 @@ +# +# 1. A key both documents hold +# +# Merging writes the value of such a key itself rather than +# copying it out of either document, so the punctuation in +# front of it is the composer's to get right. +# +SELECT JSON_MERGE_PRESERVE('{"a": 1}', '{"a": 2}') AS literal; +literal +{"a": [1, 2]} +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('a', 2)) AS built; +built +{"a": [1, 2]} +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), +JSON_EXTRACT('{"a": 2, "b": 3}', '$')) AS cut; +cut +{"a": [1, 2], "b": 3} +# +# The same where the shared value is a container, which the +# merging walks into and composes a level further down. +# +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_OBJECT('x', 1)), +JSON_OBJECT('a', JSON_OBJECT('y', 2))) AS objects; +objects +{"a": {"x": 1, "y": 2}} +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_ARRAY(1, 2)), +JSON_OBJECT('a', JSON_ARRAY(3, 4))) AS arrays; +arrays +{"a": [1, 2, 3, 4]} +# +# A key only one of them holds is copied across with the +# spacing it was written with, and is the control for the +# three above. +# +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) AS apart; +apart +{"a": 1, "b": 2} +# +# 2. Removing the first element of an array +# +# What is taken out reaches to the comma; the space after it +# belongs to the element that stays, and has to end up in +# front of it and not behind the bracket. +# +SELECT JSON_REMOVE('[1, 2, 3]', '$[0]') AS literal; +literal +[2, 3] +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') AS cut; +cut +[2, 3] +SELECT JSON_REMOVE(JSON_ARRAY(1, 2, 3), '$[0]') AS built; +built +[2, 3] +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": [1, 2, 3]}', '$'), '$.a[0]') AS nested; +nested +{"a": [2, 3]} +SELECT JSON_REMOVE(JSON_EXTRACT('[[1], 2, 3]', '$'), '$[0]') AS container; +container +[2, 3] +# +# Taking out anything other than the first element ends at a +# comma with nothing after it, so those are the controls. +# +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[1]') AS middle; +middle +[1, 3] +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[2]') AS last; +last +[1, 2] +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": 1, "b": 2}', '$'), '$.a') AS first_key; +first_key +{"b": 2} +# +# Where the document is nobody's word, what is composed is not +# the answer but what the reading back at the end is made +# from, and that reading is what complains about anything +# written after the document. It counts from the start of the +# text it was handed, so that text has to be the one a +# released server would have handed it - including the space +# the taking out would otherwise carry off, which that server +# leaves standing and drops later while writing the answer. +# +SELECT JSON_REMOVE('[1, 2] x', '$[0]') AS pos_first; +pos_first +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 6 +SELECT JSON_REMOVE('[1, 2, 3] x', '$[0]') AS pos_first_three; +pos_first_three +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 9 +# +# The controls, none of which has such a space to lose: one +# document written with no spacing at all, one where what is +# taken out is not the first piece, and one where it is a key. +# +SELECT JSON_REMOVE('[1,2] x', '$[0]') AS pos_compact; +pos_compact +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 5 +SELECT JSON_REMOVE('[1, 2] x', '$[1]') AS pos_second; +pos_second +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 5 +SELECT JSON_REMOVE('{"a": 1, "b": 2} x', '$.a') AS pos_key; +pos_key +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 10 +# +# 3. A value put where the document would end up too deep +# +# How deep the value ends up is the level it is put at plus +# its own; the level it is put at is read off the scanner, +# which by then has already left the thing being put into. +# +SET @deep30= CONCAT(REPEAT('[', 30), '{"k": 1}', REPEAT(']', 30)); +SET @newkey= CONCAT('$', REPEAT('[0]', 30), '.n'); +SET @pastend= CONCAT('$', REPEAT('[0]', 30), '[1]'); +SELECT JSON_INSERT(@deep30, @newkey, JSON_EXTRACT('[5]', '$')) +IS NULL AS literal_isnull; +literal_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 44 +SELECT JSON_INSERT(JSON_EXTRACT(@deep30, '$'), @newkey, +JSON_EXTRACT('[5]', '$')) IS NULL AS cut_isnull; +cut_isnull +1 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_insert' at position 3 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 45 +SELECT JSON_SET(JSON_EXTRACT(@deep30, '$'), @newkey, +JSON_EXTRACT('[5]', '$')) IS NULL AS set_isnull; +set_isnull +1 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_set' at position 3 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 45 +SET @arr30= CONCAT(REPEAT('[', 30), '[1]', REPEAT(']', 30)); +SELECT JSON_INSERT(@arr30, @pastend, JSON_EXTRACT('[5]', '$')) +IS NULL AS end_literal_isnull; +end_literal_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 35 +SELECT JSON_INSERT(JSON_EXTRACT(@arr30, '$'), @pastend, +JSON_EXTRACT('[5]', '$')) IS NULL AS end_cut_isnull; +end_cut_isnull +1 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_insert' at position 3 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 35 +# +# One level shallower, the same statements answer, and that +# is what says the two above are refused for their depth and +# not for their shape. +# +SET @deep20= CONCAT(REPEAT('[', 20), '{"k": 1}', REPEAT(']', 20)); +SET @newkey20= CONCAT('$', REPEAT('[0]', 20), '.n'); +SELECT JSON_INSERT(JSON_EXTRACT(@deep20, '$'), @newkey20, +JSON_EXTRACT('[5]', '$')) IS NULL AS shallow_isnull; +shallow_isnull +0 +# +# And what it actually composed, which is where a document +# that came out a character longer or shorter than a released +# server's would show itself. Asking only whether an answer +# came back would not. +# +SELECT JSON_INSERT(JSON_EXTRACT('[[{"k": 1}]]', '$'), '$[0][0].n', +JSON_EXTRACT('[5]', '$')) AS composed; +composed +[[{"k": 1, "n": [5]}]] +SELECT JSON_INSERT(JSON_EXTRACT('[[1, 2]]', '$'), '$[0][2]', +JSON_EXTRACT('[5]', '$')) AS composed_past_end; +composed_past_end +[[1, 2, [5]]] +SELECT JSON_SET(JSON_EXTRACT('{"a": {"b": 1}}', '$'), '$.a.c', +JSON_EXTRACT('{"z": 9}', '$')) AS composed_set; +composed_set +{"a": {"b": 1, "c": {"z": 9}}} +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', +JSON_EXTRACT('[3, 4]', '$')) AS composed_append; +composed_append +[1, 2, [3, 4]] +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', +JSON_EXTRACT('[3, 4]', '$')) AS composed_ainsert; +composed_ainsert +[[3, 4], 1, 2] +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('{"a": 1}', '$'), '$', +JSON_EXTRACT('[3, 4]', '$')) AS composed_autowrap; +composed_autowrap +[{"a": 1}, [3, 4]] +SELECT JSON_EXTRACT('{"a": [1, 2], "b": {"c": 3}}', '$.a', '$.b') +AS composed_two_paths; +composed_two_paths +[[1, 2], {"c": 3}] +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), JSON_EXTRACT('2', '$')) +AS composed_merge_wrap; +composed_merge_wrap +[{"a": 1}, 2] +# +# 4. The bracket a reading of more than one path adds +# +# Two matches are returned wrapped in an array, and that +# wrapper is a level nobody asked the document about. +# +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SELECT JSON_VALID(@d31) AS legal_on_its_own; +legal_on_its_own +1 +SELECT JSON_EXTRACT(@d31, '$') IS NULL AS one_path_isnull; +one_path_isnull +0 +SELECT JSON_EXTRACT(@d31, '$', '$') IS NULL AS two_paths_isnull; +two_paths_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_extract' at position 32 +SET @d30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_EXTRACT(@d30, '$', '$') IS NULL AS two_paths_fit_isnull; +two_paths_fit_isnull +0 +# +# 5. The bracket a merge adds +# +# Merging a document with something it cannot be merged into +# puts both in a new array, which is a level as well. +# +SET @o31= CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SELECT JSON_VALID(@o31) AS legal_on_its_own; +legal_on_its_own +1 +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('1', '$')) +IS NULL AS scalar_wrap_isnull; +scalar_wrap_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 182 +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('{"a": 1}', '$')) +IS NULL AS shared_key_isnull; +shared_key_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 182 +SET @o30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SELECT JSON_MERGE(JSON_EXTRACT(@o30, '$'), JSON_EXTRACT('1', '$')) +IS NULL AS wrap_fits_isnull; +wrap_fits_isnull +0 +# +# 6. The value an autowrap keeps +# +# Appending to something that is not an array wraps what is +# already there, so the KEPT value goes a level down too - +# not only the one being appended. +# +SET @p30= CONCAT('$', REPEAT('.a', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o31, '$'), @p30, 1) +IS NULL AS append_wrap_isnull; +append_wrap_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 182 +SELECT JSON_INSERT(JSON_EXTRACT(@o31, '$'), CONCAT(@p30, '[1]'), 1) +IS NULL AS insert_wrap_isnull; +insert_wrap_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 182 +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o30, '$'), @p29, 1) +IS NULL AS append_wrap_fits_isnull; +append_wrap_fits_isnull +0 +# +# 7. A document that fits, and the diagnostic it must not get +# +# The level a value is appended at is read off the scanner +# after it has already been counted, so counting it again +# refuses a document that fits - or lets it through with a +# note saying it did not fit, which is worse, the note being +# the only thing an application would go by. +# +SET @nest30= CONCAT(REPEAT('[', 30), REPEAT(']', 30)); +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_EXTRACT(@nest30, '$')) +IS NULL AS literal_isnull; +literal_isnull +0 +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1]', '$'), '$', +JSON_EXTRACT(@nest30, '$')) IS NULL AS cut_isnull; +cut_isnull +0 +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1]', '$'), '$[0]', +JSON_EXTRACT(@nest30, '$')) IS NULL AS insert_isnull; +insert_isnull +0 diff --git a/mysql-test/main/func_json_trusted.test b/mysql-test/main/func_json_trusted.test new file mode 100644 index 0000000000000..57f8c8b7d8c37 --- /dev/null +++ b/mysql-test/main/func_json_trusted.test @@ -0,0 +1,230 @@ +# +# The seven functions that used to read their own answer back now hand +# it as it was composed, whenever the item behind every document +# they were given attests is_valid and is_nice. +# +# Nothing else in the suite goes down that path. A string literal +# attests is_valid false for its document, so the reading back still +# happens and the composition is never the answer; a suite built out of +# literals can be entirely green while the composing is entirely wrong. +# Every case here is therefore built from a producer that attests to +# what it passes - JSON_OBJECT, JSON_ARRAY or JSON_EXTRACT - and is +# paired with the same statement written with a literal, which is what +# a released server runs. The pair has to agree. +# +# The answers in this file were recorded on a released 10.11 server, so +# what they say is what that server says, not what this one does. Five +# lines of it are not, and they are the only five: +# +# - Three of the refusals below carry an added Note naming the argument +# whose value would not fit, in front of the warning a released +# server gives on its own. Nothing is said less; something is said +# more, and what it says is true. +# +# - Two warning positions differ by one character from a released +# server's. A position is an offset into +# the text that was composed, and the text composed here is already +# written the loose way where a released server composed it compact +# and tidied it afterwards. Since these are the documents that are +# refused, the tidying never happened there, so its offsets are +# into a document that server threw away. Ours are into the one it +# would have returned. +# +# Every answer, and every one of the compositions written out below in +# full, is byte for byte what a released server gives. +# + +--enable_warnings + +--echo # +--echo # 1. A key both documents hold +--echo # +--echo # Merging writes the value of such a key itself rather than +--echo # copying it out of either document, so the punctuation in +--echo # front of it is the composer's to get right. +--echo # +SELECT JSON_MERGE_PRESERVE('{"a": 1}', '{"a": 2}') AS literal; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('a', 2)) AS built; +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), + JSON_EXTRACT('{"a": 2, "b": 3}', '$')) AS cut; + +--echo # +--echo # The same where the shared value is a container, which the +--echo # merging walks into and composes a level further down. +--echo # +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_OBJECT('x', 1)), + JSON_OBJECT('a', JSON_OBJECT('y', 2))) AS objects; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_ARRAY(1, 2)), + JSON_OBJECT('a', JSON_ARRAY(3, 4))) AS arrays; + +--echo # +--echo # A key only one of them holds is copied across with the +--echo # spacing it was written with, and is the control for the +--echo # three above. +--echo # +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) AS apart; + +--echo # +--echo # 2. Removing the first element of an array +--echo # +--echo # What is taken out reaches to the comma; the space after it +--echo # belongs to the element that stays, and has to end up in +--echo # front of it and not behind the bracket. +--echo # +SELECT JSON_REMOVE('[1, 2, 3]', '$[0]') AS literal; +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') AS cut; +SELECT JSON_REMOVE(JSON_ARRAY(1, 2, 3), '$[0]') AS built; +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": [1, 2, 3]}', '$'), '$.a[0]') AS nested; +SELECT JSON_REMOVE(JSON_EXTRACT('[[1], 2, 3]', '$'), '$[0]') AS container; + +--echo # +--echo # Taking out anything other than the first element ends at a +--echo # comma with nothing after it, so those are the controls. +--echo # +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[1]') AS middle; +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[2]') AS last; +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": 1, "b": 2}', '$'), '$.a') AS first_key; + +--echo # +--echo # Where the document is nobody's word, what is composed is not +--echo # the answer but what the reading back at the end is made +--echo # from, and that reading is what complains about anything +--echo # written after the document. It counts from the start of the +--echo # text it was handed, so that text has to be the one a +--echo # released server would have handed it - including the space +--echo # the taking out would otherwise carry off, which that server +--echo # leaves standing and drops later while writing the answer. +--echo # +SELECT JSON_REMOVE('[1, 2] x', '$[0]') AS pos_first; +SELECT JSON_REMOVE('[1, 2, 3] x', '$[0]') AS pos_first_three; + +--echo # +--echo # The controls, none of which has such a space to lose: one +--echo # document written with no spacing at all, one where what is +--echo # taken out is not the first piece, and one where it is a key. +--echo # +SELECT JSON_REMOVE('[1,2] x', '$[0]') AS pos_compact; +SELECT JSON_REMOVE('[1, 2] x', '$[1]') AS pos_second; +SELECT JSON_REMOVE('{"a": 1, "b": 2} x', '$.a') AS pos_key; + +--echo # +--echo # 3. A value put where the document would end up too deep +--echo # +--echo # How deep the value ends up is the level it is put at plus +--echo # its own; the level it is put at is read off the scanner, +--echo # which by then has already left the thing being put into. +--echo # +SET @deep30= CONCAT(REPEAT('[', 30), '{"k": 1}', REPEAT(']', 30)); +SET @newkey= CONCAT('$', REPEAT('[0]', 30), '.n'); +SET @pastend= CONCAT('$', REPEAT('[0]', 30), '[1]'); + +SELECT JSON_INSERT(@deep30, @newkey, JSON_EXTRACT('[5]', '$')) + IS NULL AS literal_isnull; +SELECT JSON_INSERT(JSON_EXTRACT(@deep30, '$'), @newkey, + JSON_EXTRACT('[5]', '$')) IS NULL AS cut_isnull; +SELECT JSON_SET(JSON_EXTRACT(@deep30, '$'), @newkey, + JSON_EXTRACT('[5]', '$')) IS NULL AS set_isnull; + +SET @arr30= CONCAT(REPEAT('[', 30), '[1]', REPEAT(']', 30)); +SELECT JSON_INSERT(@arr30, @pastend, JSON_EXTRACT('[5]', '$')) + IS NULL AS end_literal_isnull; +SELECT JSON_INSERT(JSON_EXTRACT(@arr30, '$'), @pastend, + JSON_EXTRACT('[5]', '$')) IS NULL AS end_cut_isnull; + +--echo # +--echo # One level shallower, the same statements answer, and that +--echo # is what says the two above are refused for their depth and +--echo # not for their shape. +--echo # +SET @deep20= CONCAT(REPEAT('[', 20), '{"k": 1}', REPEAT(']', 20)); +SET @newkey20= CONCAT('$', REPEAT('[0]', 20), '.n'); +SELECT JSON_INSERT(JSON_EXTRACT(@deep20, '$'), @newkey20, + JSON_EXTRACT('[5]', '$')) IS NULL AS shallow_isnull; + +--echo # +--echo # And what it actually composed, which is where a document +--echo # that came out a character longer or shorter than a released +--echo # server's would show itself. Asking only whether an answer +--echo # came back would not. +--echo # +SELECT JSON_INSERT(JSON_EXTRACT('[[{"k": 1}]]', '$'), '$[0][0].n', + JSON_EXTRACT('[5]', '$')) AS composed; +SELECT JSON_INSERT(JSON_EXTRACT('[[1, 2]]', '$'), '$[0][2]', + JSON_EXTRACT('[5]', '$')) AS composed_past_end; +SELECT JSON_SET(JSON_EXTRACT('{"a": {"b": 1}}', '$'), '$.a.c', + JSON_EXTRACT('{"z": 9}', '$')) AS composed_set; +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', + JSON_EXTRACT('[3, 4]', '$')) AS composed_append; +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', + JSON_EXTRACT('[3, 4]', '$')) AS composed_ainsert; +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('{"a": 1}', '$'), '$', + JSON_EXTRACT('[3, 4]', '$')) AS composed_autowrap; +SELECT JSON_EXTRACT('{"a": [1, 2], "b": {"c": 3}}', '$.a', '$.b') + AS composed_two_paths; +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), JSON_EXTRACT('2', '$')) + AS composed_merge_wrap; + +--echo # +--echo # 4. The bracket a reading of more than one path adds +--echo # +--echo # Two matches are returned wrapped in an array, and that +--echo # wrapper is a level nobody asked the document about. +--echo # +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SELECT JSON_VALID(@d31) AS legal_on_its_own; +SELECT JSON_EXTRACT(@d31, '$') IS NULL AS one_path_isnull; +SELECT JSON_EXTRACT(@d31, '$', '$') IS NULL AS two_paths_isnull; + +SET @d30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_EXTRACT(@d30, '$', '$') IS NULL AS two_paths_fit_isnull; + +--echo # +--echo # 5. The bracket a merge adds +--echo # +--echo # Merging a document with something it cannot be merged into +--echo # puts both in a new array, which is a level as well. +--echo # +SET @o31= CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SELECT JSON_VALID(@o31) AS legal_on_its_own; +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('1', '$')) + IS NULL AS scalar_wrap_isnull; +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('{"a": 1}', '$')) + IS NULL AS shared_key_isnull; + +SET @o30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SELECT JSON_MERGE(JSON_EXTRACT(@o30, '$'), JSON_EXTRACT('1', '$')) + IS NULL AS wrap_fits_isnull; + +--echo # +--echo # 6. The value an autowrap keeps +--echo # +--echo # Appending to something that is not an array wraps what is +--echo # already there, so the KEPT value goes a level down too - +--echo # not only the one being appended. +--echo # +SET @p30= CONCAT('$', REPEAT('.a', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o31, '$'), @p30, 1) + IS NULL AS append_wrap_isnull; +SELECT JSON_INSERT(JSON_EXTRACT(@o31, '$'), CONCAT(@p30, '[1]'), 1) + IS NULL AS insert_wrap_isnull; + +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o30, '$'), @p29, 1) + IS NULL AS append_wrap_fits_isnull; + +--echo # +--echo # 7. A document that fits, and the diagnostic it must not get +--echo # +--echo # The level a value is appended at is read off the scanner +--echo # after it has already been counted, so counting it again +--echo # refuses a document that fits - or lets it through with a +--echo # note saying it did not fit, which is worse, the note being +--echo # the only thing an application would go by. +--echo # +SET @nest30= CONCAT(REPEAT('[', 30), REPEAT(']', 30)); +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_EXTRACT(@nest30, '$')) + IS NULL AS literal_isnull; +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1]', '$'), '$', + JSON_EXTRACT(@nest30, '$')) IS NULL AS cut_isnull; +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1]', '$'), '$[0]', + JSON_EXTRACT(@nest30, '$')) IS NULL AS insert_isnull; diff --git a/mysql-test/main/func_json_unfinished.result b/mysql-test/main/func_json_unfinished.result new file mode 100644 index 0000000000000..e7774234a2dcf --- /dev/null +++ b/mysql-test/main/func_json_unfinished.result @@ -0,0 +1,76 @@ +# +# A path counted from the end of an array in a document that does +# not end. +# +# A step written with 'last' is not known until the array it is +# counted in has been counted, so a document carrying one is read +# through twice: once to find out how many elements each array +# holds, and once to answer. The counting pass can run off the +# end of a document that stops in the middle, and what it finds +# there has to be carried out of the loop rather than left to the +# answering pass, which would otherwise answer about an array it +# never finished counting. +# +SET NAMES utf8mb4; +# +# 1. The counting pass reaching the end of the text +# +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'one', '$.a[last]') AS one_path; +one_path +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'all', '$.a[last-1]', '$.a[0]') +AS all_paths; +all_paths +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'one', '$[1][last]') AS nested; +nested +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'all', '$[0][last]', '$[1][last]') +AS nested_all; +nested_all +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +# +# 2. The same documents finished, which is what the answers above +# are read against +# +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'one', '$.a[last]') AS one_path; +one_path +1 +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'all', '$.a[last-1]', '$.a[0]') +AS all_paths; +all_paths +1 +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'one', '$[1][last]') AS nested; +nested +1 +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'all', '$[0][last]', '$[1][last]') +AS nested_all; +nested_all +1 +# +# 3. The reading functions over the same shapes +# +SELECT JSON_EXTRACT('{"a":[1,2,3', '$.a[last]') AS unfinished; +unfinished +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":[1,2,3]}', '$.a[last]') AS finished; +finished +3 +SELECT JSON_EXTRACT('[[1,2],[3', '$[1][last]') AS unfinished; +unfinished +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('[[1,2],[3]]', '$[1][last]') AS finished; +finished +3 diff --git a/mysql-test/main/func_json_unfinished.test b/mysql-test/main/func_json_unfinished.test new file mode 100644 index 0000000000000..94f2d3c1505be --- /dev/null +++ b/mysql-test/main/func_json_unfinished.test @@ -0,0 +1,44 @@ +--echo # +--echo # A path counted from the end of an array in a document that does +--echo # not end. +--echo # +--echo # A step written with 'last' is not known until the array it is +--echo # counted in has been counted, so a document carrying one is read +--echo # through twice: once to find out how many elements each array +--echo # holds, and once to answer. The counting pass can run off the +--echo # end of a document that stops in the middle, and what it finds +--echo # there has to be carried out of the loop rather than left to the +--echo # answering pass, which would otherwise answer about an array it +--echo # never finished counting. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. The counting pass reaching the end of the text +--echo # +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'one', '$.a[last]') AS one_path; +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'all', '$.a[last-1]', '$.a[0]') + AS all_paths; +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'one', '$[1][last]') AS nested; +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'all', '$[0][last]', '$[1][last]') + AS nested_all; + +--echo # +--echo # 2. The same documents finished, which is what the answers above +--echo # are read against +--echo # +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'one', '$.a[last]') AS one_path; +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'all', '$.a[last-1]', '$.a[0]') + AS all_paths; +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'one', '$[1][last]') AS nested; +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'all', '$[0][last]', '$[1][last]') + AS nested_all; + +--echo # +--echo # 3. The reading functions over the same shapes +--echo # +SELECT JSON_EXTRACT('{"a":[1,2,3', '$.a[last]') AS unfinished; +SELECT JSON_EXTRACT('{"a":[1,2,3]}', '$.a[last]') AS finished; +SELECT JSON_EXTRACT('[[1,2],[3', '$[1][last]') AS unfinished; +SELECT JSON_EXTRACT('[[1,2],[3]]', '$[1][last]') AS finished; diff --git a/mysql-test/main/func_json_walk_oom.result b/mysql-test/main/func_json_walk_oom.result new file mode 100644 index 0000000000000..51fab2d1e41e8 --- /dev/null +++ b/mysql-test/main/func_json_walk_oom.result @@ -0,0 +1,75 @@ +# +# The room running out at every point of writing a value out. +# +# A value that is a document but is not written the loose way is +# written out again as it joins a document that is, and that +# writing puts down five kinds of thing: the bracket the value +# opens with, a comma before an element, a key with its colon, +# the bracket a value inside it opens with, and the bracket each +# of them closes with. Every one of them can be the append that +# finds no room, and each has its own way of giving up. +# +# Reaching them needs the room to run out at a chosen place, and +# the place is not something a single statement can aim at: what +# is left for the value is what the document did not spend, and +# where inside the value that runs out depends on how much +# punctuation stands before the point of interest. So the value +# is swept past the point instead. Each element of the run +# before the tail spends a fixed number of characters more in the +# loose form, and the key in front of the run spends one per +# letter, so the two together move the stopping point one +# character at a time across a tail that carries all of them. +# +# A function that edits a document takes room for its answer up +# front and one that reads a document does not, so the two are +# swept apart: the first runs out of room a fixed distance into +# the value, the second wherever the buffer last happened to +# grow to. +# +# What is pinned is not which statement fails - that is a fact +# about allocation sizes and is allowed to move. It is that +# every answer that does come back parses, and that some of them +# were refused, so the sweep is known to have reached the +# failures at all. A document with a piece missing out of the +# middle is not an answer, and no caller checks for one. +# +SET NAMES utf8mb4; +# +# Nothing came back half written, and the sweep did reach the +# failures. +# +SELECT @bad AS answers_that_do_not_parse, @refused > 0 AS some_were_refused; +answers_that_do_not_parse some_were_refused +0 1 +# +# The same values with the room there, so what the sweeps above +# compare against is an answer that arrives. +# +SET @a = CONCAT('{"a":1,', REPEAT('"p":1,', 110), +'"kkkk":{"b":[]},"z":[[]],"y":2}'); +SET @d = CONCAT('{"a":[', REPEAT('1,', 40), +'{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('z',1), '$.d', +JSON_QUERY(@a,'$')), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@d, '$**.k'), '')) AS parses; +parses +1 +SELECT JSON_EXTRACT(@d, '$.b') AS answer; +answer +{"c": [6, [7]]} +# +# A value that stops in the middle of a key, which is the one way +# the walk gives up that is not about room. +# +SELECT JSON_EXTRACT('{"a":{"bb', '$.a') AS unfinished_key; +unfinished_key +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":[1,{"c', '$.a') AS unfinished_key; +unfinished_key +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' diff --git a/mysql-test/main/func_json_walk_oom.test b/mysql-test/main/func_json_walk_oom.test new file mode 100644 index 0000000000000..f37018a085b8d --- /dev/null +++ b/mysql-test/main/func_json_walk_oom.test @@ -0,0 +1,188 @@ +--source include/have_debug.inc + +--echo # +--echo # The room running out at every point of writing a value out. +--echo # +--echo # A value that is a document but is not written the loose way is +--echo # written out again as it joins a document that is, and that +--echo # writing puts down five kinds of thing: the bracket the value +--echo # opens with, a comma before an element, a key with its colon, +--echo # the bracket a value inside it opens with, and the bracket each +--echo # of them closes with. Every one of them can be the append that +--echo # finds no room, and each has its own way of giving up. +--echo # +--echo # Reaching them needs the room to run out at a chosen place, and +--echo # the place is not something a single statement can aim at: what +--echo # is left for the value is what the document did not spend, and +--echo # where inside the value that runs out depends on how much +--echo # punctuation stands before the point of interest. So the value +--echo # is swept past the point instead. Each element of the run +--echo # before the tail spends a fixed number of characters more in the +--echo # loose form, and the key in front of the run spends one per +--echo # letter, so the two together move the stopping point one +--echo # character at a time across a tail that carries all of them. +--echo # +--echo # A function that edits a document takes room for its answer up +--echo # front and one that reads a document does not, so the two are +--echo # swept apart: the first runs out of room a fixed distance into +--echo # the value, the second wherever the buffer last happened to +--echo # grow to. +--echo # +--echo # What is pinned is not which statement fails - that is a fact +--echo # about allocation sizes and is allowed to move. It is that +--echo # every answer that does come back parses, and that some of them +--echo # were refused, so the sweep is known to have reached the +--echo # failures at all. A document with a piece missing out of the +--echo # middle is not an answer, and no caller checks for one. +--echo # + +SET NAMES utf8mb4; + +--disable_query_log +--disable_result_log +--disable_warnings + +let $m= 0; +while ($m < 9) +{ + let $n= 96; + while ($n < 122) + { + eval SET @a = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"kkkk":{"b":[]},"z":[[]],"y":2}'); + eval SET @b = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"c":{},"d":[],"e":{"f":{}},"g":3}'); + eval SET @c = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"h":[1,2,[3,[4]]],"i":4}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + --error 0,5 + SELECT JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@a,'$')); + --error 0,5 + SELECT JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@b,'$')); + --error 0,5 + SELECT JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@c,'$')); + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +let $m= 0; +while ($m < 4) +{ + let $n= 0; + while ($n < 200) + { + eval SET @d = CONCAT('{"a', REPEAT('x', $m), '":[', REPEAT('1,', $n), + '{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + --error 0,5 + SELECT JSON_EXTRACT(@d, '$**.k'); + --error 0,5 + SELECT JSON_EXTRACT(@d, '$.b'); + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +# The same shapes again, with each answer read back to see that it +# parses. It is a pass of its own because reading the answer back is +# itself work in the buffer, and that moves the point the room runs out +# at - so a sweep that checks is not the same sweep as one that does +# not, and both are wanted. +SET @bad = 0; +SET @refused = 0; + +let $m= 0; +while ($m < 4) +{ + let $n= 96; + while ($n < 122) + { + eval SET @a = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"kkkk":{"b":[]},"z":[[]],"y":2}'); + eval SET @c = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"h":[1,2,[3,[4]]],"i":4}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + + SET @r = NULL; + --error 0,5 + SET @r = JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@a,'$')); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET @r = NULL; + --error 0,5 + SET @r = JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@c,'$')); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +let $m= 0; +while ($m < 2) +{ + let $n= 0; + while ($n < 60) + { + eval SET @d = CONCAT('{"a', REPEAT('x', $m), '":[', REPEAT('1,', $n), + '{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + + SET @r = NULL; + --error 0,5 + SET @r = JSON_EXTRACT(@d, '$**.k'); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET @r = NULL; + --error 0,5 + SET @r = JSON_EXTRACT(@d, '$.b'); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +--enable_warnings +--enable_result_log +--enable_query_log + +--echo # +--echo # Nothing came back half written, and the sweep did reach the +--echo # failures. +--echo # +SELECT @bad AS answers_that_do_not_parse, @refused > 0 AS some_were_refused; + +--echo # +--echo # The same values with the room there, so what the sweeps above +--echo # compare against is an answer that arrives. +--echo # +SET @a = CONCAT('{"a":1,', REPEAT('"p":1,', 110), + '"kkkk":{"b":[]},"z":[[]],"y":2}'); +SET @d = CONCAT('{"a":[', REPEAT('1,', 40), + '{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('z',1), '$.d', + JSON_QUERY(@a,'$')), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@d, '$**.k'), '')) AS parses; +SELECT JSON_EXTRACT(@d, '$.b') AS answer; + +--echo # +--echo # A value that stops in the middle of a key, which is the one way +--echo # the walk gives up that is not about room. +--echo # +SELECT JSON_EXTRACT('{"a":{"bb', '$.a') AS unfinished_key; +SELECT JSON_EXTRACT('{"a":[1,{"c', '$.a') AS unfinished_key; diff --git a/mysql-test/main/func_json_wide_compose.result b/mysql-test/main/func_json_wide_compose.result new file mode 100644 index 0000000000000..d5d73b3abbb5d --- /dev/null +++ b/mysql-test/main/func_json_wide_compose.result @@ -0,0 +1,154 @@ +# +# 1. Merging, which writes its own punctuation between a key and +# the value copied in after it +# +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) +AS ucs2_patch_agrees; +ucs2_patch_agrees +1 +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf16), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING utf16), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) +AS utf16_patch_agrees; +utf16_patch_agrees +1 +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf32), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING utf32), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) +AS utf32_patch_agrees; +utf32_patch_agrees +1 +# +# A key both documents hold, where the value is written out +# rather than copied and there is no space to inherit. +# +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"a":9}', '$')) +AS ucs2_shared_key_agrees; +ucs2_shared_key_agrees +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"a":9}', '$')) +AS ucs2_preserve_agrees; +ucs2_preserve_agrees +1 +# +# Two arrays, and an array with something that is not one. +# These take the arm of the merging that copies whole spans of +# both documents rather than walking their keys, so they are +# where a copy put through a converting append shows up. +# +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('[3, 4]' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), +JSON_EXTRACT('[3, 4]', '$')) +AS ucs2_arrays_agree; +ucs2_arrays_agree +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('[1, 2]' USING utf32), '$'), +JSON_EXTRACT(CONVERT('[3, 4]' USING utf32), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), +JSON_EXTRACT('[3, 4]', '$')) +AS utf32_arrays_agree; +utf32_arrays_agree +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('{"a": 1}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('2' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a": 1}', '$'), +JSON_EXTRACT('2', '$')) +AS ucs2_wrap_agrees; +ucs2_wrap_agrees +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"a": 3}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), +JSON_EXTRACT('{"a": 3}', '$')) +AS ucs2_array_object_agree; +ucs2_array_object_agree +1 +# +# 2. The other six, each composing the same document a different +# way, so that a one-byte assumption anywhere shows here +# +SELECT CONVERT(JSON_INSERT(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), +'$'), '$.c', 3) USING utf8mb4) += JSON_INSERT(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.c', 3) +AS ucs2_insert_agrees; +ucs2_insert_agrees +1 +SELECT CONVERT(JSON_SET(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), +'$'), '$.a', 9) USING utf8mb4) += JSON_SET(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) +AS ucs2_set_agrees; +ucs2_set_agrees +1 +SELECT CONVERT(JSON_REPLACE(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), +'$'), '$.a', 9) USING utf8mb4) += JSON_REPLACE(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) +AS ucs2_replace_agrees; +ucs2_replace_agrees +1 +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), +'$'), '$[0]') USING utf8mb4) += JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') +AS ucs2_remove_agrees; +ucs2_remove_agrees +1 +SELECT CONVERT(JSON_ARRAY_APPEND(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), +'$'), '$', 3) USING utf8mb4) += JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', 3) +AS ucs2_append_agrees; +ucs2_append_agrees +1 +SELECT CONVERT(JSON_ARRAY_INSERT(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), +'$'), '$[0]', 9) USING utf8mb4) += JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', 9) +AS ucs2_array_insert_agrees; +ucs2_array_insert_agrees +1 +# +# 3. What the wide answers actually say, written out, so that a +# change to BOTH sides at once cannot pass unseen +# +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) +USING utf8mb4) AS ucs2_patch_written; +ucs2_patch_written +{"a": 1, "b": 2, "c": 3} +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), +'$'), '$[0]') USING utf8mb4) +AS ucs2_remove_written; +ucs2_remove_written +[2, 3] +SELECT JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) AS utf8_patch_written; +utf8_patch_written +{"a": 1, "b": 2, "c": 3} diff --git a/mysql-test/main/func_json_wide_compose.test b/mysql-test/main/func_json_wide_compose.test new file mode 100644 index 0000000000000..c96f6d8330c7b --- /dev/null +++ b/mysql-test/main/func_json_wide_compose.test @@ -0,0 +1,152 @@ +# +# Composing a document in a character set that writes a letter, and a +# space, in more than one byte. +# +# A set that cannot encode a document at all is refused elsewhere; these +# are the sets that CAN, and so are the ones where composing has to +# count characters rather than bytes. A space in ucs2 is two bytes, in +# utf32 four, and the first of those bytes is a zero - so anything that +# looks at one byte to decide whether it is standing on a space decides +# wrongly, and quietly, in exactly these sets. +# +# Every case is written as a comparison against the same composition in +# utf8, which is the answer these have to agree with. Recording this +# file cannot therefore write a defect down as expected: a doubled or +# missing space makes the two sides differ whatever is recorded. +# + +--enable_warnings + +--echo # +--echo # 1. Merging, which writes its own punctuation between a key and +--echo # the value copied in after it +--echo # +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) + AS ucs2_patch_agrees; + +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf16), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING utf16), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) + AS utf16_patch_agrees; + +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf32), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING utf32), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) + AS utf32_patch_agrees; + +--echo # +--echo # A key both documents hold, where the value is written out +--echo # rather than copied and there is no space to inherit. +--echo # +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"a":9}', '$')) + AS ucs2_shared_key_agrees; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"a":9}', '$')) + AS ucs2_preserve_agrees; + +--echo # +--echo # Two arrays, and an array with something that is not one. +--echo # These take the arm of the merging that copies whole spans of +--echo # both documents rather than walking their keys, so they are +--echo # where a copy put through a converting append shows up. +--echo # +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('[3, 4]' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), + JSON_EXTRACT('[3, 4]', '$')) + AS ucs2_arrays_agree; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('[1, 2]' USING utf32), '$'), + JSON_EXTRACT(CONVERT('[3, 4]' USING utf32), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), + JSON_EXTRACT('[3, 4]', '$')) + AS utf32_arrays_agree; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('{"a": 1}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('2' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a": 1}', '$'), + JSON_EXTRACT('2', '$')) + AS ucs2_wrap_agrees; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"a": 3}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), + JSON_EXTRACT('{"a": 3}', '$')) + AS ucs2_array_object_agree; + +--echo # +--echo # 2. The other six, each composing the same document a different +--echo # way, so that a one-byte assumption anywhere shows here +--echo # +SELECT CONVERT(JSON_INSERT(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), + '$'), '$.c', 3) USING utf8mb4) + = JSON_INSERT(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.c', 3) + AS ucs2_insert_agrees; + +SELECT CONVERT(JSON_SET(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), + '$'), '$.a', 9) USING utf8mb4) + = JSON_SET(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) + AS ucs2_set_agrees; + +SELECT CONVERT(JSON_REPLACE(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), + '$'), '$.a', 9) USING utf8mb4) + = JSON_REPLACE(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) + AS ucs2_replace_agrees; + +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), + '$'), '$[0]') USING utf8mb4) + = JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') + AS ucs2_remove_agrees; + +SELECT CONVERT(JSON_ARRAY_APPEND(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), + '$'), '$', 3) USING utf8mb4) + = JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', 3) + AS ucs2_append_agrees; + +SELECT CONVERT(JSON_ARRAY_INSERT(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), + '$'), '$[0]', 9) USING utf8mb4) + = JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', 9) + AS ucs2_array_insert_agrees; + +--echo # +--echo # 3. What the wide answers actually say, written out, so that a +--echo # change to BOTH sides at once cannot pass unseen +--echo # +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) + USING utf8mb4) AS ucs2_patch_written; +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), + '$'), '$[0]') USING utf8mb4) + AS ucs2_remove_written; +SELECT JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) AS utf8_patch_written; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 6dbad93abf7e8..803d9fcc224fb 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -31,6 +31,95 @@ int dbug_json_check_min_stack_requirement() extern void pause_execution(THD *thd, double timeout); + +#ifndef DBUG_OFF +/* + Counts a reading of a JSON value. + + What these functions are being taught is not to read a value they have + already read, and a value read one time fewer returns exactly what + it gave back before - so there is nothing in any answer that says the + saving happened. The count is the only place it shows, which is why + it exists at all. + + A scanner started while there is no session to charge is left + uncounted rather than refused. + + Written the way json_lib calls it, that being what its address is + handed to. +*/ +extern "C" void json_count_scan() +{ + THD *thd= current_thd; + + if (thd) + status_var_increment(thd->status_var.json_scans); +} + + +/* + Asks json_lib to say when a value is read. + + The count is taken where the reading begins rather than where it is + asked for. Every way of asking - reading a path, asking whether a + value is a document, normalizing one - ends up at json_scan_start(), + so a count taken there is a count of readings. A count taken at the + asking would have to name every way of asking, and one such list has + already been wrong: normalizing was reading twice and being counted + none, because it does its reading from inside json_lib where a list + kept here cannot see it. + + What is counted is therefore every reading this session does, and not + only the ones these functions ask for by name. +*/ +static struct Json_scan_count_hook +{ + Json_scan_count_hook() { json_scan_start_hook= json_count_scan; } +} json_scan_count_hook; +#endif + + +#ifdef DBUG_ASSERT_EXISTS +/* + Takes a reading back off the count. + + A value is read back here to check what was claimed about it, and that + reading exists only because the assertion checking the claim does. + Counting it would put work no released server does into a number kept + to watch the work it does, and the number would then move for reasons + that have nothing to do with anybody's query. + + Where the count is not kept there is nothing to take anything off: + Json_scans belongs to a debug build, while the assertions that make + these readings are compiled by one build more than that. +*/ +static inline void json_uncount_scan() +{ +#ifndef DBUG_OFF + THD *thd= current_thd; + + if (thd) + status_var_decrement(thd->status_var.json_scans); +#endif +} + + +/* + The way to the scanner for a reading that is not the server's work: + counted like any other and then taken back off again, so that the one + place the counting happens stays the only place it happens. +*/ +static inline int json_scan_start_unbilled(json_engine_t *je, + CHARSET_INFO *i_cs, + const uchar *str, const uchar *end) +{ + int rc= json_scan_start(je, i_cs, str, end); + + json_uncount_scan(); + return rc; +} +#endif + /* Allocating memory and *also* using it (reading and writing from it) because some build instructions cause @@ -97,6 +186,50 @@ append_simple(String *s, const uchar *a, size_t a_len) static void report_bad_chr_note(const char *fname, int n_arg); +/* + Says whether a key can go between a pair of quotes without reaching + past them. A quote ends the string where it stands, a backslash begins + an escape that may swallow the quote meant to end it, and a character + below a space is not allowed inside a string unescaped at all. A key + holding none of the three is just so many characters of a name. + + Nothing is done about a key that holds one. Whether the document that + comes of splicing it can still be read is a property of the WHOLE + composition and not of the key: a key of `a":1,"b` closes one member + and opens another, and what comes out is a document that has always + been given back. Refusing it here would take that answer away. + + So the one thing this decides is whether the answer can be given + without being read back. A key it turns down is spliced exactly as it + always was, and the reading back settles it exactly as it always did. + + The key is read a character at a time rather than a byte at a time, + because a byte is not a character in every character set: in ucs2 the + letter 'b' is written 00 62, and a reading that went by bytes would + take the 00 for a control character and turn down every key there is. + A key holding something that is not a character of its own character + set at all is turned down, nothing being known about what it holds. +*/ +static bool json_key_span_is_inert(CHARSET_INFO *cs, const char *key, + size_t key_len) +{ + const uchar *p= (const uchar *) key, *end= p + key_len; + + while (p < end) + { + my_wc_t wc; + int c_len= cs->mb_wc(&wc, p, end); + + if (c_len <= 0 || wc == '"' || wc == '\\' || wc < 0x20) + return false; + + p+= c_len; + } + + return true; +} + + /* Appends the key of a path step to the document being built. @@ -116,11 +249,15 @@ static void report_bad_chr_note(const char *fname, int n_arg); Nothing else is done to the key: one holding a character that is not allowed unescaped inside a JSON string stays invalid, and is caught where it was caught before. + + 'inert' comes back saying whether the key went in as so many characters + of a name and nothing more, so that a caller which would rather not + read its answer back can tell when it has to. */ static bool __attribute__((warn_unused_result)) append_json_path_key(String *s, const json_path_step_t *step, CHARSET_INFO *path_cs, String *tmp_key, - const char *fname, int n_arg) + const char *fname, int n_arg, bool *inert) { size_t key_len= (size_t) (step->key_end - step->key); uint errors; @@ -133,10 +270,22 @@ append_json_path_key(String *s, const json_path_step_t *step, return true; /* Out of memory. */ if (!errors) + { + *inert= json_key_span_is_inert(s->charset(), tmp_key->ptr(), + tmp_key->length()); return append_simple(s, tmp_key->ptr(), tmp_key->length()); + } report_bad_chr_note(fname, n_arg); + /* + The key goes in written the way the path was, which is not the way + the document is. What those bytes come to mean once they are being + read as the document cannot be told from the key alone. + */ + *inert= false; } + else + *inert= json_key_span_is_inert(path_cs, (const char *) step->key, key_len); return append_simple(s, step->key, key_len); } @@ -381,10 +530,18 @@ static const char json_loose_colon[]= "\": "; has always done with it, and is not for this to change. What it is for is to keep that result from being taken for a document later. - Only the functions that write their own punctuation have to ask. The - ones that read their whole result back afterwards read it in the - character set it is written in, so a document they accept is one that - reads there, and asking would tell them nothing new. + Only the functions that write their own punctuation have to ask. One + that reads its whole result back afterwards reads it in the character + set it is written in, so a document it accepts is one that reads + there, and asking would tell it nothing new - which is why the seven + that always read back never asked, and why they ask now that they read + back only what their arguments left is_valid or is_nice false for. + + A set that cannot encode the punctuation can still hold a document, + but only a scalar one: writing a container in it would take the + brackets it cannot encode. So a function that has to write a + container to give its answer cannot give one there at all, and a + function given a document in such a set was given a scalar. What decides it is how narrow a character is, not whether the set is ASCII-compatible. MY_CS_NONASCII marks two unrelated things: sets that @@ -401,6 +558,70 @@ static inline bool is_json_compatible_charset(CHARSET_INFO *cs) } +/* + Whether the document argument leaves a function free to return what + it composes, instead of reading the whole answer again at the end to + find out what it is. + + Three things at once, and each of them is about the argument rather + than about anything composed from it. The value has to read as a + document, which is what the argument is asked. It has to be written + the way this writes, because what is composed keeps the argument's own + spacing between the pieces the function writes itself. And the set it + is written in has to be able to encode the punctuation that gets added + - nothing composed in a set that cannot is a document, however sound + the pieces were. + + Asked once, before anything is composed, because what is written early + is written before the later arguments have been looked at, so the + answer cannot wait for them. A later argument that turns out not to + be attested to does not come back here; it clears the splice marks + instead, and the answer goes through the reading back after all. +*/ + +static inline bool document_arg_composes_final(const Item *arg, + const String *js) +{ + return is_json_compatible_charset(js->charset()) && + arg->is_valid_json() && arg->is_nice_json(); +} + + +/* + Steps over the space standing at 'str', if one is standing there, and + returns where the next character begins. + + Composing writes some of its punctuation and copies the rest out of + the document it is working from, and where the two meet one side has + to give up the space between them. Which bytes that is depends on the + set the document is written in: a space is one byte in utf8, two in + ucs2 (0020), four in utf32 - and in the wide ones the FIRST of those + bytes is a zero. So a byte compared against ' ' answers no in exactly + the sets that can encode a document but do not write one character in + one byte, and the space is left where it stood, doubled. + + Reading one character and asking what it is answers the question the + way it was meant to be asked. +*/ +static const uchar *json_skip_space(CHARSET_INFO *cs, + const uchar *str, const uchar *end) +{ + my_wc_t wc; + int len; + + if (str >= end) + return str; + + len= cs->mb_wc(&wc, str, end); + return (len > 0 && wc == (my_wc_t) ' ') ? str + len : str; +} + + +/* Written out below, and named here because the loose form goes to it. */ +static int json_walk_nice_value(json_engine_t *je, String *to, + int &max_level); + + static int json_nice(json_engine_t *je, String *nice_js, Item_func_json_format::formats mode, int tab_size=4) { @@ -419,38 +640,62 @@ static int json_nice(json_engine_t *je, String *nice_js, if (nice_js->alloc(je->s.str_end - je->s.c_str + 32)) goto error; - /* - The buffer above is a guess, and the loose form can outgrow it: it - spends two characters where the compact form spends one, so an input - with more than sixteen separators in it needs more room than was - asked for. Everything written below therefore may have to grow the - buffer, and this is where a test arranges for that growing to fail. - Disarmed at both ways out, so that only the writing done here is - affected. - */ - DBUG_EXECUTE_IF("json_nice_append_out_of_memory", - DBUG_SET("+d,simulate_realloc_out_of_memory");); - - DBUG_ASSERT(mode != Item_func_json_format::DETAILED || (tab_size >= 0 && tab_size <= TAB_SIZE_LIMIT)); if (mode == Item_func_json_format::LOOSE) { - comma_len= 2; - colon_len= 3; - } - else if (mode == Item_func_json_format::DETAILED) - { - comma_len= 1; - colon_len= 3; - } - else - { - comma_len= 1; - colon_len= 2; + /* + The loose form is written by the walk, which is this loop with the + other two modes taken out of it and stopped where the value ends + instead of where the document does. Written out twice they would + have to go on agreeing byte for byte, and what says whether they + still do is a check that measures a value by writing it through + HERE - so the day the two drifted apart, the drift is what would + be approved. + + The walk wants the value's head read, and returns where the + value ended. A document is a value with nothing after it, so what + is left is read through to find out whether anything is. + */ + int reached= 0; + + if (json_read_value(je) || json_walk_nice_value(je, nice_js, reached)) + { + /* + Nothing was read wrong, so what went wrong was the writing: + there is no room for the answer and no answer to give. + */ + if (!je->s.error) + return 1; + } + else + { + while (json_scan_next(je) == 0) + {} + } + goto done; } + /* + The buffer above is a guess, and the indented form can outgrow it: + it spends a line and an indent where the compact form spends + nothing, so an input with more than sixteen separators in it needs + more room than was asked for. Everything written below therefore + may have to grow the buffer, and this is where a test arranges for + that growing to fail. Disarmed at both ways out, so that only the + writing done here is affected. + */ + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("+d,simulate_realloc_out_of_memory");); + + /* + What is left is the compact form and the indented one, which put the + same comma between values and differ over the space after a colon. + */ + comma_len= 1; + colon_len= (mode == Item_func_json_format::DETAILED) ? 3 : 2; + do { curr_state= je->state; @@ -557,6 +802,8 @@ static int json_nice(json_engine_t *je, String *nice_js, DBUG_EXECUTE_IF("json_nice_append_out_of_memory", DBUG_SET("-d,simulate_realloc_out_of_memory");); + +done: return je->s.error || *je->killed_ptr; error: @@ -566,6 +813,180 @@ static int json_nice(json_engine_t *je, String *nice_js, } +/* + The room to write in, made to run out for as long as one of these is + standing. + + json_nice() arms the failure of its own appends around its own + writing, so that a test can see what happens when one of them fails. + The writing has moved out of it for the functions that no longer read + their result back, and the way of failing it had to move with it - + otherwise the checking of these appends would stop being tested by the + thing that was testing it, without anybody noticing that it did. + + Disarmed however the writing is left, so that only the writing done + while this stands is affected. +*/ +class Json_room_made_to_run_out +{ +public: + Json_room_made_to_run_out() + { + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("+d,simulate_realloc_out_of_memory");); + } + ~Json_room_made_to_run_out() + { + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("-d,simulate_realloc_out_of_memory");); + } +}; + + +/* + Writes the value the scanner is sitting on to the end of 'to', in the + loose form, walking the scanner over the value as it goes. + + This is json_nice() taken apart and put back together to be usable + during a reading instead of after one. json_nice() empties its output + and reads to the end of the document, which suits a function that has + finished and wants its whole result formatted again; it is no use to one + that is partway through a document and wants THIS value written into + what it has written so far. The formatting itself is the same, and has + to be: the two write the same punctuation from the same constants, so + a value put out by either is put out identically. + + The scanner must have read the value's head already - which is where + json_get_path_next() leaves it - and is left on the value's last + token, so the caller can carry on from there exactly as if it had + walked over the value itself. + + A scalar has no punctuation of its own and so is already written the + loose way wherever it came from; it is copied across as it stands, + which is what json_nice() does with one too. + + 'max_level' comes back holding the deepest the walk went, counted the + way a plain reading of the same value counts it - which is what a + caller that has to say how deep its answer ends up would otherwise + have to take a second reading to find out. It is raised and never + lowered, so a caller starting one walk where the last left off gets + the deepest of them all. + + Every writing of a value goes through here, which is what makes this + the one place the failure of these appends has to be arranged. + + Returns non-zero on a write that failed or a value that did not end. +*/ +static int json_walk_nice_value(json_engine_t *je, String *to, + int &max_level) +{ + Json_room_made_to_run_out room; + int depth; + int first_value= 1; + + if (je->stack_p > max_level) + max_level= je->stack_p; + + if (json_value_scalar(je)) + return append_simple(to, je->value_begin, + je->value_end - je->value_begin); + + if (to->append((je->value_type == JSON_VALUE_OBJECT) ? "{" : "[", 1)) + return 1; + depth= 1; + + while (json_scan_next(je) == 0) + { + if (je->stack_p > max_level) + max_level= je->stack_p; + + /* + The step above is what a killed query is stopped by here, and it + is the same step, taken the same number of times, that the reading + back used to be stopped by - so nothing is asked of this loop that + the reading it replaces was not already asked. + + A kill arriving before the writing begins is stopped by whichever + reading got here, which is the only kill the tests could reach + until now. This arms one that arrives partway through, so that + what stops it is the step above and nothing else. + */ + DBUG_EXECUTE_IF("json_kill_while_emitting", + { current_thd->set_killed(KILL_QUERY); }); + + switch (je->state) + { + case JST_KEY: + { + const uchar *key_start= je->s.c_str; + const uchar *key_end; + + do + { + key_end= je->s.c_str; + } while (json_read_keyname_chr(je) == 0); + + if (unlikely(je->s.error)) + return 1; + + if (!first_value && to->append(json_loose_comma, 2)) + return 1; + + if (to->append('"') || + append_simple(to, key_start, key_end - key_start) || + to->append(json_loose_colon, 3)) + return 1; + } + /* The key's value comes next, so there is no break here. */ + DBUG_ASSERT(je->state == JST_VALUE); + goto handle_value; + + case JST_VALUE: + if (!first_value && to->append(json_loose_comma, 2)) + return 1; + +handle_value: + if (json_read_value(je)) + return 1; + + if (je->stack_p > max_level) + max_level= je->stack_p; + + if (json_value_scalar(je)) + { + if (append_simple(to, je->value_begin, + je->value_end - je->value_begin)) + return 1; + first_value= 0; + } + else + { + if (to->append((je->value_type == JSON_VALUE_OBJECT) ? "{" : "[", 1)) + return 1; + first_value= 1; + depth++; + } + break; + + case JST_OBJ_END: + case JST_ARRAY_END: + if (to->append((je->state == JST_OBJ_END) ? "}" : "]", 1)) + return 1; + first_value= 0; + if (--depth == 0) + return je->s.error != 0; + break; + + default: + break; + } + } + + /* The document ended in the middle of the value. */ + return 1; +} + + /* The two below are read by DBUG_ASSERT and by nothing else, so they are here wherever a DBUG_ASSERT is compiled rather than wherever a debug @@ -584,12 +1005,49 @@ static bool json_value_reads_as_document(const String *str) { /* Which is the question json_valid() is, so it is asked rather than - asked again here. + asked again here. The reading it does begins at json_scan_start() + like every other, so it is counted like every other and taken back + off afterwards - the same two steps json_scan_start_unbilled() makes, + in the order the borrowed reading leaves them in. */ - return json_valid(str->ptr(), str->length(), str->charset()) != 0; + bool valid= json_valid(str->ptr(), str->length(), str->charset()) != 0; + + json_uncount_scan(); + return valid; } +/* + A kill is not simulated in a reading this file makes for itself. + + The arming that raises one partway through a walk is aimed at the + walks a statement makes on its way to an answer. A debug build makes + one walk MORE than a release build does - the one just below, which + writes a value out again to find out whether it was already written + that way - and a kill raised in THAT walk would be a debug build + ending a statement a release build finishes. What is checked here is + the code, and a check that changes the answer is not one. + + So it is held off for as long as the reading lasts, the way the + reading itself is held off the count of readings. +*/ +class Json_kill_unsimulated +{ + bool m_armed; +public: + Json_kill_unsimulated() : m_armed(DBUG_IF("json_kill_while_emitting")) + { + if (m_armed) + DBUG_SET("-d,json_kill_while_emitting"); + } + ~Json_kill_unsimulated() + { + if (m_armed) + DBUG_SET("+d,json_kill_while_emitting"); + } +}; + + /* Writes the value out again in the loose form and asks whether that changed anything. A value already written that way comes back byte @@ -597,11 +1055,12 @@ static bool json_value_reads_as_document(const String *str) */ static bool json_value_is_nice(const String *str) { + Json_kill_unsimulated unkilled; json_engine_t je; StringBuffer nice; - json_scan_start(&je, str->charset(), (const uchar *) str->ptr(), - (const uchar *) str->ptr() + str->length()); + json_scan_start_unbilled(&je, str->charset(), (const uchar *) str->ptr(), + (const uchar *) str->ptr() + str->length()); if (json_nice(&je, &nice, Item_func_json_format::LOOSE)) return false; @@ -610,6 +1069,57 @@ static bool json_value_is_nice(const String *str) #endif +/* + A document being edited is walked once to find the place to edit, and + the pieces of it that go into the answer are copied from where that + walk left off - so the walk's pointers into it stay live across + everything that happens in between. What happens in between includes + working out a path and working out a value, and either can be any + expression a caller cares to write. + + Nothing here owns the document. It can be a table's row, a routine's + variable or another statement's user variable, and what an expression + writes is not this function's to know. That the bytes stay put is + true today because of the shapes an expression can take and not + because anything makes it so, which is exactly the kind of thing that + stops being true quietly. So a debug build holds on to a copy and + says whether it still matches. + + A release build keeps nothing and asks nothing, this being a check on + the code rather than on the data. +*/ +class Json_source_watch +{ +#ifndef DBUG_OFF + String m_held; + bool m_holding; +#endif +public: +#ifdef DBUG_OFF + void take(const String *) {} + bool unchanged(const String *) const { return true; } +#else + Json_source_watch() : m_holding(false) {} + void take(const String *js) + { + /* + A copy that cannot be made says nothing either way, and must not + turn into a complaint: the room to make it is made to run out on + purpose by the tests that check what happens when it does. + + A document that is not there says nothing either. Asking for one + is how a function finds out whether there is one, and the watch is + taken before anything else can run, so the two orders would + otherwise have to be told apart by every caller. + */ + m_holding= js && !m_held.copy(js->ptr(), js->length(), js->charset()); + } + bool unchanged(const String *js) const + { return !m_holding || !stringcmp(js, &m_held); } +#endif +}; + + /* A mark is a promise about a run of bytes, and the bytes are right here, so a debug build keeps the promise by reading them. @@ -852,10 +1362,12 @@ bool Item_func_json_equals::val_bool() String a_tmp, b_tmp; THD *thd; json_engine_t je; + Json_source_watch watch; String *a= args[0]->val_json(&a_tmp); if ((null_value= a == nullptr || args[0]->null_value)) return 1; + watch.take(a); String *b= args[1]->val_json(&b_tmp); if ((null_value= b == nullptr || args[1]->null_value)) return 1; @@ -879,6 +1391,7 @@ bool Item_func_json_equals::val_bool() JSON_DO_PAUSE_EXECUTION(thd, 0.0002); je.killed_ptr= (uint32_t *) &thd->killed; + DBUG_ASSERT(watch.unchanged(a)); if (json_normalize_engine(&je, &a_res, a->ptr(), a->length(), a->charset())) goto return_null; @@ -919,6 +1432,7 @@ bool Item_func_json_exists::fix_length_and_dec(THD *thd) bool Item_func_json_exists::val_bool() { json_engine_t je; + Json_source_watch watch; int array_counters[JSON_DEPTH_LIMIT]= {0}; THD *thd= current_thd; @@ -926,6 +1440,7 @@ bool Item_func_json_exists::val_bool() String *js= args[0]->val_json(&tmp_js); + watch.take(js); if (!path.parsed) { String *s_p= args[1]->val_str(&tmp_path); @@ -943,6 +1458,7 @@ bool Item_func_json_exists::val_bool() } null_value= 0; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -992,7 +1508,9 @@ bool Json_path_extractor::extract(String *str, Item *item_js, Item *item_jp, String *js= item_js->val_json(&tmp_js); int error= 0; int array_counters[JSON_DEPTH_LIMIT]= {0}; + Json_source_watch watch; + watch.take(js); if (!parsed) { String *s_p= item_jp->val_str(&tmp_path); @@ -1026,6 +1544,7 @@ bool Json_path_extractor::extract(String *str, Item *item_js, Item *item_jp, if (item_js->null_value || item_jp->null_value) return true; + DBUG_ASSERT(watch.unchanged(js)); Json_engine_scan je(*js); str->length(0); str->set_charset(cs); @@ -1465,13 +1984,16 @@ String *Item_func_json_extract::read_json(String *str, json_engine_t je, sav_je; json_path_t p; const uchar *value; + uint32 copy_start, copy_len; int not_first_value= 0, count_path= 0; uint n_arg; - size_t v_len; int possible_multiple_values; + /* How far down the deepest of the values written out goes. */ + int deepest_value= 0; int array_size_counter[JSON_DEPTH_LIMIT]; uint has_negative_path= 0; THD *thd; + Json_source_watch watch; m_marks.clear(); @@ -1482,6 +2004,7 @@ String *Item_func_json_extract::read_json(String *str, JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); for (n_arg=1; n_arg < arg_count; n_arg++) { json_path_with_flags *c_path= paths + n_arg - 1; @@ -1521,8 +2044,16 @@ String *Item_func_json_extract::read_json(String *str, goto error; } + DBUG_ASSERT(watch.unchanged(js)); json_get_path_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length(), &p); + /* + This walk is now the only one there is, so it is the one that has to + notice being killed. It used to be worth noticing only over the + result, that being read separately afterwards; the reading of the + document itself went uninterrupted however long the document was. + */ + je.killed_ptr= (uint32_t *) &thd->killed; while (json_get_path_next(&je, &p) == 0) { @@ -1549,46 +2080,90 @@ String *Item_func_json_extract::read_json(String *str, goto return_ok; } + if ((not_first_value && str->append(STRING_WITH_LEN(json_loose_comma))) || + (not_first_value && DBUG_IF("json_extract_comma_out_of_memory"))) + goto error; + + copy_start= str->length(); + if (json_value_scalar(&je)) - v_len= je.value_end - value; + { + /* + A scalar is punctuated with nothing, so however it is written in + the document is how it is written in the loose form as well, and + it is copied across as it stands - which is the copy json_nice() + used to make of it, made here instead. Copied and not written, + so it must not be converted on the way: it came out of the + document, and the document is in the character set the result is + being built in. + */ + if (append_simple(str, value, je.value_end - value) || + DBUG_IF("json_extract_scalar_out_of_memory")) + goto error; + } else { + /* + A container is written out rather than copied. How it looks in + the document is not how the loose form looks, and finding that + out by writing the whole result and reading it back again is the + reading being done away with here. + + The walk over the value is the walk this loop was going to make + anyway - json_skip_level() went over exactly these characters + without writing anything down. + */ if (possible_multiple_values) sav_je= je; - if (json_skip_level(&je)) - goto error; - v_len= je.s.c_str - value; + /* + How far down this value goes, counted from itself rather than + from the document it was cut out of. Reading a container has + already put it on the stack, so the level here is the value's + own outermost one, and what the walk reaches above that is what + it holds. The bracket written round the whole answer sits one + further out again - see where it is closed. + */ + { + int base= je.stack_p, reached= je.stack_p; + + if (json_walk_nice_value(&je, str, reached)) + goto error; + if (reached - base + 1 > deepest_value) + deepest_value= reached - base + 1; + } if (possible_multiple_values) je= sav_je; } - if ((not_first_value && str->append(", ", 2))) - goto error; - while(count_path) + /* + A value that matched more than one of the paths asked for is + written once and then repeated from where it was written, rather + than being formatted again or held anywhere of its own. The room is + taken first, so that the read below is from a buffer that has + already finished moving. + */ + copy_len= str->length() - copy_start; + while (--count_path) { - /* - The value is copied, not written: it came out of the document - and is already in the character set the result is being built - in, so it goes in as it stands. The separators around it are - written, and are converted on the way, which is why they are - appended a different way from this. - */ - if (append_simple(str, value, v_len)) + if (str->append(STRING_WITH_LEN(json_loose_comma)) || + str->reserve(copy_len)) goto error; - count_path--; - if (count_path) - { - if (str->append(", ", 2)) - goto error; - } + str->q_append(str->ptr() + copy_start, copy_len); } not_first_value= 1; if (!possible_multiple_values) { - /* Loop to the end of the JSON just to make sure it's valid. */ - while (json_scan_next(&je) == 0) {} + /* + The rest of the document is parsed only to check that it is one. + Not done at all where the item has already attested that the + value is_valid. + */ + if (!args[0]->is_valid_json()) + { + while (json_scan_next(&je) == 0) {} + } break; } } @@ -1605,18 +2180,76 @@ String *Item_func_json_extract::read_json(String *str, if (possible_multiple_values && str->append(']')) goto error; /* Out of memory. */ - js= str; - json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), - (const uchar *) js->ptr() + js->length()); - je.killed_ptr= (uint32_t *) &thd->killed; + /* + In a character set that cannot encode the punctuation this function + writes - see is_json_compatible_charset() - what has been composed is + not a document and no reasoning about it can make it one, the + brackets holding the values having come out as national letters. + Such a set can still hold a document, so the walk above succeeded + and there is nothing else to notice it by. Reading the result back + is what has always noticed it, and where it cannot encode that + reading stays exactly where it was. + */ + /* + And where the bracket written round the values takes the answer + deeper than a document is allowed to go. That bracket is a level + this function adds: every value in it was measured against the + document it came out of, where the bracket did not exist. A + released server found this out by reading the answer back and being + refused, so being refused by the same reading is what it is owed - + working it out here and complaining directly would put the + complaint somewhere else in the text. + */ + if (!is_json_compatible_charset(str->charset()) || + (possible_multiple_values && deepest_value + 1 >= JSON_DEPTH_LIMIT)) + { + js= str; + json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), + (const uchar *) js->ptr() + js->length()); + je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, &tmp_js, Item_func_json_format::LOOSE)) - goto error; + if (json_nice(&je, &tmp_js, Item_func_json_format::LOOSE)) + goto error; + + /* + Both marks come from the reading back, for the reason given where + Item_func_json_insert::val_str() marks the same answer. + */ + m_marks.set(&tmp_js, true, true); + return &tmp_js; + } - /* The reading back just above is what sets both marks. */ - m_marks.set(&tmp_js, true, true); + /* + Nothing is read back here. Every value in the result was written + out in the loose form while the document was being walked, and the + brackets that hold them are written the same way, so the result is + already written as reading it back would have written it. And it is + a document because each part of it was read as one on the way in. + + Unlike the six that EDIT a document, this asks nothing of the + argument: nothing of it is kept. What comes back is written here + out of values this function itself walked to and read, so the only + thing that can be wrong with it is what holds those values + together - which is why the one condition above is whether the + brackets can be written at all, and why there is no trust predicate + here to ask. m_marks.set() reads it back in a debug build all the + same. + + This is also why the result is returned in the caller's own + buffer now. It used to be returned in tmp_js, which is the + scratch the argument may have been read into - so the answer was + being written over the document it was made from, and only got away + with it because the document had been finished with. + */ + m_marks.set(str, true, true); + return str; return_ok: + /* + A caller that passed no buffer wanted the value picked out, not + written out, and reads it through out_val. What comes back only has + to say that something was found. + */ return &tmp_js; error: @@ -1909,6 +2542,7 @@ bool Item_func_json_contains::val_bool() json_engine_t je, ve; int result; THD *thd; + Json_source_watch watch; if ((null_value= args[0]->null_value)) return 0; @@ -1916,6 +2550,7 @@ bool Item_func_json_contains::val_bool() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); if (!a2_parsed) { val= args[1]->val_json(&tmp_val); @@ -1928,6 +2563,7 @@ bool Item_func_json_contains::val_bool() return 0; } + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -1954,6 +2590,7 @@ bool Item_func_json_contains::val_bool() goto return_null; path.cur_step= path.p.steps; + DBUG_ASSERT(watch.unchanged(js)); if (json_find_path(&je, &path.p, &path.cur_step, array_counters)) { if (je.s.error) @@ -2150,7 +2787,10 @@ bool Item_func_json_contains_path::val_bool() int UNINIT_VAR(n_found); int array_sizes[JSON_DEPTH_LIMIT]; uint has_negative_path= 0; + /* Asked once: the walk below reaches it on every step. */ + bool js_attested; THD *thd; + Json_source_watch watch; if ((null_value= args[0]->null_value)) return 0; @@ -2158,6 +2798,7 @@ bool Item_func_json_contains_path::val_bool() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); if (parse_one_or_all(this, args[1], &ooa_parsed, ooa_constant, &mode_one)) goto null_return;; @@ -2184,6 +2825,7 @@ bool Item_func_json_contains_path::val_bool() goto null_return; } + DBUG_ASSERT(watch.unchanged(js)); json_get_path_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length(), &p); je.killed_ptr= (uint32_t *) &thd->killed; @@ -2195,6 +2837,7 @@ bool Item_func_json_contains_path::val_bool() } result= 0; + js_attested= args[0]->is_valid_json(); while (json_get_path_next(&je, &p) == 0) { int n_path= arg_count - 2; @@ -2226,6 +2869,15 @@ bool Item_func_json_contains_path::val_bool() p_found[n_path-1]= TRUE; } } + + /* + Nothing left that could unsettle the answer. The walk goes on past + a settled one only so that a fault later in the document can take + it back, and a value the item has attested is_valid has no such + fault to be found. + */ + if (result && js_attested) + break; } if (likely(je.s.error == 0)) @@ -2286,6 +2938,31 @@ struct Json_splice_marks { bool is_valid; /* everything that went in read as a document */ bool is_nice; /* everything that went in was written the loose way */ + /* + Set when the two above were cleared because a value goes past the + depth limit only once it is in place. Kept apart from the other + reasons, so that a caller which still reads its whole answer back + can check the depth it reckoned at the splice against what that + reading found: a reading that reached the end is an answer inside + the limit, and nothing should have said otherwise. + + Only that check ever reads it, so it is only there where it can be + read. That is not the same set of builds as the debug one: an + assertion survives into a build with DBUG_OFF set where it was asked + to print rather than to stop, and the expression it is given is + compiled there. + */ +#ifdef DBUG_ASSERT_EXISTS + bool is_deep; +#endif + + Json_splice_marks() + : is_valid(true), is_nice(true) + { +#ifdef DBUG_ASSERT_EXISTS + is_deep= false; +#endif + } }; @@ -2327,16 +3004,33 @@ struct Json_splice_marks 'marks' is where the parse goes, the bytes having gone in whatever it found. See Json_splice_marks. + + 'is_valid' and 'is_nice' are what the item handing the value over + says about it. is_valid true is the one case where the parse can be + left undone; is_nice false is what the writing out further down is + for. 'need_nice' is the caller saying that what it composes is what + it returns, so a value that is not is_nice has to be made so. */ static int append_json_typed_value(String *str, const String *sv, uint depth, const char *fname, int n_param, - Json_splice_marks *marks) + Json_splice_marks *marks, + bool caller_reads_back, bool is_valid, + bool is_nice, bool need_nice) { StringBuffer cnv; const char *ptr= sv->ptr(); size_t length= sv->length(); json_engine_t je; int max_level= 0; + bool reformat; + uint32 sav_len; + /* + Whether the bytes at 'ptr' are written in the character set the + result is being built in. What an item answers about a value is + about the characters it holds, so this is what says whether the + answer is about the bytes that are going in. + */ + bool in_result_charset= my_charset_same(sv->charset(), str->charset()); /* Bytes that carry no character set are left where they are, in both @@ -2349,7 +3043,7 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, appended in the character set of the result, so bytes that do not encode a JSON value there are still refused. */ - if (!my_charset_same(sv->charset(), str->charset()) && + if (!in_result_charset && sv->charset() != &my_charset_bin && str->charset() != &my_charset_bin) { uint errors; @@ -2365,14 +3059,60 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, changed value in would be worse than leaving the bytes as they arrived, which is what was always done with them. The parse below reads them as the result will be read and says so. + + One that lost nothing kept the characters it was given, and being + a document is a property of characters, so a value that arrived as + one is still one after it - and still formatted the way it was, the + loose form's spacing being characters like any other. So what was + answered about it before the conversion is answered about it + after, and the shortcut below can have it. */ if (!errors) { ptr= cnv.ptr(); length= cnv.length(); + in_result_charset= true; } } + /* + A value somebody has already attested to is copied in without being + read, which is the whole of what attesting buys. Three things + have to hold besides. + + It must be written the way the result is going to be written, or the + caller must not care. A value copied in as it stands brings its own + spacing with it, so a result that has to come out in the loose form + cannot take one that is not in it without reading it - which is what + the writing out further down is for. + + It must be going in in the character set the result is read in - + either because it arrived in it, or because it was converted into it + just above without losing anything. A lossy conversion, and equally + the untouched copy made when there is no conversion to be had, leave + bytes that nothing has attested to and fall through to the parse. + + And it must not make the result too deep to read back. The value's + own nesting is what the parse below measures, and the shortcut is not + doing that - but it does not have to, because a value can only nest + as deeply as it is long: every level takes a character to open and + one to close, so a value of n characters reaches at most n/2 levels, + and there are at most length/mbminlen characters in it. Where that + much nesting still leaves the result inside the limit, the parse + could not have found a breach to report and skipping it takes nothing + away. The bound is exact for the values that matter here - a run of + nothing but brackets is all opening and closing - and gives up on + long ones, which is the safe direction: giving up costs a reading and + nothing else. + */ + if (is_valid && (is_nice || !need_nice) && in_result_charset && + depth + length / (2 * str->charset()->mbminlen) < JSON_DEPTH_LIMIT) + { + if (marks && !is_nice) + marks->is_nice= false; + return append_simple(str, ptr, length); + } + /* The deepest the value goes is read off the scanner as it runs rather than charged to it in advance. Starting the scan with stack_p @@ -2382,16 +3122,126 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, pointer past it leaves unwritten slots underneath and the scan never finishes. */ + /* + A value that is not already written the loose way is written out + again as it is read, where the caller has said that what it composes + is what it returns. The reading has to happen either way, and a + reading that writes as it goes costs nothing more than one that does + not - so the value arrives in the form the result needs instead of + the caller reading its whole answer afterwards to put it in that + form. + + Where the caller does NOT say that, the value is left exactly as it + arrived. Two quite different callers say nothing here. One builds + an array or an object and returns what it built, so a value put + in as it stands is what a released server returns too and writing + it out afresh would change an answer. The other reads its whole + result back before answering, and that reading is where the spacing + gets settled - and it complains, when it has to, about positions in + the text it read, so a value that arrived any longer or shorter than + it used to would move them. + + A character set that cannot encode the punctuation cannot be written + into at all, and is left alone for the same reason it is everywhere + else here. + */ + reformat= need_nice && !is_nice && + is_json_compatible_charset(str->charset()); + sav_len= str->length(); + json_scan_start(&je, str->charset(), (const uchar *) ptr, (const uchar *) ptr + length); - while (json_scan_next(&je) == 0) + /* + Every reading here has to let go of a killed query, and this one is + no different for being over a value rather than over a document. + json_scan_start() leaves the scanner deaf to a kill - it points at a + word that is always zero - so a reading that is meant to hear one + says so on the line after, as every other reading in this file does. + + It matters more here than the size of a value suggests. A released + server does no reading at all at this point: it copies the bytes in + and goes on, so a kill arriving while a value is going in is heard + at the very next step. The reading is ours, and an unheard kill + would be ours too - a stretch of work, as long as the value, that + nothing could interrupt and that nobody had before. + */ + je.killed_ptr= (uint32_t *) ¤t_thd->killed; + + if (reformat) + { + if (json_read_value(&je) || json_walk_nice_value(&je, str, max_level)) + { + /* + Nothing was read wrong, so what went wrong was the writing: + there is no room for the answer and no answer to give. + */ + if (!je.s.error) + return 1; + } + else + { + /* + The walk stops where the value ends, and a value is only a + document when nothing follows it. + */ + while (json_scan_next(&je) == 0) + {} + } + + /* + Written out again only if it came out whole and fits where it is + going. Either way the caller is owed what a released server + returns, so anything else is taken back off and the bytes go in + as they arrived - which is also what leaves the complaint below + about the same text it has always been about. + */ + if (je.s.error || max_level + (int) depth >= JSON_DEPTH_LIMIT) + { + str->length(sav_len); + reformat= false; + } + } + else { - if (je.stack_p > max_level) - max_level= je.stack_p; + while (json_scan_next(&je) == 0) + { + if (je.stack_p > max_level) + max_level= je.stack_p; + } } + /* + Spliced as it stands, so the document being built is written the + loose way only if this value was. Nothing is measured to find that + out - the item was asked, and one that says nothing is taken at its + word. + */ + if (!reformat && marks && !is_nice) + marks->is_nice= false; + + /* + Too deep only once it is in place. There is nothing wrong with the + value itself, so a caller that reads its whole result back says it + about the answer rather than about this one value - which is what it + has always said, and where it has always said it. All that is owed + here is to stop such an answer being taken on trust. + + A caller that does not read back has nobody else to hear it from, so + for that one the scanner is put into error and the note goes out as + it always has. + */ if (je.s.error == 0 && max_level + (int) depth >= JSON_DEPTH_LIMIT) - json_error(&je.s, JE_DEPTH); + { + if (marks) + { + marks->is_valid= marks->is_nice= false; +#ifdef DBUG_ASSERT_EXISTS + marks->is_deep= true; +#endif + } + if (!caller_reads_back) + json_error(&je.s, JE_DEPTH); + } /* Said here rather than left to the caller: the callers report through @@ -2413,7 +3263,7 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, Sql_condition::WARN_LEVEL_NOTE); } - return append_simple(str, ptr, length); + return reformat ? 0 : append_simple(str, ptr, length); } @@ -2471,8 +3321,35 @@ __attribute__((warn_unused_result)) */ static int append_json_value(String *str, Item *item, String *tmp_val, uint depth, const char *fname, int n_param, - Json_splice_marks *marks) + Json_splice_marks *marks, + bool caller_reads_back, bool need_nice) { + /* + Too deep is settled before anything is looked at, because a value + put where the limit has already been reached is past it whatever it + turns out to be: the arms below add whatever nesting the value has + of its own and none of them takes any away. Only the arm that + READS the value can say how much further down it reaches, so this + is the whole of the question for the three that read nothing - a + bare word, a null, and a value written out as a string. + + Saying so is all that is owed here. A value only arrives this deep + through a function that EDITS a document, by putting a new array + where a value already sat; every one of those returns what it + composed only while these marks stand, so taking them away puts it + back on the reading that a released server always did, and that + reading complains about the answer in the words it has always used. + The functions that BUILD a document put their values one level down + and cannot get here. + */ + if (marks && depth >= JSON_DEPTH_LIMIT) + { + marks->is_valid= marks->is_nice= false; +#ifdef DBUG_ASSERT_EXISTS + marks->is_deep= true; +#endif + } + if (item->type_handler()->is_bool_type()) { longlong v_int= item->val_int(); @@ -2502,17 +3379,16 @@ static int append_json_value(String *str, Item *item, String *tmp_val, if (item->null_value) goto append_null; if (is_json_type(item)) - { - /* - Spliced as it stands, so the document being built is written the - loose way only if this value was. Nothing is measured here to - find that out - the item is asked, and one that says nothing is - taken at its word. - */ - if (marks && !item->is_nice_json()) - marks->is_nice= false; - return append_json_typed_value(str, sv, depth, fname, n_param, marks); - } + return append_json_typed_value(str, sv, depth, fname, n_param, marks, + caller_reads_back, item->is_valid_json(), + item->is_nice_json(), need_nice); + + /* + Being a document is what gets a value spliced rather than quoted, + so an item attesting to one it is not going to be asked about has + lost track of which of the two it returns. + */ + DBUG_ASSERT(!item->is_valid_json()); rc= append_escaped_value(str, sv, item->result_type() == STRING_RESULT, fname, n_param); @@ -2567,16 +3443,21 @@ static int append_json_value_from_field(String *str, if (f->is_null_in_record(key)) goto append_null; if (is_json_type(i)) - { /* The bytes come out of a record rather than off an item, and a record says nothing about how what is in it was written, so the document being built cannot be said to be written the loose way. + Nothing has attested to them either: what an item says is about + the value its last evaluation returned, and this one was read + straight out of a record instead. + + Nor is anything written out again here. The only callers that + read a value out of a record are building a document and handing + back what they built, and a value put in as it stands is what a + released server returns. */ - if (marks) - marks->is_nice= false; - return append_json_typed_value(str, sv, depth, fname, n_param, marks); - } + return append_json_typed_value(str, sv, depth, fname, n_param, marks, + false, false, false, false); rc= append_escaped_value(str, sv, i->result_type() == STRING_RESULT, fname, n_param); @@ -2621,7 +3502,8 @@ static json_append_result append_json_keyname(String *str, Item *item, return rc; } - return str->append(STRING_WITH_LEN(json_loose_colon)) ? + return (str->append(STRING_WITH_LEN(json_loose_colon)) || + DBUG_IF("json_keyname_colon_out_of_memory")) ? JSON_APPEND_OOM : JSON_APPEND_OK; append_null: @@ -2680,7 +3562,7 @@ String *Item_func_json_array::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; - Json_splice_marks marks= {true, true}; + Json_splice_marks marks; m_marks.clear(); @@ -2689,14 +3571,15 @@ String *Item_func_json_array::val_str(String *str) if (str->append('[') || ((arg_count > 0) && - append_json_value(str, args[0], &tmp_val, 1, func_name(), 0, &marks))) + append_json_value(str, args[0], &tmp_val, 1, func_name(), 0, &marks, + false, false))) goto err_return; for (n_arg=1; n_arg < arg_count; n_arg++) { if (str->append(STRING_WITH_LEN(json_loose_comma)) || append_json_value(str, args[n_arg], &tmp_val, 1, func_name(), - (int) n_arg, &marks)) + (int) n_arg, &marks, false, false)) goto err_return; } @@ -2713,7 +3596,11 @@ String *Item_func_json_array::val_str(String *str) what was learned putting it together: the brackets and separators are written here in the loose form, the values that were quoted came out of json_escape() and are documents on their own, - and the values that went in as they stand were read as they went. + and the values that went in as they stand were either read as they + went or passed by something that had already read them. + Which of the two it was does not matter here - marks.is_valid and + marks.is_nice are cleared by whichever of them did not hold, and + that is the whole of what this has to go on. All of which holds only where the brackets written above are brackets - see is_json_compatible_charset(). @@ -2760,6 +3647,34 @@ bool Item_func_json_array_append::fix_length_and_dec(THD *thd) } +/* + Returns the document in 'from' as this function's answer instead of + reading it again to find out what it is. + + A caller gets here only where everything that went into the answer is + attested to, so what it returns is a document written the loose + way and nothing has to look at it to say so. Which of the two + the answer is already sitting in is the one thing the callers differ + in, so the copy is made only where it is owed. + + 'to' is what comes back, or NULL where the buffer would not grow. + Reporting that is the caller's as well: they do not all report it in + the same place, some of them having a document engine to complain + through and some not. +*/ + +String *Item_json_func::return_json(String *to, const String *from) +{ + if ((from != to && to->copy(from->ptr(), from->length(), from->charset())) || + DBUG_IF("json_return_out_of_memory")) + return NULL; + + null_value= 0; + m_marks.set(to, true, true); + return to; +} + + String *Item_func_json_array_append::val_str(String *str) { json_engine_t je; @@ -2767,7 +3682,17 @@ String *Item_func_json_array_append::val_str(String *str) uint n_arg, n_path; size_t str_rest_len; const uchar *ar_end; + const char *js_end; THD *thd; + String *const to= str; + bool compose_final; + uint depth; + Json_source_watch watch; + /* + Cleared by anything spliced in that leaves is_valid or is_nice + false for the answer, the depth a path reaches among it. + */ + Json_splice_marks splice; DBUG_ASSERT(fixed()); m_marks.clear(); @@ -2778,10 +3703,25 @@ String *Item_func_json_array_append::val_str(String *str) thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, for the reason given where + Item_func_json_insert::val_str() asks it. + */ + compose_final= document_arg_composes_final(args[0], js); + for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { int array_counters[JSON_DEPTH_LIMIT]= {0}; json_path_with_flags *c_path= paths + n_path; + + /* + Taken before the path is worked out, and afresh every time round. + See Item_func_json_insert::val_str(). + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -2800,10 +3740,18 @@ String *Item_func_json_array_append::val_str(String *str) if (args[n_arg]->null_value) goto return_null; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; + /* + Where the document ends, read where the walk over it starts rather + than after the value has been worked out, so that the end copied + from is the end that was walked to. See Json_source_watch. + */ + js_end= js->end(); + c_path->cur_step= c_path->p.steps; if (json_find_path(&je, &c_path->p, &c_path->cur_step, array_counters)) @@ -2817,6 +3765,20 @@ String *Item_func_json_array_append::val_str(String *str) if (json_read_value(&je)) goto js_error; + /* + How many structures the value will sit inside once it is in + place. Read off here because walking to the end of the array + below pops the scanner back out of it. + + Reading a container puts it on the stack; reading a scalar does + not. So for an array, and for an object about to be wrapped in a + new one, the scanner is already counting the structure the value + is going into, and counting it again would refuse a document that + fits. Only a scalar being wrapped needs the one added, the new + array being a structure the scanner has never seen. + */ + depth= (uint) je.stack_p + (json_value_scalar(&je) ? 1 : 0); + str->length(0); str->set_charset(js->charset()); if (str->reserve(js->length() + 8, 1024)) @@ -2833,8 +3795,9 @@ String *Item_func_json_array_append::val_str(String *str) str->q_append(js->ptr(), ar_end-(const uchar *) js->ptr()); if (n_items) str->append(", ", 2); - if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL)) + if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, &splice, + !compose_final, compose_final)) goto return_null; /* Out of memory. */ if (str->reserve(str_rest_len, 1024)) @@ -2851,6 +3814,13 @@ String *Item_func_json_array_append::val_str(String *str) if (je.value_type == JSON_VALUE_OBJECT) { + /* + Wrapping puts what was already there inside a new array, so + the KEPT value goes down a level too - and how far down it + already reached was never measured, it having been copied + rather than read. Say so and let the answer be read back. + */ + splice.is_valid= splice.is_nice= false; if (json_skip_level(&je)) goto js_error; c_to= je.s.c_str; @@ -2870,13 +3840,15 @@ String *Item_func_json_array_append::val_str(String *str) if (str->append('[') || append_simple(str, c_from, c_to - c_from) || str->append(", ", 2) || - append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL) || + append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, &splice, + !compose_final, compose_final) || str->append(']') || append_simple(str, je.s.c_str, - js->end() - (const char *) je.s.c_str)) + js_end - (const char *) je.s.c_str)) goto return_null; /* Out of memory. */ } + DBUG_ASSERT(watch.unchanged(js)); { /* Swap str and js. */ if (str == &tmp_js) @@ -2892,13 +3864,30 @@ String *Item_func_json_array_append::val_str(String *str) } } + /* + A document that was already written the loose way, with a value + written that way put inside it, is written that way already. See + Item_func_json_insert::val_str() for why the answer is in js, and + for what makes this safe to believe. + */ + if (compose_final && splice.is_valid && splice.is_nice) + { + if (!return_json(to, js)) + goto return_null; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; if (json_nice(&je, str, Item_func_json_format::LOOSE)) goto js_error; - /* The reading back just above is what sets both marks. */ + /* + Both marks come from the reading back, for the reason given where + Item_func_json_insert::val_str() marks the same answer. + */ m_marks.set(str, true, true); return str; @@ -2916,7 +3905,17 @@ String *Item_func_json_array_insert::val_str(String *str) json_engine_t je; String *js= args[0]->val_json(&tmp_js); uint n_arg, n_path; + const char *js_end; THD *thd; + String *const to= str; + bool compose_final; + uint depth; + Json_source_watch watch; + /* + Cleared by anything spliced in that leaves is_valid or is_nice + false for the answer, the depth a path reaches among it. + */ + Json_splice_marks splice; DBUG_ASSERT(fixed()); m_marks.clear(); @@ -2927,6 +3926,14 @@ String *Item_func_json_array_insert::val_str(String *str) thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, for the reason given where + Item_func_json_insert::val_str() asks it. + */ + compose_final= document_arg_composes_final(args[0], js); + for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { int array_counters[JSON_DEPTH_LIMIT]= {0}; @@ -2934,6 +3941,12 @@ String *Item_func_json_array_insert::val_str(String *str) const char *item_pos; int n_item, corrected_n_item; + /* + Taken before the path is worked out, and afresh every time round. + See Item_func_json_insert::val_str(). + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -2962,10 +3975,17 @@ String *Item_func_json_array_insert::val_str(String *str) if (args[n_arg]->null_value) goto return_null; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; + /* + Where the document ends, read where the walk over it starts. See + Json_source_watch. + */ + js_end= js->end(); + c_path->cur_step= c_path->p.steps; if (json_find_path(&je, &c_path->p, &c_path->cur_step, array_counters)) @@ -2986,6 +4006,15 @@ String *Item_func_json_array_insert::val_str(String *str) continue; } + /* + How many structures the value will sit inside once it is in + place: the ones holding the array, and the array itself. Reading + the array has already put it on the stack, so the scanner is + counting it; read off here, before the walk below moves the + scanner about inside it. + */ + depth= (uint) je.stack_p; + item_pos= 0; n_item= 0; corrected_n_item= c_path->p.last_step[1].n_item; @@ -3031,8 +4060,9 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 1); goto return_null; /* Out of memory. */ } - if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL)) + if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, &splice, + !compose_final, compose_final)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ @@ -3047,7 +4077,7 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 1); goto return_null; /* Out of memory. */ } - size= js->end() - item_pos; + size= js_end - item_pos; if (append_simple(str, item_pos, size)) { my_error(ER_OUTOFMEMORY, MYF(0), (int) size); @@ -3071,13 +4101,14 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 2); goto return_null; /* Out of memory. */ } - if (append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL)) + if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, &splice, + !compose_final, compose_final)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ } - size= js->end() - item_pos; + size= js_end - item_pos; if (append_simple(str, item_pos, size)) { my_error(ER_OUTOFMEMORY, MYF(0), (int) size); @@ -3085,6 +4116,7 @@ String *Item_func_json_array_insert::val_str(String *str) } } + DBUG_ASSERT(watch.unchanged(js)); { /* Swap str and js. */ if (str == &tmp_js) @@ -3100,13 +4132,30 @@ String *Item_func_json_array_insert::val_str(String *str) } } + /* + A document that was already written the loose way, with a value + written that way put inside it, is written that way already. See + Item_func_json_insert::val_str() for why the answer is in js, and + for what makes this safe to believe. + */ + if (compose_final && splice.is_valid && splice.is_nice) + { + if (!return_json(to, js)) + goto return_null; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; if (json_nice(&je, str, Item_func_json_format::LOOSE)) goto js_error; - /* The reading back just above is what sets both marks. */ + /* + Both marks come from the reading back, for the reason given where + Item_func_json_insert::val_str() marks the same answer. + */ m_marks.set(str, true, true); return str; @@ -3123,7 +4172,7 @@ String *Item_func_json_object::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; - Json_splice_marks marks= {true, true}; + Json_splice_marks marks; m_marks.clear(); @@ -3133,7 +4182,8 @@ String *Item_func_json_object::val_str(String *str) if (str->append('{') || (arg_count > 0 && (append_json_keyname(str, args[0], &tmp_val, func_name(), 0) || - append_json_value(str, args[1], &tmp_val, 1, func_name(), 1, &marks)))) + append_json_value(str, args[1], &tmp_val, 1, func_name(), 1, &marks, + false, false)))) goto err_return; for (n_arg=2; n_arg < arg_count; n_arg+=2) @@ -3142,7 +4192,7 @@ String *Item_func_json_object::val_str(String *str) append_json_keyname(str, args[n_arg], &tmp_val, func_name(), (int) n_arg) || append_json_value(str, args[n_arg+1], &tmp_val, 1, func_name(), - (int) n_arg + 1, &marks)) + (int) n_arg + 1, &marks, false, false)) goto err_return; } @@ -3174,7 +4224,15 @@ String *Item_func_json_object::val_str(String *str) } -static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) +/* + 'wrapped' is raised where the merging puts what it was given inside a + new array. That array is a level of its own, and how deep either + document already went was never measured - they were copied, not read. + Whoever is composing an answer out of this therefore cannot say how + deep the answer is, and has to have it read back rather than guess. +*/ +static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2, + bool *wrapped, uint colon_len) { DBUG_EXECUTE_IF("json_check_min_stack_requirement", return dbug_json_check_min_stack_requirement();); @@ -3221,6 +4279,22 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) *je2= sav_je2; } + /* + Written the compact way on purpose, but only half of what + follows earns it. A key only this document holds is copied out + of it below from where the space after the colon stands, so + writing one here would write it twice. A key both documents + hold is composed instead, and a composed value begins at the + value itself - so that arm writes the space it needs before it + starts, and it is the only one that has to. + + It writes one only where the answer is going out as it is + composed, which is what colon_len says. Where the answer is to + be read back, the reading writes the whole of it anyway, and + writing a space here would only move every offset a released + server reports - including the ones it reports about documents + it then refuses. + */ if (str->append('"') || append_simple(str, key_start, key_end - key_start) || str->append("\":", 2)) @@ -3240,7 +4314,9 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) } /* Json_2 has same key as Json_1. Merge them. */ - if ((ires= do_merge(str, je1, je2))) + if (colon_len == 3 && str->append(' ')) + return 3; + if ((ires= do_merge(str, je1, je2, wrapped, colon_len))) return ires; goto merged_j1; } @@ -3333,6 +4409,8 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) } else { + /* A level of its own - see where this is declared. */ + *wrapped= true; if (str->append('[')) return 3; if (je1->value_type == JSON_VALUE_OBJECT) @@ -3345,7 +4423,16 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) end1= je1->value_end; } - if (str->append((const char*) beg1, end1 - beg1)) + /* + Copied, not written: these bytes came out of a document and are + already in the character set the answer is being built in. The + appends that convert are for punctuation written here, which + arrives as plain ASCII and has to be turned into the set; putting + a copied span through one of those writes it a second time, and in + a set that spends more than one byte on a character what comes out + is not a document at all. + */ + if (append_simple(str, beg1, end1 - beg1)) return 3; if (json_value_scalar(je2)) @@ -3370,8 +4457,9 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) end2= je2->s.c_str; } + /* The comma is written and converts; the span is copied and must not. */ if ((n_items1 && n_items2 && str->append(", ", 2)) || - str->append((const char*) beg2, end2 - beg2)) + append_simple(str, beg2, end2 - beg2)) return 3; if (je2->value_type != JSON_VALUE_ARRAY && @@ -3430,6 +4518,16 @@ String *Item_func_json_merge::val_str(String *str) String *js1= args[0]->val_json(&tmp_js1), *js2=NULL; uint n_arg; THD *thd; + String *const to= str; + bool compose_final; + /* + The formatting of the punctuation written here, settled from the first + document alone and before anything is composed - see below. + */ + uint colon_len; + /* Raised where the merging puts what it was given inside a new array. */ + bool wrapped= false; + Json_source_watch watch; m_marks.clear(); @@ -3439,15 +4537,44 @@ String *Item_func_json_merge::val_str(String *str) thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + /* + Whether what is composed here is what will be returned. Every + document merged in has a say, and they are only read one at a time, + so the answer is settled over the loop. The FORMATTING cannot wait + for that - what is written early is written before the later + arguments have been looked at - so it follows the first document + alone, which is known here. A later argument answering is_valid or + is_nice false then sends the answer through the reading back after + all, and the + positions it reports are offsets into text this function composed + rather than into anything a caller wrote. + */ + compose_final= document_arg_composes_final(args[0], js1); + colon_len= compose_final ? 3 : 2; + for (n_arg=1; n_arg < arg_count; n_arg++) { str->set_charset(js1->charset()); str->length(0); + /* + The document on the left is not read until below, and working out + the one on the right can be any expression at all. See + Json_source_watch. + */ + watch.take(js1); js2= args[n_arg]->val_json(&tmp_js2); + DBUG_ASSERT(watch.unchanged(js1)); if (args[n_arg]->null_value) goto null_return; + /* + Asked after the argument has been evaluated, what it answers being + about the value it has just passed. + */ + if (!args[n_arg]->is_valid_json() || !args[n_arg]->is_nice_json()) + compose_final= false; + json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; @@ -3456,9 +4583,19 @@ String *Item_func_json_merge::val_str(String *str) (const uchar *) js2->ptr() + js2->length()); je2.killed_ptr= (uint32_t *) &thd->killed; - if (do_merge(str, &je1, &je2)) + if (do_merge(str, &je1, &je2, &wrapped, colon_len)) goto error_return; + /* + Putting the two inside a new array makes the answer a level deeper + than either of them, and how deep either of them went was never + measured. Have it read back, the same reading a released server + always did, which is also what puts the complaint in the place + that server put it. + */ + if (wrapped) + compose_final= false; + /* Both were read as far as their first value and no further. Only the first turn of the loop has a document argument on the left: @@ -3484,6 +4621,25 @@ String *Item_func_json_merge::val_str(String *str) } } + /* + Documents that were all written the loose way, joined with + punctuation written the same way, make a document written that way. + See Item_func_json_insert::val_str() for why the answer is in js1, + and for what makes this safe to believe. + + Nothing is spliced in here - every piece of the answer is a document + argument, and each of them has already had its say in compose_final + as it came round the loop - so there is no set of splice marks to + ask, and the one condition is the whole of it. + */ + if (compose_final) + { + if (!return_json(to, js1)) + goto error_return; + + return to; + } + json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; @@ -3492,7 +4648,10 @@ String *Item_func_json_merge::val_str(String *str) goto error_return; null_value= 0; - /* The reading back just above is what sets both marks. */ + /* + Both marks come from the reading back, for the reason given where + Item_func_json_insert::val_str() marks the same answer. + */ m_marks.set(str, true, true); return str; @@ -3569,7 +4728,7 @@ static int copy_value_patch(String *str, json_engine_t *je) static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, - bool *empty_result) + bool *empty_result, uint colon_len) { DBUG_EXECUTE_IF("json_check_min_stack_requirement", return dbug_json_check_min_stack_requirement();); @@ -3622,7 +4781,7 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, if (str->append('"') || append_simple(str, key_start, key_end - key_start) || - str->append("\":", 2)) + str->append(json_loose_colon, colon_len)) return 3; while (json_scan_next(je2) == 0 && @@ -3639,7 +4798,7 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, } /* Json_2 has same key as Json_1. Merge them. */ - if ((ires= do_merge_patch(str, je1, je2, &mrg_empty))) + if ((ires= do_merge_patch(str, je1, je2, &mrg_empty, colon_len))) return ires; if (mrg_empty) @@ -3657,6 +4816,15 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, /* Just append the Json_1 key value. */ if (json_skip_key(je1)) return 1; + /* + This span begins where the colon left off, and in a document + written the loose way that is the space after it - the space + just written above. The other way out of this loop has the + value written afresh, with no space of its own, which is why + the colon writes one at all. + */ + if (colon_len == 3) + key_start= json_skip_space(je1->s.cs, key_start, je1->s.c_str); if (append_simple(str, key_start, je1->s.c_str - key_start)) return 3; first_key= 0; @@ -3713,7 +4881,7 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, if (str->append('"') || append_simple(str, key_start, key_end - key_start) || - str->append("\":", 2)) + str->append(json_loose_colon, colon_len)) return 3; if (json_read_value(je2)) @@ -3757,6 +4925,16 @@ String *Item_func_json_merge_patch::val_str(String *str) uint n_arg; bool empty_result, merge_to_null; THD *thd= current_thd; + String *const to= str; + bool compose_final; + uint colon_len; + Json_source_watch watch; + /* + Cleared by anything that goes into the answer that the answer cannot + be said to hold: a document taken over without having been read on + the way in, and a document answering is_nice false. + */ + Json_splice_marks splice; m_marks.clear(); @@ -3766,15 +4944,44 @@ String *Item_func_json_merge_patch::val_str(String *str) je1.s.error= je2.s.error= 0; merge_to_null= args[0]->null_value; + /* + The formatting follows the first document alone, for the reason given + where Item_func_json_merge::val_str() picks its own. A first + argument that is SQL NULL contributes nothing to the answer and is + not asked, and leaves nothing to read the character set off either. + */ + compose_final= !merge_to_null && document_arg_composes_final(args[0], js1); + colon_len= compose_final ? 3 : 2; + if (!compose_final) + splice.is_nice= false; + for (n_arg=1; n_arg < arg_count; n_arg++) { + /* + The document on the left is not read until below, and working out + the one on the right can be any expression at all. See + Json_source_watch. A left-hand side that is SQL NULL has no bytes + to keep an eye on. + */ + if (!merge_to_null) + watch.take(js1); js2= args[n_arg]->val_json(&tmp_js2); + DBUG_ASSERT(merge_to_null || watch.unchanged(js1)); if (args[n_arg]->null_value) { merge_to_null= true; goto cont_point; } + /* + Asked after the argument has been evaluated, what it answers being + about the value it has just passed. A document merged in is + read as it is copied, so nothing here is about whether it is a + document - only about how it is written. + */ + if (!args[n_arg]->is_nice_json()) + splice.is_nice= false; + json_scan_start(&je2, js2->charset(),(const uchar *) js2->ptr(), (const uchar *) js2->ptr() + js2->length()); je2.killed_ptr= (uint32_t *) &thd->killed; @@ -3825,6 +5032,23 @@ String *Item_func_json_merge_patch::val_str(String *str) if (str->copy(js2->ptr(), js2->length(), js2->charset())) goto error_return; + /* + The whole of it goes in and only its first value was read, so + whatever stands after that value goes in unread. Text standing + there is no reason to refuse the document: what comes of it is + a matter for whatever the answer turns out to be, and a later + argument merging over this one leaves it out of the answer + altogether, which is an answer that has always been given back. + + A document whose item attests is_valid has nothing against it + to begin with, and goes in the way it was written. Anything + else has to be read back, so say here that it has to be. + */ + if (!args[n_arg]->is_valid_json()) + splice.is_valid= splice.is_nice= false; + else if (!args[n_arg]->is_nice_json()) + splice.is_nice= false; + goto cont_point; } @@ -3836,7 +5060,7 @@ String *Item_func_json_merge_patch::val_str(String *str) (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; - if (do_merge_patch(str, &je1, &je2, &empty_result)) + if (do_merge_patch(str, &je1, &je2, &empty_result, colon_len)) goto error_return; /* @@ -3871,6 +5095,28 @@ String *Item_func_json_merge_patch::val_str(String *str) if (merge_to_null) goto null_return; + /* + Documents that were all written the loose way, patched with + punctuation written the same way, make a document written that way. + See Item_func_json_insert::val_str() for why the answer is in js1, + and for what makes this safe to believe. + + Asked of the splice marks alone, where the others ask compose_final + too, and it comes to the same thing: what the others keep in that + variable this one keeps in colon_len, which is the wider formatting + only where the first document answers is_valid and is_nice and can + be written - + and anything else clears is_nice on the spot. A document taken over + whole clears both marks where it was taken. + */ + if (splice.is_valid && splice.is_nice) + { + if (!return_json(to, js1)) + goto error_return; + + return to; + } + json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; @@ -3878,7 +5124,10 @@ String *Item_func_json_merge_patch::val_str(String *str) goto error_return; null_value= 0; - /* The reading back just above is what sets both marks. */ + /* + Both marks come from the reading back, for the reason given where + Item_func_json_insert::val_str() marks the same answer. + */ m_marks.set(str, true, true); return str; @@ -3907,6 +5156,7 @@ longlong Item_func_json_length::val_int() { String *js= args[0]->val_json(&tmp_js); json_engine_t je; + Json_source_watch watch; uint length= 0; int array_counters[JSON_DEPTH_LIMIT]= {0}; int err; @@ -3918,6 +5168,7 @@ longlong Item_func_json_length::val_int() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -3944,6 +5195,7 @@ longlong Item_func_json_length::val_int() goto null_return; path.cur_step= path.p.steps; + DBUG_ASSERT(watch.unchanged(js)); if (json_find_path(&je, &path.p, &path.cur_step, array_counters)) { if (je.s.error) @@ -3978,9 +5230,13 @@ longlong Item_func_json_length::val_int() }; } - if (!err) + /* + The rest of the document is parsed only to check that it is one. + Not done at all where the item has already attested that the value + is_valid: the parse could only report what is already known. + */ + if (!err && !args[0]->is_valid_json()) { - /* Parse to the end of the JSON just to check it's valid. */ while (json_scan_next(&je) == 0) {} } @@ -4065,14 +5321,35 @@ String *Item_func_json_type::val_str(String *str) String *js= args[0]->val_json(&tmp_js); json_engine_t je; const char *type; + THD *thd; if ((null_value= args[0]->null_value)) return 0; + thd= current_thd; + je.killed_ptr= (uint32_t *) &thd->killed; + + /* + Returns the type of the document's first value, or NULL if the + document does not parse all the way through. Parsing it through + comes first, then: a type worked out for text that turns out not to + be a document is a type that is never returned. + + Not done at all where the item has already attested that the value + is_valid. The parse could only report what is already known, and it + is the parse this whole change is about not making. + + What follows it is not that parse repeated. json_read_value() stops + at the bracket that opens a container - see read_obj() - so for any + document but a bare scalar it is one step and not a second walk. + */ + if (!args[0]->is_valid_json() && + !json_valid_engine(&je, js->ptr(), js->length(), js->charset())) + goto error; json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); - je.killed_ptr= (uint32_t *) ¤t_thd->killed; + je.killed_ptr= (uint32_t *) &thd->killed; if (json_read_value(&je)) goto error; @@ -4100,12 +5377,6 @@ String *Item_func_json_type::val_str(String *str) break; } - /* ensure the json is at least valid. */ - while(json_scan_next(&je) == 0) {} - - if (je.s.error) - goto error; - str->set(type, strlen(type), &my_charset_utf8mb3_general_ci); return str; @@ -4150,7 +5421,18 @@ String *Item_func_json_insert::val_str(String *str) uint n_arg, n_path; json_string_t key_name; StringBuffer tmp_key; + const char *js_end; THD *thd; + String *const to= str; + bool compose_final; + uint colon_len; + Json_source_watch watch; + /* + Cleared by anything spliced in that leaves is_valid or is_nice false + for the answer, the depth a path reaches among it. Nothing reads it yet; the reading + back below is still what settles the answer. + */ + Json_splice_marks splice; DBUG_ASSERT(fixed()); m_marks.clear(); @@ -4158,6 +5440,25 @@ String *Item_func_json_insert::val_str(String *str) if ((null_value= args[0]->null_value)) return 0; + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, because the composing has to know it: + an answer that is going to be written out again must be composed + exactly as it always was, positions reported against it being + offsets into it. A document whose item attests neither is_valid nor + is_nice is such an answer, and it is the one a caller wrote out by + hand. + + A character set that cannot encode the punctuation written here is + the other one - see is_json_compatible_charset(). Nothing composed in + such a set is a document, however sound the pieces were, and until + the reading back went away it was the only thing that ever noticed. + So it is kept exactly where it was. + */ + compose_final= document_arg_composes_final(args[0], js); + colon_len= compose_final ? 3 : 2; + thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); @@ -4172,6 +5473,18 @@ String *Item_func_json_insert::val_str(String *str) json_path_step_t *lp; int corrected_n_item; + /* + Taken before the path is worked out. A path is any expression a + caller cares to write, and it is the first thing here that can + reach the document; once the walk below has started, its pointers + into the document are live and there is nothing left to catch. + + Taken afresh every time round, too: the end of this loop swaps + what was composed here into js, so what the document IS changes + between one path and the next, on purpose. + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -4196,10 +5509,17 @@ String *Item_func_json_insert::val_str(String *str) if (args[n_arg]->null_value) goto return_null; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; + /* + Where the document ends, read where the walk over it starts. See + Json_source_watch. + */ + js_end= js->end(); + if (c_path->p.last_step < c_path->p.steps) goto v_found; @@ -4253,6 +5573,16 @@ String *Item_func_json_insert::val_str(String *str) if (je.value_type == JSON_VALUE_OBJECT) { + /* + Wrapping puts what was already there inside a new array, so + the KEPT value goes down a level as well as the one being + put in - and how far down the kept one already reached was + never measured, it having been copied rather than read. Say + so and let the answer be read back. A scalar has nothing + inside it and so needs no saying. + */ + if (do_array_autowrap) + splice.is_valid= splice.is_nice= false; if (json_skip_level(&je)) goto js_error; } @@ -4260,10 +5590,12 @@ String *Item_func_json_insert::val_str(String *str) if ((do_array_autowrap && (append_simple(str, v_from, je.s.c_str - v_from) || str->append(", ", 2))) || - append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL) || + append_json_value(str, args[n_arg+1], &tmp_val, + (uint) je.stack_p + (do_array_autowrap ? 1 : 0), + func_name(), (int) n_arg + 1, &splice, + !compose_final, compose_final) || (do_array_autowrap && str->append(']')) || - append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) + append_simple(str, je.s.c_str, js_end - (const char *) je.s.c_str)) goto js_error; /* Out of memory. */ goto continue_point; @@ -4301,16 +5633,28 @@ String *Item_func_json_insert::val_str(String *str) v_to= (const char *) (je.s.c_str - je.sav_c_len); str->length(0); + /* + One deeper than the scanner says. Getting here means the walk + ran off the end of the array, and an array is taken off the + stack BEFORE the state that says it ended is set - so by now the + array the value is going into is no longer counted, and the + value goes inside it. The arm that finds the place it was + looking for reads the level after skipping back over the value, + with the array still counted, and so needs no such correction. + */ if (append_simple(str, js->ptr(), v_to - js->ptr()) || (n_item > 0 && str->append(", ", 2)) || - append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL) || - append_simple(str, v_to, js->end() - v_to)) + append_json_value(str, args[n_arg+1], &tmp_val, + (uint) je.stack_p + 1, + func_name(), (int) n_arg + 1, &splice, + !compose_final, compose_final) || + append_simple(str, v_to, js_end - v_to)) goto js_error; /* Out of memory. */ } else /*JSON_PATH_KEY*/ { uint n_key= 0; + bool key_inert= false; if (je.value_type != JSON_VALUE_OBJECT) continue; @@ -4350,17 +5694,41 @@ String *Item_func_json_insert::val_str(String *str) str->length(0); if (append_simple(str, js->ptr(), v_to - js->ptr()) || (n_key > 0 && str->append(", ", 2)) || - str->append('"')) + str->append('"') || + DBUG_IF("json_insert_key_out_of_memory")) goto js_error; /* Out of memory. */ if (append_json_path_key(str, lp, c_path->p.s.cs, &tmp_key, - func_name(), (int) n_arg + 1)) + func_name(), (int) n_arg + 1, &key_inert) || + DBUG_IF("json_insert_path_key_out_of_memory")) goto js_error; /* Out of memory. */ - if (str->append("\":", 2) || - append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL) || - append_simple(str, v_to, js->end() - v_to)) + /* + A key that reaches past the quotes around it can make the answer + anything at all - unreadable, or a different document that reads + perfectly well. Which of the two it is can only be told by + reading the whole of it, so say here that it has to be. + */ + if (!key_inert) + splice.is_valid= splice.is_nice= false; + + /* + Written the loose way when that is how the answer is going out, + like the comma above it and the value after it. A key put in + here is the only punctuation the editing adds of its own, so it + is the only place the two formats can differ. + */ + /* + One deeper than the scanner says, for the reason given at the + end of the array above: the object is taken off the stack before + the state that says it ended, and the new key goes inside it. + */ + if (str->append(json_loose_colon, colon_len) || + append_json_value(str, args[n_arg+1], &tmp_val, + (uint) je.stack_p + 1, + func_name(), (int) n_arg + 1, &splice, + !compose_final, compose_final) || + append_simple(str, v_to, js_end - v_to)) goto js_error; /* Out of memory. */ } @@ -4383,11 +5751,13 @@ String *Item_func_json_insert::val_str(String *str) } if (append_simple(str, js->ptr(), v_to - js->ptr()) || - append_json_value(str, args[n_arg+1], &tmp_val, 0, func_name(), - (int) n_arg + 1, NULL) || - append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) + append_json_value(str, args[n_arg+1], &tmp_val, (uint) je.stack_p, + func_name(), (int) n_arg + 1, &splice, + !compose_final, compose_final) || + append_simple(str, je.s.c_str, js_end - (const char *) je.s.c_str)) goto js_error; /* Out of memory. */ continue_point: + DBUG_ASSERT(watch.unchanged(js)); { /* Swap str and js. */ if (str == &tmp_js) @@ -4403,13 +5773,79 @@ String *Item_func_json_insert::val_str(String *str) } } + /* + A document that was already written the loose way, edited with + pieces that were themselves written that way, is written that way + already: reading it back would write it exactly as it stands. So it + is passed as it stands, and the reading below is the reading + this whole exercise is about doing away with. + + THIS IS THE CANONICAL SITE. The other five that edit a document + point here rather than repeat it. + + What makes the answer A DOCUMENT. What was kept of the document was + read as one on the way in - that is compose_final's + args[0]->is_valid_json() - and is copied out of it whole, at offsets + the walk above computed rather than guessed. What was put in was + either read here as it went in, or attested to by whoever produced + it, and either way splice.is_valid says so. What was written here + is punctuation and a key, and a key that could reach past its own + quotes clears the same mark rather than being refused. + + What makes it WRITTEN THE LOOSE WAY. The kept parts were, by + args[0]->is_nice_json(); the values put in were, by splice.is_nice; + and the punctuation written here was, colon_len having been picked + from compose_final before anything was composed. + + And none of that means anything where the punctuation cannot be + written at all, which is compose_final's third part - see + is_json_compatible_charset(). A document in such a character set is a + scalar, because a container would take brackets it does not have. + + Nothing here takes any of it on faith. m_marks.set() reads the + answer back in a debug build and stops the server if these + conditions ever hold over something that is not a document written + that way - which is the same reading, kept exactly where it is worth + its cost and nowhere else. + + The answer is in js rather than str: each pass writes into str and + then swaps the two, so after an odd number of passes str is the + scratch. It is returned in the buffer the caller supplied, which + every path through this function has always done. + */ + if (compose_final && splice.is_valid && splice.is_nice) + { + if (!return_json(to, js)) + goto js_error; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; if (json_nice(&je, str, Item_func_json_format::LOOSE)) goto js_error; - /* The reading back just above is what sets both marks. */ + /* + The reading back just above is what sets both marks. It also says + whether the depth reckoned at each splice was reckoned right: a + reading that got to the end is an answer inside the limit, so + nothing should have been marked as taking it past one. The other + way round is allowed - a value can be marked for reasons that leave + a readable answer behind, an empty one among them. + */ + DBUG_ASSERT(!splice.is_deep); + + /* + Both marks come from the reading back and from it alone: json_nice() + got to the end of what was composed, which is what makes it a + document, and what is marked is what json_nice() WROTE rather than + what it read, which is what makes it written the loose way. This is + the reading that stays wherever the document handed in was nobody's + word - it is not a leftover, and taking it out would take 10.11's + answers with it. + */ m_marks.set(str, true, true); return str; @@ -4439,6 +5875,10 @@ String *Item_func_json_remove::val_str(String *str) uint n_arg, n_path; json_string_t key_name; THD *thd; + String *const to= str; + bool compose_final; + uint comma_len; + Json_source_watch watch; DBUG_ASSERT(fixed()); m_marks.clear(); @@ -4451,6 +5891,15 @@ String *Item_func_json_remove::val_str(String *str) str->set_charset(js->charset()); + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, for the reason given where + Item_func_json_insert::val_str() asks it. + */ + compose_final= document_arg_composes_final(args[0], js); + comma_len= compose_final ? 2 : 1; + for (n_arg=1, n_path=0; n_arg < arg_count; n_arg++, n_path++) { int array_counters[JSON_DEPTH_LIMIT]= {0}; @@ -4459,6 +5908,13 @@ String *Item_func_json_remove::val_str(String *str) json_path_step_t *lp; int n_item= 0; + /* + Taken afresh every time round: the end of this loop swaps what + was composed here into js, so what the document IS changes + between one path and the next, on purpose. + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -4489,6 +5945,7 @@ String *Item_func_json_remove::val_str(String *str) if (args[n_arg]->null_value) goto null_return; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -4592,13 +6049,42 @@ String *Item_func_json_remove::val_str(String *str) if (json_skip_key(&je) || json_scan_next(&je)) goto js_error; - rem_end= (je.state == JST_VALUE && n_item == 0) ? + rem_end= (je.state == JST_VALUE && n_item == 0) ? (const char *) je.s.c_str : (const char *) (je.s.c_str - je.sav_c_len); + /* + Taking out the first piece of an array takes the comma after it + with it, and in a document written the loose way a space stands + behind that comma. That space belonged to the piece that has just + gone; leaving it puts it behind the bracket instead, in front of a + piece that already has all the spacing it needs. Every other + piece ends AT its comma and so leaves nothing over. + + Only where what is composed here is the answer. Where it is not, + the reading back at the end settles the spacing and drops that + space itself, so skipping it changes no answer - but the reading + complains, when it has to, about a position in the text it was + given, and a text composed a character shorter than a released + server composed it moves every such position by one. This is the + same reason the comma below is written the width a released + server writes it. + */ + if (compose_final && je.state == JST_VALUE && n_item == 0) + rem_end= (const char *) json_skip_space(je.s.cs, (const uchar *) rem_end, + (const uchar *) js->end()); + str->length(0); + /* + What is removed reaches from just before the piece to just before + the piece after it, so the two ends join without punctuation + everywhere but between two keys, where one comma has to be put + back. It is the only punctuation this function writes, and so + the only place the two formats can differ. + */ if (append_simple(str, js->ptr(), rem_start - js->ptr()) || - (je.state == JST_KEY && n_item > 0 && str->append(",", 1)) || + (je.state == JST_KEY && n_item > 0 && + str->append(json_loose_comma, comma_len)) || append_simple(str, rem_end, js->end() - rem_end)) goto js_error; /* Out of memory. */ @@ -4617,6 +6103,22 @@ String *Item_func_json_remove::val_str(String *str) } } + /* + Nothing is spliced in here - what is written is what was read, + less the piece that was asked for - so a document that was written + the loose way is still written that way with the piece gone, and + reading it back would write it exactly as it stands. See + Item_func_json_insert::val_str() for why the answer is in js, and + for what makes this safe to believe. + */ + if (compose_final) + { + if (!return_json(to, js)) + goto js_error; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -4624,7 +6126,10 @@ String *Item_func_json_remove::val_str(String *str) goto js_error; null_value= 0; - /* The reading back just above is what sets both marks. */ + /* + Both marks come from the reading back, for the reason given where + Item_func_json_insert::val_str() marks the same answer. + */ m_marks.set(str, true, true); return str; @@ -4685,6 +6190,7 @@ static int check_key_in_list(String *res, String *Item_func_json_keys::val_str(String *str) { json_engine_t je; + Json_source_watch watch; String *js= args[0]->val_json(&tmp_js); uint n_keys= 0; int array_counters[JSON_DEPTH_LIMIT]= {0}; @@ -4696,6 +6202,7 @@ String *Item_func_json_keys::val_str(String *str) thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -4724,6 +6231,7 @@ String *Item_func_json_keys::val_str(String *str) path.cur_step= path.p.steps; + DBUG_ASSERT(watch.unchanged(js)); if (json_find_path(&je, &path.p, &path.cur_step, array_counters)) { if (je.s.error) @@ -4924,7 +6432,9 @@ static int append_json_path(String *str, const json_path_t *p, String *Item_func_json_search::val_str(String *str) { + Json_source_watch watch; String *js= args[0]->val_json(&tmp_js); + watch.take(js); String *s_str= args[2]->val_str(&tmp_path); json_engine_t je; json_path_t p, sav_path; @@ -4968,6 +6478,7 @@ String *Item_func_json_search::val_str(String *str) goto null_return; } + DBUG_ASSERT(watch.unchanged(js)); json_get_path_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length(), &p); @@ -5101,11 +6612,13 @@ String *Item_func_json_format::val_str(String *str) { String *js= args[0]->val_json(&tmp_js); json_engine_t je; + Json_source_watch watch; int tab_size= 4; THD *thd; m_marks.clear(); + watch.take(js); if ((null_value= args[0]->null_value)) return 0; @@ -5129,6 +6642,7 @@ String *Item_func_json_format::val_str(String *str) tab_size= TAB_SIZE_LIMIT; } + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), (const uchar *) js->ptr()+js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -5330,10 +6844,11 @@ void Item_func_json_arrayagg::clear() String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) { int rc; - Json_splice_marks marks= {true, true}; + Json_splice_marks marks; m_tmp_json.length(0); - rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0, &marks); + rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0, &marks, + false, false); if (!marks.is_valid) m_elements_valid= false; @@ -5351,7 +6866,7 @@ String *Item_func_json_arrayagg::get_str_from_field(Item *i,Field *f, String *tmp, const uchar *key, size_t offset) { int rc; - Json_splice_marks marks= {true, true}; + Json_splice_marks marks; m_tmp_json.length(0); @@ -5575,7 +7090,7 @@ void Item_func_json_objectagg::clear() bool Item_func_json_objectagg::add() { StringBuffer buf; - Json_splice_marks marks= {true, true}; + Json_splice_marks marks; String *key; int rc; @@ -5618,7 +7133,8 @@ bool Item_func_json_objectagg::add() goto bad_pair; buf.length(0); - rc= append_json_value(&result, args[1], &buf, 1, func_name(), 1, &marks); + rc= append_json_value(&result, args[1], &buf, 1, func_name(), 1, &marks, + false, false); if (!marks.is_valid) m_pairs_valid= false; if (rc == JSON_APPEND_OOM) @@ -6156,6 +7672,7 @@ bool Item_func_json_overlaps::val_bool() json_engine_t je, ve; int result; THD *thd; + Json_source_watch watch; if ((null_value= (js == nullptr) || args[0]->null_value)) return 0; @@ -6163,6 +7680,7 @@ bool Item_func_json_overlaps::val_bool() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); if (!a2_parsed) { val= args[1]->val_json(&tmp_val); @@ -6175,6 +7693,7 @@ bool Item_func_json_overlaps::val_bool() return 0; } + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index bedb9b06bcc92..5217a201b3c46 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -184,6 +184,12 @@ class Item_json_func: public Item_str_func { protected: Json_result_marks m_marks; + /* + Return a document as this function's answer rather than reading it + again to find out what it is - out of line, where the rule it stands + on is written. + */ + String *return_json(String *to, const String *from); public: Item_json_func(THD *thd) :Item_str_func(thd) { } @@ -253,10 +259,15 @@ class Item_func_json_query: public Item_json_func, if (null_value) return NULL; /* - Not said to be nicely formatted: the piece is copied out with - whatever spacing the document it came from was written with. + The piece is copied out with whatever spacing the document it came + from was written with, so it is formatted the loose way exactly when + that document was. Cutting it out cannot change that: the loose + form writes the same punctuation wherever a value sits, so a value + inside a document is written there the way it would be written on + its own, and the two ends of the cut are the two ends of the value + with no spacing of the document's left on either side. */ - m_marks.set(to, true, false); + m_marks.set(to, true, args[0]->is_nice_json()); return to; } bool check_and_get_value(Json_engine_scan *je, @@ -371,9 +382,13 @@ class Item_func_json_extract: public Item_json_str_multipath my_decimal *val_decimal(my_decimal *) override; uint get_n_paths() const override { return arg_count - 1; } /* - The values found are put together into a result of their own, and - that result is then read back through json_nice() before it is - passed on. Anything it cannot read comes back as NULL. + The values found are put together into a result of their own. Each + of them was read as a document on the way in and is written back out + as one, so what holds them is all that can be wrong with the result + - and that is the brackets, written only when several values can + match. In a character set that cannot encode those, the result is + read back through json_nice() as it always was, and anything that + will not read comes back as NULL. */ bool is_valid_json_static() const override { return true; } @@ -478,11 +493,16 @@ class Item_func_json_array_append: public Item_json_str_multipath return name; } /* - Every function that EDITS a document reads the whole of what it has - written back through json_nice() before handing it over, and hands - back NULL for anything that will not read. That reading is what - makes is_valid true for the result, and it is the same in all six - of them. + Each of the six functions that EDIT a document returns a document + or NULL, and for the same two reasons in all of them. + + Where the document it was given is_valid, and the result character + set can represent the punctuation being written, the result is a + document by construction: the parts retained were parsed as one on + input, and what goes between them is written here. Where either + does not hold, the whole result is re-parsed through json_nice() + before being returned, and anything that fails to parse gives + NULL. */ bool is_valid_json_static() const override { return true; } @@ -543,7 +563,10 @@ class Item_func_json_merge: public Item_func_json_array static LEX_CSTRING name= {STRING_WITH_LEN("json_merge_preserve") }; return name; } - /* Reads its result back, as the other editing functions do. */ + /* + A document or nothing, for the reason given where + Item_func_json_array_append declares the same. + */ bool is_valid_json_static() const override { return true; } protected: @@ -586,6 +609,11 @@ class Item_func_json_normalize: public Item_json_func return name; } bool fix_length_and_dec(THD *thd) override; + /* + Written out afresh from a document read through in full, and always + in utf8mb4, which can encode everything written into it whatever the + argument arrived in. + */ bool is_valid_json_static() const override { return true; } Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } @@ -681,7 +709,10 @@ class Item_func_json_insert: public Item_json_str_multipath bool fix_length_and_dec(THD *thd) override; String *val_str(String *) override; uint get_n_paths() const override { return arg_count/2; } - /* Reads its result back, as the other editing functions do. */ + /* + A document or nothing, for the reason given where + Item_func_json_array_append declares the same. + */ bool is_valid_json_static() const override { return true; } LEX_CSTRING func_name_cstring() const override { @@ -713,7 +744,10 @@ class Item_func_json_remove: public Item_json_str_multipath static LEX_CSTRING name= {STRING_WITH_LEN("json_remove") }; return name; } - /* Reads its result back, as the other editing functions do. */ + /* + A document or nothing, for the reason given where + Item_func_json_array_append declares the same. + */ bool is_valid_json_static() const override { return true; } protected: @@ -770,8 +804,12 @@ class Item_func_json_search: public Item_json_str_multipath String *val_str(String *) override; uint get_n_paths() const override { return arg_count > 4 ? arg_count - 4 : 0; } /* - A path is a JSON string, and every path this returns is built out - of pieces of a document that has just been read through. + A path is a JSON string, and every path returned here is built from + pieces of a document that has just been parsed. Several of them go + inside brackets, which needs a character set that can represent a + bracket - and a document in a character set that cannot is a + scalar, there being no way to write a container in it, so it holds + one value and yields one path. */ bool is_valid_json_static() const override { return true; } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e980b79f5c5d5..0c1e348877881 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7691,6 +7691,9 @@ SHOW_VAR status_vars[]= { {"Handler_tmp_write", (char*) offsetof(STATUS_VAR, ha_tmp_write_count), SHOW_LONG_STATUS}, {"Handler_update", (char*) offsetof(STATUS_VAR, ha_update_count), SHOW_LONG_STATUS}, {"Handler_write", (char*) offsetof(STATUS_VAR, ha_write_count), SHOW_LONG_STATUS}, +#ifndef DBUG_OFF + {"Json_scans", (char*) offsetof(STATUS_VAR, json_scans), SHOW_LONG_STATUS}, +#endif SHOW_FUNC_ENTRY("Key", &show_default_keycache), {"optimizer_join_prefixes_check_calls", (char*) offsetof(STATUS_VAR, optimizer_join_prefixes_check_calls), SHOW_LONG_STATUS}, {"Last_query_cost", (char*) offsetof(STATUS_VAR, last_query_cost), SHOW_DOUBLE_STATUS}, diff --git a/sql/sql_class.h b/sql/sql_class.h index 20b3385b0582d..84602e6679e93 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -997,6 +997,22 @@ typedef struct system_status_var ulong filesort_scan_count_; ulong filesort_pq_sorts_; ulong optimizer_join_prefixes_check_calls; +#ifndef DBUG_OFF + /* + How many times the JSON functions have started the parser, over whole + documents and over single spliced values alike. + + It counts work rather than results, and that is the point of it: the + JSON functions are meant to give the same answers however many times + they read a value, so the number of readings is the one thing a test + can watch that is allowed to change. Nothing else would show a + reading that stopped happening. + + Debug builds only. There is nothing here for a released server to + offer anybody. + */ + ulong json_scans; +#endif /* Features used */ ulong feature_custom_aggregate_functions; /* +1 when custom aggregate diff --git a/strings/json_lib.c b/strings/json_lib.c index 1a77082d9ff8c..5fbfbc4cbeeac 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -821,11 +821,38 @@ static json_state_handler json_actions[NR_JSON_STATES][NR_C_CLASSES]= +#ifndef DBUG_OFF +/* + Called whenever a value is about to be read, if anybody has asked to + be told. + + Every reading of a JSON value begins here - the calls that look like + they start one of their own (json_valid(), json_valid_engine(), + json_get_path_start(), and the normalizing build) all come through + this function - so this is the one place where a count of readings is + a count of readings rather than a list of the ways of asking for one. + Counting anywhere else means keeping such a list, and a list of that + kind is wrong the day something is added to it. + + Nobody is told unless somebody asks: the server sets this and nothing + else does, so the tools and the tests that link this code in are + unaffected. Debug builds only, and there is nothing here for a + released server to do. +*/ +void (*json_scan_start_hook)(void)= NULL; +#endif + + int json_scan_start(json_engine_t *je, CHARSET_INFO *i_cs, const uchar *str, const uchar *end) { static const uint32_t no_time_to_die= 0; +#ifndef DBUG_OFF + if (json_scan_start_hook) + json_scan_start_hook(); +#endif + json_string_setup(&je->s, i_cs, str, end); je->stack[0]= JST_DONE; je->stack_p= 0; From fef1d95319e609b786e3a093e332764c84c0a332 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Wed, 5 Aug 2026 04:54:16 -0400 Subject: [PATCH 13/30] Stop reading a column back to find out what was just put in it A column declared JSON carries a check constraint, json_valid(col), which reads every value stored into it to find out whether it is a document. Where the value arrived from something that has already attested it, and the store put those same characters down, that reading finds out nothing that was not already known. This leaves it undone. ## What has to hold Five things, all of them asked of the store as it happens: 1. **The Item attested `is_valid_json()`.** A column never does: the bytes in a row are whatever was put there, and a row can hold bytes no check ever saw - a replica applies rows unchecked, a session can turn the checks off, a tablespace can be imported. 2. **The store answered 0**, so every character mapped across with nothing put in its place. 3. **The field puts down the characters it is given.** `CHAR` and `BINARY` pad, `ENUM` and `SET` keep the member they matched rather than the text that matched it, geometry and the compressed types store something else - and every one of them answers 0 while doing it. `Field::is_character_preserving()` asks this of `real_type()` rather than by a virtual, so that a field type has to be NAMED to be trusted: `Field_geom` derives from `Field_blob` and answers `MYSQL_TYPE_GEOMETRY`, so naming the blobs does not name it. A switch can only refuse what says something, though, and the compressed classes say nothing there: they derive from a named type and go on answering its `real_type()`, saying what they are through `binlog_type()`, which nothing here consults. They are therefore asked about their compression ahead of the names, that being the one thing they do say. A virtual is safe THAT way round: it is asked in order to REFUSE, so a field type that has never heard of it is left where it started, and the inheritance that costs the trust hands none out. Without that question, j LONGTEXT COMPRESSED CHECK (JSON_VALID(j)) is marked as holding a document over a record that holds an image of one behind a header of its own. Nothing comes out wrong by it and nothing was going to, every reader going through `val_str()`, which unpacks; what goes wrong is that the check quietly stops running while the rule that left it unrun still says it has not. 4. **The characters mean the same at both ends of the store.** Two sets that are the same are copied and two that differ are converted, and both of those put the same characters down; a binary set at either end does neither. It relabels the bytes, so a ucs2 document stored into a `BLOB` arrives as bytes that read back as nothing at all while the mark says a document is there and the check is left unrun. `String_copier::conversion_keeps_characters()` sits on the class whose branch it reads: `well_formed_copy()` takes that branch by asking `conversion_copies_bytes()`, and the character question is the same call read the other way round, so the two cannot drift apart. It is deliberately not the plain `my_charset_same()` the two standing grants ask for: those are said once, about values not yet made and stores not yet done, while this one is said with the store behind it - and asking it here would take the mark away from a `utf8mb4` document going into a `latin1` column, which is the commonest shape there is, and the whole saving with it. 5. **Nothing has already gone wrong.** A value that did not arrive says nothing about what is in the column, and what is in it is whatever the store got as far as putting down before it gave up - which it can do while answering 0. The question is put to the session rather than to the store, an error raised while the value was being worked out being no less an error than one the store raised itself. Anything else clears the mark. Not marking costs a reading that was going to pass anyway; marking wrongly admits a value that is not a document, so every rule here is written to fail the first way. `Field::is_attestation_preserved()` asks the five, and the store site is written in terms of it rather than in terms of them. The item is asked before the field type is: every value a table is given reaches here, and one virtual answering no for everything that is not a document costs less than the two virtuals and the switch that work out what this field would have done with it. Where a value is stored by `save_val()` nothing is said about how the store went, so nothing can be attested - and nothing has to be taken back either, the marks having been cleared where that row image was begun and this route setting none. ## How long a mark lasts One row image. The marks are cleared as an image is BEGUN, rather than once something is finished with them, because a row whose writing ends before anything reads what was written leaves its marks standing: an update that finds nothing to change never reaches the reading, and neither does a view whose own check refuses the row. Clearing on the way in is reached by every row that can set a mark. Clearing on the way out is not. `TABLE::verify_constraints` clears on every way out of itself as well, which is what closes the two readings of `INSERT ... ON DUPLICATE KEY UPDATE`: the first is drained before `restore_record()` puts the old row's bytes back. Whatever replaces a row image wholesale leaves nothing that was said about the one before it still true, and the list of places that do it is open where the list of places that say something is closed. `restore_record()` is the one macro every such overwrite goes through, and it is `TABLE::restore_record_image()` behind that name - the memcpy it always was, followed by the drain - because naming the sites one at a time is how one of them gets missed, and a missed one is a check that quietly stops being run. An update over part of a period is where that shows. The pieces the period does not cover are written by putting the saved row back into the buffer the checks read, once per piece, and the check on the second piece runs over bytes the first piece's trigger attested. A stored generated column is written twice - once from the value it was given and again from its own expression - so the recompute clears what the first writing said. ## Which check is the column's own `json_valid(other_column)` types a column as JSON, and has done for as long as the type has existed, so the question the type handler asks is not the question this one needs. What was written into one column says nothing about another. The check must therefore be json_valid of the very field it guards, which is asked once at open and kept in `Virtual_column_info::json_valid_field_index`. A check wrapped in anything else, and a check belonging to the table rather than to a column, are never left unrun. ## Nothing is taken on faith The places that SET a mark are three, and that is a closed list. Each of them asks first whether any column of the table carries a check that could ever read one - the assignment in a trigger among them, so that `SET NEW.col` does not put a mark down on a table where nothing can pick it up. The places that must CLEAR one are every place that writes `record[0]` without going through those three, and that list is open: a site added later that forgets produces no wrong answer, no warning and no failing test, only a constraint that quietly stops being enforced. So a debug build runs the check anyway wherever one is about to be left unrun, and stops the server if it would not have passed. The reading is taken back off `Json_scans`, being the debug build's work rather than the server's: `Json_scans_unbilled` holds the count still across a whole check constraint and sits beside the helper that takes a single reading off, the one being an expression whose readings the caller cannot count and the other one call. Backing any one of the rules above out turns a value silently getting in into an assertion on the first row that reaches it. A debug build asks the gate once more at the field itself, because the closed list is what that reading rests on: a site added later that marks a table nobody asked about is a site whose marks nothing polices, and it would be exactly as quiet. Nothing reads a mark through the field. The one place that reads one has the bitmap in front of it and asks it by name, so `Field::is_valid_json()` would be the shortest question on a class that answers several better-scoped ones - true of one row image between a store and the next check, and false everywhere a reader would think to ask. There is no such accessor, and the comment where it would sit says why there is nothing there to reach for. ## The function the check is made of A column's check IS `json_valid()`, so the reading left unrun at the field boundary is the reading `Item_func_json_valid` makes - and that one was still being made where the value had already answered. It now returns true on `args[0]->is_valid_json()`. The two cannot disagree: a mark says the characters read back as a document, which is exactly what the walk would go and see. ## Asked only where it can be answered `Field::set_is_valid_json()` is an out-of-line call, and both `fill_record()` variants made it for every stored column of every row whatever the table looked like - a hundred calls a row on a hundred column table, discarded unless that column carries a check about itself. Whether any column does is worked out once, where the checks are, and kept in `TABLE::has_own_json_valid_check`. The bitmap is drained through `TABLE::clear_is_valid_json_marks()`, which asks that question first, exactly as the three store sites do. Where no column of a table carries such a check no bit can ever be set, so an unguarded drain walks a bitmap nothing will ever read - and these drains are not on the tables this is about. They are on every row written or updated in the instance. ## Measured Writing a document built out of scalars into a checked column falls from one reading per row to none, and editing one from two to one. A value written out as a literal still costs its reading: nothing has read it, so nothing can attest to it. ## Tests `func_json_check_skip` says what the check still refuses, and was recorded before any of this was written. `func_json_check_store` says the same for the stores that do not put down what they are given, and for the rows whose writing ends early; its answers were recorded against a released server. Its section 5 has both directions of the store that only looks like it kept the value - a wide document into a binary column and a binary one into a wide column - beside the conversion that really does write the characters again and keeps its mark. Its section 6 is the row split in two by an update over part of its period, which writes the pieces the period does not cover as new rows by putting the saved copy of the row back into the buffer the checks read: a mark standing from the value the update wrote would otherwise be believed of a value it never saw. A trigger writing the document column while the piece is being made is what gives the check something to catch, and the row the update itself writes stands beside them, refused where it is set to something that is not a document. `func_json_check_scan_count` counts the readings, which is the only place the saving shows, `json_valid()` asked directly among them: nothing for two values built out of scalars, one reading each for the same two written out as literals. It has the compressed columns too - both classes, a short value put down as it stands and a long one packed - counting a reading per row, beside the same two values in uncompressed columns of the same two types counting none, which is what makes those counts a fact about compressing rather than about the values or the widths. It has the two trigger shapes as well: a checked column written by a trigger and by nothing else, which costs no reading at all, and the same trigger over a table where no column carries a check, which is where a debug build stops if a mark is set that nothing can read. It counts the row split in two by an update over part of its period as well: three readings, one for the update itself and one for each of the two pieces. A piece written over an answer left standing from the piece before would count two. --- .../main/func_json_check_scan_count.result | 334 +++++++++++++++++ .../main/func_json_check_scan_count.test | 305 ++++++++++++++++ mysql-test/main/func_json_check_skip.result | 335 ++++++++++++++++++ mysql-test/main/func_json_check_skip.test | 301 ++++++++++++++++ mysql-test/main/func_json_check_store.result | 280 +++++++++++++++ mysql-test/main/func_json_check_store.test | 249 +++++++++++++ sql/field.cc | 111 ++++++ sql/field.h | 38 +- sql/item.cc | 2 + sql/item_jsonfunc.cc | 25 ++ sql/item_jsonfunc.h | 22 ++ sql/sql_base.cc | 35 +- sql/sql_insert.cc | 10 + sql/sql_select.cc | 3 + sql/sql_string.cc | 5 +- sql/sql_string.h | 32 ++ sql/sql_type_json.cc | 32 ++ sql/sql_type_json.h | 2 + sql/sql_update.cc | 9 + sql/table.cc | 106 +++++- sql/table.h | 64 ++++ sql/unireg.h | 2 +- 22 files changed, 2293 insertions(+), 9 deletions(-) create mode 100644 mysql-test/main/func_json_check_scan_count.result create mode 100644 mysql-test/main/func_json_check_scan_count.test create mode 100644 mysql-test/main/func_json_check_skip.result create mode 100644 mysql-test/main/func_json_check_skip.test create mode 100644 mysql-test/main/func_json_check_store.result create mode 100644 mysql-test/main/func_json_check_store.test diff --git a/mysql-test/main/func_json_check_scan_count.result b/mysql-test/main/func_json_check_scan_count.result new file mode 100644 index 0000000000000..37322d92ffdaf --- /dev/null +++ b/mysql-test/main/func_json_check_scan_count.result @@ -0,0 +1,334 @@ +# +# How many times a column's check constraint reads what is stored. +# +# A JSON column's check reads every value written into it to find +# out whether it is a document. Where the value came out of a JSON +# function, that has been answered already, and the reading finds +# out nothing. +# +# No query can see the difference - a value that is a document is +# stored either way - so nothing but Json_scans says whether the +# reading happened. The counts below are the record of it. A count +# that goes up is a reading that came back. +# +# A debug build reads values back twice over to check what was +# claimed about them - once for what the JSON functions claim, and +# once, wherever a check is left unrun, for what the column was +# marked with. Neither reading is counted: they are the debug +# build's work and not the server's, and counting them would move +# these numbers for reasons no query is responsible for. A count +# of 0 below therefore means the check did not read, not that the +# reading went uncounted. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, j VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +# +# ONE row, from a function that builds a document out of scalars. +# Building it reads nothing - there is nothing there to read - so +# every reading counted here is the check reading the value back. +# +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# TWO such rows. The check reads once per row, so this is twice +# the count above and nothing else. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)), (3, JSON_ARRAY(1, 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# TWO rows written out as literals. Nothing has read them, so the +# check is the only thing that can find out whether they are +# documents, and its reading is not work that can be taken away. +# Two rows, one reading each. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (4, '{"a":1}'), (5, '[1,2]'); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO rows, each put through JSON_COMPACT on the way in - so TWO +# calls, not one. Writing a document out compactly means reading +# it, and that reading is the function's own and stays. Each row +# therefore costs one reading for the function, plus whatever the +# check still costs. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (6, JSON_COMPACT('{"a":1}')), (7, JSON_COMPACT('[1,2]')); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO rows updated, so TWO calls to JSON_SET. Editing a document +# that came out of a column costs the function TWO readings of its +# own - one to find the place to edit and one to read its result +# back, the column having no say in how it was formatted. Four +# readings for the two rows, plus whatever the check still costs. +# +DELETE FROM t1; +INSERT INTO t1 VALUES (8, JSON_OBJECT('a', 1)), (9, JSON_OBJECT('b', 2)); +FLUSH STATUS; +UPDATE t1 SET j = JSON_SET(j, '$.c', 3); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# TWO rows read out of another table, one JSON_COMPACT call each, +# counting the same way as the two-call case above. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 SELECT id, JSON_COMPACT(j) FROM t2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# ONE row, written twice over, and read once per writing: what the +# first writing attested does not carry over to the second. +# Both values are built out of scalars, so neither costs a reading +# of its own and the count is the two checks alone. +# +DELETE FROM t1; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# ONE row into a column with no check at all, built out of scalars. +# Nothing reads it anywhere, and nothing is counted. +# +FLUSH STATUS; +INSERT INTO t2 VALUES (3, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +DROP TABLE t2; +# +# TWO rows through a trigger that edits the value on the way in. +# The trigger's JSON_INSERT is handed NEW.j, which is a column and +# says nothing about itself, so it costs the same TWO readings per +# row as the update above - four for the two rows - plus whatever +# the check still costs. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +DROP TABLE t1; +# +# ONE row, into a column whose check reads a DIFFERENT column. +# What was written into the one says nothing about the other, so +# the check keeps reading: one reading, and it stays. +# +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +DROP TABLE t1; +# +# TWO rows into a column whose check asks something else as well. +# Both values are built out of scalars, so the count is the checks +# alone - one per row, and both stay. +# +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 40)); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# TWO rows into a table with a table-level check, which is not a +# column's check whatever it asks. One reading per row, and both +# stay. +# +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# The same function asked directly, outside any check. +# +# A column's check IS json_valid(), so the reading left unrun +# above is the one json_valid() itself makes, and it is left +# unrun for the same reason wherever the value has answered. +# TWO values built out of scalars, neither of them read. +# +FLUSH STATUS; +SELECT JSON_VALID(JSON_OBJECT('a', 1)) AS built_object, +JSON_VALID(JSON_ARRAY(1, 2)) AS built_array; +built_object built_array +1 1 +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# The same two values written out, which nothing has answered +# for. One reading each, and both stay. +# +FLUSH STATUS; +SELECT JSON_VALID('{"a": 1}') AS written_object, +JSON_VALID('[1, 2]') AS written_array; +written_object written_array +1 1 +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO values that are not documents at all. Each is read, and +# each says so. +# +FLUSH STATUS; +SELECT JSON_VALID('not a document') AS text, +JSON_VALID('[1, 2') AS unfinished; +text unfinished +0 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO rows into a COMPRESSED column whose check asks nothing but +# whether it holds a document. A compressed store does not put +# down the characters it was handed: it puts down an image of them +# behind a header of its own. Nothing the value answered can be +# carried by bytes like that, so the check reads for itself - one +# reading per row, and both stay. +# +# The two values are built out of scalars and cost no reading of +# their own. They differ in length on purpose: the short one is +# put down as it stands behind the header and the long one is +# packed, and neither is the text that was passed. +# +CREATE TABLE t1 (j VARCHAR(1024) COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# The same two rows into a LONGTEXT COMPRESSED column, which is a +# different class of field storing the same way. Both compressed +# classes answer the same question about themselves, so neither +# has to be named for either to be refused. +# +CREATE TABLE t1 (j LONGTEXT COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# THE CONTROL: the same two values into the same two column types +# with the compression taken off. These put down what they were +# handed, so the check is left unrun and nothing is counted - which +# is what makes the counts above a fact about compressing rather +# than about the values, the widths or the types. +# +CREATE TABLE t1 (j VARCHAR(1024) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (j LONGTEXT CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +INSERT INTO t2 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +DROP TABLE t2; +# +# TWO rows whose checked column is written by a trigger and by +# nothing else. An assignment in a trigger is a store like any +# other and attests to what it put down, so the check is left +# unrun. The value is built out of scalars and reads nothing of +# its own, so nothing at all is counted. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +# +# The same trigger over a table where no column carries a check. +# Nothing here can ever be asked whether it holds a document, so +# the assignment does not stop to attest to what it stored - the +# restraint the other two store sites already showed. A debug +# build stops the server where a mark is set that nothing can +# read, which is what says the restraint is being shown. +# +CREATE TABLE t1 (id INT, j VARCHAR(64)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +# +# A row cut in two, where each piece is written over the row image +# the piece before it left behind. +# +# An update over part of a period writes back, as rows of their +# own, the pieces the update did not cover. Each piece is built by +# putting the row back as it stood and writing one end of the +# period over it, and the check runs on what that came to - so the +# answer a trigger gave for the piece before is an answer about +# bytes that are no longer there. +# +# THREE readings: one for the update itself, whose SET does not +# touch the checked column, and one for each of the two pieces. A +# piece written over an answer that was left standing would show up +# as a count of two. +# +CREATE TABLE t1 (id INT, s DATE, e DATE, j VARCHAR(64) CHECK (JSON_VALID(j)), +PERIOD FOR p(s, e)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_OBJECT('seen', NEW.id); +INSERT INTO t1 (id, s, e) VALUES (1, '2000-01-01', '2010-01-01'); +FLUSH STATUS; +UPDATE t1 FOR PORTION OF p FROM '2002-01-01' TO '2003-01-01' SET id = 2; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +SELECT id, s, e, j FROM t1 ORDER BY s; +id s e j +1 2000-01-01 2002-01-01 {"seen": 1} +2 2002-01-01 2003-01-01 {"seen": 1} +1 2003-01-01 2010-01-01 {"seen": 1} +DROP TABLE t1; diff --git a/mysql-test/main/func_json_check_scan_count.test b/mysql-test/main/func_json_check_scan_count.test new file mode 100644 index 0000000000000..0616abe425f7b --- /dev/null +++ b/mysql-test/main/func_json_check_scan_count.test @@ -0,0 +1,305 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a column's check constraint reads what is stored. +--echo # +--echo # A JSON column's check reads every value written into it to find +--echo # out whether it is a document. Where the value came out of a JSON +--echo # function, that has been answered already, and the reading finds +--echo # out nothing. +--echo # +--echo # No query can see the difference - a value that is a document is +--echo # stored either way - so nothing but Json_scans says whether the +--echo # reading happened. The counts below are the record of it. A count +--echo # that goes up is a reading that came back. +--echo # +--echo # A debug build reads values back twice over to check what was +--echo # claimed about them - once for what the JSON functions claim, and +--echo # once, wherever a check is left unrun, for what the column was +--echo # marked with. Neither reading is counted: they are the debug +--echo # build's work and not the server's, and counting them would move +--echo # these numbers for reasons no query is responsible for. A count +--echo # of 0 below therefore means the check did not read, not that the +--echo # reading went uncounted. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, j VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +--echo # +--echo # ONE row, from a function that builds a document out of scalars. +--echo # Building it reads nothing - there is nothing there to read - so +--echo # every reading counted here is the check reading the value back. +--echo # +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO such rows. The check reads once per row, so this is twice +--echo # the count above and nothing else. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)), (3, JSON_ARRAY(1, 2)); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows written out as literals. Nothing has read them, so the +--echo # check is the only thing that can find out whether they are +--echo # documents, and its reading is not work that can be taken away. +--echo # Two rows, one reading each. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (4, '{"a":1}'), (5, '[1,2]'); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows, each put through JSON_COMPACT on the way in - so TWO +--echo # calls, not one. Writing a document out compactly means reading +--echo # it, and that reading is the function's own and stays. Each row +--echo # therefore costs one reading for the function, plus whatever the +--echo # check still costs. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (6, JSON_COMPACT('{"a":1}')), (7, JSON_COMPACT('[1,2]')); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows updated, so TWO calls to JSON_SET. Editing a document +--echo # that came out of a column costs the function TWO readings of its +--echo # own - one to find the place to edit and one to read its result +--echo # back, the column having no say in how it was formatted. Four +--echo # readings for the two rows, plus whatever the check still costs. +--echo # +DELETE FROM t1; +INSERT INTO t1 VALUES (8, JSON_OBJECT('a', 1)), (9, JSON_OBJECT('b', 2)); +FLUSH STATUS; +UPDATE t1 SET j = JSON_SET(j, '$.c', 3); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows read out of another table, one JSON_COMPACT call each, +--echo # counting the same way as the two-call case above. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 SELECT id, JSON_COMPACT(j) FROM t2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # ONE row, written twice over, and read once per writing: what the +--echo # first writing attested does not carry over to the second. +--echo # Both values are built out of scalars, so neither costs a reading +--echo # of its own and the count is the two checks alone. +--echo # +DELETE FROM t1; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # ONE row into a column with no check at all, built out of scalars. +--echo # Nothing reads it anywhere, and nothing is counted. +--echo # +FLUSH STATUS; +INSERT INTO t2 VALUES (3, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # TWO rows through a trigger that edits the value on the way in. +--echo # The trigger's JSON_INSERT is handed NEW.j, which is a column and +--echo # says nothing about itself, so it costs the same TWO readings per +--echo # row as the update above - four for the two rows - plus whatever +--echo # the check still costs. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # ONE row, into a column whose check reads a DIFFERENT column. +--echo # What was written into the one says nothing about the other, so +--echo # the check keeps reading: one reading, and it stays. +--echo # +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # TWO rows into a column whose check asks something else as well. +--echo # Both values are built out of scalars, so the count is the checks +--echo # alone - one per row, and both stay. +--echo # +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 40)); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # TWO rows into a table with a table-level check, which is not a +--echo # column's check whatever it asks. One reading per row, and both +--echo # stay. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # The same function asked directly, outside any check. +--echo # +--echo # A column's check IS json_valid(), so the reading left unrun +--echo # above is the one json_valid() itself makes, and it is left +--echo # unrun for the same reason wherever the value has answered. +--echo # TWO values built out of scalars, neither of them read. +--echo # +FLUSH STATUS; +SELECT JSON_VALID(JSON_OBJECT('a', 1)) AS built_object, + JSON_VALID(JSON_ARRAY(1, 2)) AS built_array; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same two values written out, which nothing has answered +--echo # for. One reading each, and both stay. +--echo # +FLUSH STATUS; +SELECT JSON_VALID('{"a": 1}') AS written_object, + JSON_VALID('[1, 2]') AS written_array; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO values that are not documents at all. Each is read, and +--echo # each says so. +--echo # +FLUSH STATUS; +SELECT JSON_VALID('not a document') AS text, + JSON_VALID('[1, 2') AS unfinished; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows into a COMPRESSED column whose check asks nothing but +--echo # whether it holds a document. A compressed store does not put +--echo # down the characters it was handed: it puts down an image of them +--echo # behind a header of its own. Nothing the value answered can be +--echo # carried by bytes like that, so the check reads for itself - one +--echo # reading per row, and both stay. +--echo # +--echo # The two values are built out of scalars and cost no reading of +--echo # their own. They differ in length on purpose: the short one is +--echo # put down as it stands behind the header and the long one is +--echo # packed, and neither is the text that was passed. +--echo # +CREATE TABLE t1 (j VARCHAR(1024) COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # The same two rows into a LONGTEXT COMPRESSED column, which is a +--echo # different class of field storing the same way. Both compressed +--echo # classes answer the same question about themselves, so neither +--echo # has to be named for either to be refused. +--echo # +CREATE TABLE t1 (j LONGTEXT COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # THE CONTROL: the same two values into the same two column types +--echo # with the compression taken off. These put down what they were +--echo # handed, so the check is left unrun and nothing is counted - which +--echo # is what makes the counts above a fact about compressing rather +--echo # than about the values, the widths or the types. +--echo # +CREATE TABLE t1 (j VARCHAR(1024) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (j LONGTEXT CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +INSERT INTO t2 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # TWO rows whose checked column is written by a trigger and by +--echo # nothing else. An assignment in a trigger is a store like any +--echo # other and attests to what it put down, so the check is left +--echo # unrun. The value is built out of scalars and reads nothing of +--echo # its own, so nothing at all is counted. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # The same trigger over a table where no column carries a check. +--echo # Nothing here can ever be asked whether it holds a document, so +--echo # the assignment does not stop to attest to what it stored - the +--echo # restraint the other two store sites already showed. A debug +--echo # build stops the server where a mark is set that nothing can +--echo # read, which is what says the restraint is being shown. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # A row cut in two, where each piece is written over the row image +--echo # the piece before it left behind. +--echo # +--echo # An update over part of a period writes back, as rows of their +--echo # own, the pieces the update did not cover. Each piece is built by +--echo # putting the row back as it stood and writing one end of the +--echo # period over it, and the check runs on what that came to - so the +--echo # answer a trigger gave for the piece before is an answer about +--echo # bytes that are no longer there. +--echo # +--echo # THREE readings: one for the update itself, whose SET does not +--echo # touch the checked column, and one for each of the two pieces. A +--echo # piece written over an answer that was left standing would show up +--echo # as a count of two. +--echo # +CREATE TABLE t1 (id INT, s DATE, e DATE, j VARCHAR(64) CHECK (JSON_VALID(j)), + PERIOD FOR p(s, e)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_OBJECT('seen', NEW.id); +INSERT INTO t1 (id, s, e) VALUES (1, '2000-01-01', '2010-01-01'); +FLUSH STATUS; +UPDATE t1 FOR PORTION OF p FROM '2002-01-01' TO '2003-01-01' SET id = 2; +SHOW STATUS LIKE 'Json_scans'; +SELECT id, s, e, j FROM t1 ORDER BY s; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_check_skip.result b/mysql-test/main/func_json_check_skip.result new file mode 100644 index 0000000000000..95b587a4e71d9 --- /dev/null +++ b/mysql-test/main/func_json_check_skip.result @@ -0,0 +1,335 @@ +# +# What a JSON column's check constraint keeps out. +# +# A column declared JSON carries a check constraint that reads every +# value stored into it and refuses the ones that are not documents. +# Where the value came from something that has already attested +# it, that reading finds out nothing that was not already known, and +# can be left undone. +# +# Everything below is a case where it CANNOT be left undone, and the +# answers here are what the column does about each of them. Nothing +# in this file is supposed to change: it says what is refused, and a +# value that stops being refused is a value that got in. +# +SET NAMES utf8mb4; +# +# 1. A document that came from a JSON function goes in and stays put. +# +CREATE TABLE t1 (id INT, j JSON); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1, 'b', 2)), +(2, JSON_ARRAY(1, 2, 3)); +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "b": 2} +2 [1, 2, 3] +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; +id still_a_document +1 1 +2 1 +# the same through every other way a row is written +INSERT INTO t1 SELECT id + 10, JSON_INSERT(j, '$.c', 3) FROM t1 ORDER BY id; +UPDATE t1 SET j = JSON_SET(j, '$.d', 4) WHERE id = 1; +REPLACE INTO t1 VALUES (2, JSON_ARRAY(9, 8)); +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; +id still_a_document +1 1 +2 1 +2 1 +11 1 +12 1 +DROP TABLE t1; +# +# 2. Text that is not a document is refused, whoever wrote it. +# +CREATE TABLE t1 (id INT, j JSON); +INSERT INTO t1 VALUES (1, 'not a document'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, '{"a":1'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, CONCAT('{"a":', '1')); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +UPDATE t1 SET j = 'not a document'; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1} +DROP TABLE t1; +# +# 3. A document that does not fit is cut, and what is left of it is +# read like anything else. The value that arrives came from a JSON +# function and was a document when it set out; what the column holds +# is not, and that is what the check has to see. +# +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2, 3, 4567)); +ERROR 23000: CONSTRAINT `t1.v` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# short enough to arrive whole +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT v FROM t1; +v +[1, 2] +DROP TABLE t1; +# a cut that happens to leave a document behind is accepted, as it +# always has been - the text after it is what went missing +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES ('[1,2,3,45],6789'); +Warnings: +Warning 1265 Data truncated for column 'v' at row 1 +SELECT v FROM t1; +v +[1,2,3,45] +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; +# +# 4. A character set that cannot encode a document does not get one. +# The brackets are not in swe7 at all, so they arrive as question +# marks and what is stored is not a document. +# +SET @@sql_mode = ''; +CREATE TABLE t1 (v VARCHAR(64) CHARACTER SET swe7 CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +ERROR 23000: CONSTRAINT `t1.v` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; +# +# 5. A row that is written twice is read twice. +# +# ON DUPLICATE KEY UPDATE writes the new row, reads it, then puts +# the old row's columns back and writes the update over the top. +# What the first writing attested is gone by then, and the +# second writing has to attest to itself. +# +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) +ON DUPLICATE KEY UPDATE j = 'not a document'; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1} +# and the update that IS a document goes through +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"c": 3} +# the insert half refused, with a second row present to prove the +# statement reached it +INSERT INTO t1 VALUES (1, JSON_OBJECT('d', 4)), (2, 'not a document') +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('e', 5); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"e": 5} +DROP TABLE t1; +# +# 6. A check that reads a DIFFERENT column is not this column's +# check, however much it looks like one. +# +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +# a is given a document, b is not; the check reads b and refuses +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), 'not a document'); +ERROR 23000: CONSTRAINT `t1.a` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# b a document, a not - accepted, the check having nothing to say +# about a +INSERT INTO t1 VALUES ('not a document', JSON_OBJECT('b', 2)); +SELECT a, b FROM t1; +a b +not a document {"b": 2} +DROP TABLE t1; +# +# 7. A check that asks anything besides whether the value is a +# document keeps asking it. +# +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 12)); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (JSON_OBJECT('aaaaaaaaaa', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT j FROM t1; +j +{"a": 1} +DROP TABLE t1; +CREATE TABLE t1 (id INT, j VARCHAR(64), +CONSTRAINT ck CHECK (id > 0 AND JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (0, JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `ck` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, 'not a document'); +ERROR 23000: CONSTRAINT `ck` failed for `test`.`t1` +SELECT id, j FROM t1; +id j +1 {"a": 1} +DROP TABLE t1; +# a table-level check that asks only about the document is still a +# table-level check +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (2, 'not a document'); +ERROR 23000: CONSTRAINT `ck` failed for `test`.`t1` +SELECT id, j FROM t1; +id j +1 {"a": 1} +DROP TABLE t1; +# +# 8. A trigger writing the column is a writer like any other. +# +CREATE TABLE t1 (id INT, j JSON); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "seen": 1} +DROP TRIGGER tr1; +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = 'not a document'; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "seen": 1} +DROP TRIGGER tr1; +# written twice in one trigger, the last writing being the one that +# counts +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +BEGIN +SET NEW.j = JSON_OBJECT('first', 1); +SET NEW.j = 'not a document'; +END| +INSERT INTO t1 VALUES (3, JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "seen": 1} +DROP TRIGGER tr1; +DROP TABLE t1; +# +# 9. Switching the checks off leaves bytes behind that the column +# would not have taken, and switching them back on does not go +# looking for them. What it does do is refuse the next bad value. +# +CREATE TABLE t1 (id INT, j JSON); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 'not a document'); +SET @@check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +id j reads_as_a_document +1 not a document 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (3, 'also not a document'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +id reads_as_a_document +1 0 +2 1 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# a copying ALTER reads every row again and finds the one that was +# let through +ALTER TABLE t1 FORCE, ALGORITHM=COPY; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +# and ignoring drops it rather than keeping it +ALTER IGNORE TABLE t1 FORCE, ALGORITHM=COPY; +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +id reads_as_a_document +2 1 +DROP TABLE t1; +# +# 10. A versioned table writes a history row that is not read, and a +# current row that is. +# +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))) +WITH SYSTEM VERSIONING; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +UPDATE t1 SET j = JSON_OBJECT('b', 2) WHERE id = 1; +UPDATE t1 SET j = 'not a document' WHERE id = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"b": 2} +SELECT j, JSON_VALID(j) AS reads_as_a_document +FROM t1 FOR SYSTEM_TIME ALL ORDER BY j; +j reads_as_a_document +{"a": 1} 1 +{"b": 2} 1 +DROP TABLE t1; +# +# 11. Every other way of writing rows. +# +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); +# INSERT ... SELECT stops at the row that is not a document, the +# rows before it having already been written +INSERT INTO t1 SELECT id, txt FROM t2 ORDER BY id; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +DELETE FROM t1; +INSERT INTO t1 SELECT id, JSON_COMPACT(txt) FROM t2 WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +# a plain UPDATE +UPDATE t1 SET j = 'not a document' WHERE id = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1 SET j = JSON_SET(j, '$.b', 2) WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "b": 2} +# an update across two tables, both of them present in t1 this time +INSERT INTO t1 VALUES (2, JSON_OBJECT('x', 9)); +UPDATE t1, t2 SET t1.j = t2.txt WHERE t1.id = t2.id AND t2.id = 2; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1, t2 SET t1.j = 'not a document' WHERE t1.id = t2.id; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1, t2 SET t1.j = JSON_COMPACT(t2.txt) WHERE t1.id = t2.id AND t2.id = 1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +2 {"x": 9} +DROP TABLE t1, t2; +# +# 12. Rows read out of a file are text and are read like text. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); +SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt'; +Warnings: +Warning 1287 ' INTO FROM...' instead +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt' INTO TABLE t1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +1 +# the same file with only the document in it +DELETE FROM t2 WHERE id = 2; +SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt'; +Warnings: +Warning 1287 ' INTO FROM...' instead +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt' INTO TABLE t1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +1 {"a":1} +DROP TABLE t1, t2; diff --git a/mysql-test/main/func_json_check_skip.test b/mysql-test/main/func_json_check_skip.test new file mode 100644 index 0000000000000..3782dd970aa35 --- /dev/null +++ b/mysql-test/main/func_json_check_skip.test @@ -0,0 +1,301 @@ +--echo # +--echo # What a JSON column's check constraint keeps out. +--echo # +--echo # A column declared JSON carries a check constraint that reads every +--echo # value stored into it and refuses the ones that are not documents. +--echo # Where the value came from something that has already attested +--echo # it, that reading finds out nothing that was not already known, and +--echo # can be left undone. +--echo # +--echo # Everything below is a case where it CANNOT be left undone, and the +--echo # answers here are what the column does about each of them. Nothing +--echo # in this file is supposed to change: it says what is refused, and a +--echo # value that stops being refused is a value that got in. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A document that came from a JSON function goes in and stays put. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1, 'b', 2)), + (2, JSON_ARRAY(1, 2, 3)); +SELECT id, j FROM t1 ORDER BY id; +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; + +--echo # the same through every other way a row is written +INSERT INTO t1 SELECT id + 10, JSON_INSERT(j, '$.c', 3) FROM t1 ORDER BY id; +UPDATE t1 SET j = JSON_SET(j, '$.d', 4) WHERE id = 1; +REPLACE INTO t1 VALUES (2, JSON_ARRAY(9, 8)); +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 2. Text that is not a document is refused, whoever wrote it. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, 'not a document'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, '{"a":1'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, CONCAT('{"a":', '1')); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = 'not a document'; +SELECT id, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 3. A document that does not fit is cut, and what is left of it is +--echo # read like anything else. The value that arrives came from a JSON +--echo # function and was a document when it set out; what the column holds +--echo # is not, and that is what the check has to see. +--echo # + +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; + +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2, 3, 4567)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # short enough to arrive whole +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT v FROM t1; +DROP TABLE t1; + +--echo # a cut that happens to leave a document behind is accepted, as it +--echo # always has been - the text after it is what went missing +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES ('[1,2,3,45],6789'); +SELECT v FROM t1; +DROP TABLE t1; + +SET @@sql_mode = @sql_mode_saved; + +--echo # +--echo # 4. A character set that cannot encode a document does not get one. +--echo # The brackets are not in swe7 at all, so they arrive as question +--echo # marks and what is stored is not a document. +--echo # + +SET @@sql_mode = ''; +CREATE TABLE t1 (v VARCHAR(64) CHARACTER SET swe7 CHECK (JSON_VALID(v))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; + +--echo # +--echo # 5. A row that is written twice is read twice. +--echo # +--echo # ON DUPLICATE KEY UPDATE writes the new row, reads it, then puts +--echo # the old row's columns back and writes the update over the top. +--echo # What the first writing attested is gone by then, and the +--echo # second writing has to attest to itself. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) + ON DUPLICATE KEY UPDATE j = 'not a document'; +SELECT id, j FROM t1 ORDER BY id; + +--echo # and the update that IS a document goes through +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SELECT id, j FROM t1 ORDER BY id; + +--echo # the insert half refused, with a second row present to prove the +--echo # statement reached it +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, JSON_OBJECT('d', 4)), (2, 'not a document') + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('e', 5); +SELECT id, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 6. A check that reads a DIFFERENT column is not this column's +--echo # check, however much it looks like one. +--echo # + +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +--echo # a is given a document, b is not; the check reads b and refuses +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), 'not a document'); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # b a document, a not - accepted, the check having nothing to say +--echo # about a +INSERT INTO t1 VALUES ('not a document', JSON_OBJECT('b', 2)); +SELECT a, b FROM t1; +DROP TABLE t1; + +--echo # +--echo # 7. A check that asks anything besides whether the value is a +--echo # document keeps asking it. +--echo # + +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 12)); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT('aaaaaaaaaa', 1)); +SELECT j FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 (id INT, j VARCHAR(64), + CONSTRAINT ck CHECK (id > 0 AND JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (0, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, 'not a document'); +SELECT id, j FROM t1; +DROP TABLE t1; + +--echo # a table-level check that asks only about the document is still a +--echo # table-level check +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (2, 'not a document'); +SELECT id, j FROM t1; +DROP TABLE t1; + +--echo # +--echo # 8. A trigger writing the column is a writer like any other. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +DROP TRIGGER tr1; + +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = 'not a document'; +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +DROP TRIGGER tr1; + +--echo # written twice in one trigger, the last writing being the one that +--echo # counts +DELIMITER |; +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +BEGIN + SET NEW.j = JSON_OBJECT('first', 1); + SET NEW.j = 'not a document'; +END| +DELIMITER ;| +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +DROP TRIGGER tr1; +DROP TABLE t1; + +--echo # +--echo # 9. Switching the checks off leaves bytes behind that the column +--echo # would not have taken, and switching them back on does not go +--echo # looking for them. What it does do is refuse the next bad value. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 'not a document'); +SET @@check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, 'also not a document'); +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; + +--echo # a copying ALTER reads every row again and finds the one that was +--echo # let through +--error ER_CONSTRAINT_FAILED +ALTER TABLE t1 FORCE, ALGORITHM=COPY; +--echo # and ignoring drops it rather than keeping it +ALTER IGNORE TABLE t1 FORCE, ALGORITHM=COPY; +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 10. A versioned table writes a history row that is not read, and a +--echo # current row that is. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))) + WITH SYSTEM VERSIONING; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +UPDATE t1 SET j = JSON_OBJECT('b', 2) WHERE id = 1; +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = 'not a document' WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +SELECT j, JSON_VALID(j) AS reads_as_a_document + FROM t1 FOR SYSTEM_TIME ALL ORDER BY j; +DROP TABLE t1; + +--echo # +--echo # 11. Every other way of writing rows. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); + +--echo # INSERT ... SELECT stops at the row that is not a document, the +--echo # rows before it having already been written +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 SELECT id, txt FROM t2 ORDER BY id; +SELECT id, j FROM t1 ORDER BY id; +DELETE FROM t1; +INSERT INTO t1 SELECT id, JSON_COMPACT(txt) FROM t2 WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; + +--echo # a plain UPDATE +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = 'not a document' WHERE id = 1; +UPDATE t1 SET j = JSON_SET(j, '$.b', 2) WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; + +--echo # an update across two tables, both of them present in t1 this time +INSERT INTO t1 VALUES (2, JSON_OBJECT('x', 9)); +--error ER_CONSTRAINT_FAILED +UPDATE t1, t2 SET t1.j = t2.txt WHERE t1.id = t2.id AND t2.id = 2; +--error ER_CONSTRAINT_FAILED +UPDATE t1, t2 SET t1.j = 'not a document' WHERE t1.id = t2.id; +UPDATE t1, t2 SET t1.j = JSON_COMPACT(t2.txt) WHERE t1.id = t2.id AND t2.id = 1; +SELECT id, j FROM t1 ORDER BY id; +DROP TABLE t1, t2; + +--echo # +--echo # 12. Rows read out of a file are text and are read like text. +--echo # + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); +--let $DATA = $MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE '$DATA' +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error ER_CONSTRAINT_FAILED +--eval LOAD DATA INFILE '$DATA' INTO TABLE t1 +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--remove_file $DATA + +--echo # the same file with only the document in it +DELETE FROM t2 WHERE id = 2; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE '$DATA' +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval LOAD DATA INFILE '$DATA' INTO TABLE t1 +SELECT id, j FROM t1 ORDER BY id; +--remove_file $DATA +DROP TABLE t1, t2; diff --git a/mysql-test/main/func_json_check_store.result b/mysql-test/main/func_json_check_store.result new file mode 100644 index 0000000000000..8e00c710bb175 --- /dev/null +++ b/mysql-test/main/func_json_check_store.result @@ -0,0 +1,280 @@ +# +# What a check constraint has to read for itself, whatever wrote the +# value. +# +# A value that arrived from something that attested to it is only +# the value the check will read if TWO things hold: the store put +# those same characters in the row and nothing has written over them +# since. Every case below is one where one of the two fails, and +# the check has to read the row for itself. +# +# These answers are what a released server gives. A value getting +# in here that reads back as not a document is a value that got past +# the constraint declaring it must be one. +# +SET NAMES utf8mb4; +# +# 1. Stores that do not put down the characters they were given. +# +# BINARY pads with NUL, and the padding is part of what is stored +CREATE TABLE t1 (b BINARY(20) CHECK (JSON_VALID(b))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `t1.b` failed for `test`.`t1` +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +ERROR 23000: CONSTRAINT `t1.b` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# CHAR pads too, with spaces, which are not read back +CREATE TABLE t1 (c CHAR(20) CHECK (JSON_VALID(c))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES ('not a document'); +ERROR 23000: CONSTRAINT `t1.c` failed for `test`.`t1` +SELECT c, JSON_VALID(c) AS reads_as_a_document FROM t1; +c reads_as_a_document +{"a": 1} 1 +DROP TABLE t1; +# ENUM keeps the member it matched, not the text it was given +CREATE TABLE t1 (e ENUM('True','banana') CHECK (JSON_VALID(e))); +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +ERROR 23000: CONSTRAINT `t1.e` failed for `test`.`t1` +INSERT INTO t1 VALUES (JSON_COMPACT('2')); +ERROR 23000: CONSTRAINT `t1.e` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# SET the same +CREATE TABLE t1 (s SET('True','1') CHECK (JSON_VALID(s))); +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +ERROR 23000: CONSTRAINT `t1.s` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# a column that DOES put down what it was given takes the document +CREATE TABLE t1 (v VARCHAR(64) CHECK (JSON_VALID(v)), +t LONGTEXT CHECK (JSON_VALID(t))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_ARRAY(1, 2)); +SELECT v, t FROM t1; +v t +{"a": 1} [1, 2] +DROP TABLE t1; +# +# 2. A stored generated column is written twice: once from what was +# given, and again from its own expression, which is what stays. +# +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; +CREATE TABLE t1 (a VARCHAR(20), j LONGTEXT AS (a) STORED CHECK (JSON_VALID(j))); +INSERT INTO t1 (a, j) VALUES ('not a document', JSON_OBJECT('k', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# and the value the expression does produce is taken +INSERT INTO t1 (a, j) VALUES ('{"k":1}', JSON_OBJECT('k', 1)); +Warnings: +Warning 1906 The value specified for generated column 'j' in table 't1' has been ignored +SELECT a, j FROM t1; +a j +{"k":1} {"k":1} +# the same on update +UPDATE t1 SET a = 'not a document'; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT a, j FROM t1; +a j +{"k":1} {"k":1} +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; +# +# 3. A row whose writing ends before the check is reached leaves +# nothing behind for the NEXT row to be read with. +# +# an update where one row comes out exactly as it went in +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('k', 1)); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (2, 0, 'not a document'); +SET @@check_constraint_checks = ON; +CREATE TRIGGER t1t BEFORE UPDATE ON t1 FOR EACH ROW +BEGIN +IF OLD.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +UPDATE t1 SET b = b + (id = 2); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, b, j FROM t1 ORDER BY id; +id b j +1 1 {"k": 1} +2 0 not a document +DROP TABLE t1; +# an insert through a view whose own check refuses the row first +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, +j VARCHAR(64) DEFAULT 'not a document' + CHECK (JSON_VALID(j))); +CREATE VIEW v1 AS SELECT id, b, j FROM t1 WHERE b > 0 WITH CHECK OPTION; +CREATE TRIGGER t1t BEFORE INSERT ON t1 FOR EACH ROW +BEGIN +IF NEW.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +INSERT IGNORE INTO v1 (id, b) VALUES (1, 0), (2, 1); +Warnings: +Warning 1369 CHECK OPTION failed `test`.`v1` +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, b, j FROM t1 ORDER BY id; +id b j +DROP VIEW v1; +DROP TABLE t1; +# +# 4. A row written twice over, where the SECOND writing does not +# touch the column at all. +# +# The first writing puts a document there and attests to it; the +# old row's bytes are then put back, and the update clause leaves +# the column alone. What the check reads is the old bytes, which +# nothing has attested. +# +CREATE TABLE t1 (id INT PRIMARY KEY, a INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 0, 'not a document'); +SET @@check_constraint_checks = ON; +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('x', 1)) +ON DUPLICATE KEY UPDATE a = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, a, j FROM t1 ORDER BY id; +id a j +1 0 not a document +# and where the update clause DOES write the column, it is read the +# ordinary way +INSERT INTO t1 VALUES (1, 2, JSON_OBJECT('x', 1)) +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('y', 2); +SELECT id, a, j FROM t1 ORDER BY id; +id a j +1 0 {"y": 2} +DROP TABLE t1; +# +# 5. Stores that put down every byte they were given and call them +# by another name. +# +# A binary column takes the bytes of a document written in some +# other character set exactly as they stand, and a column in some +# other set takes the bytes of a binary one the same way. Neither +# writes the characters again, so what the row holds is not the +# characters that were attested, and the check has to read it. +# +# a document written in a wide set, into a binary column +CREATE TABLE t1 (j BLOB CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (CONVERT(JSON_OBJECT('a', 1) USING ucs2)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (CONVERT(JSON_ARRAY(1, 2) USING utf16)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# the same documents in a set the column can take go in +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +j +{"a": 1} +[1, 2] +DROP TABLE t1; +# a binary document, into a column written in a wide set +CREATE TABLE t1 (j TEXT CHARACTER SET ucs2 CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (JSON_ARRAY(_binary'ab')); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (JSON_OBJECT(_binary'k', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# a store that does write the characters again, in the column's +# own set, keeps the document +CREATE TABLE t1 (j TEXT CHARACTER SET latin1 CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +j +{"a": 1} +[1, 2] +DROP TABLE t1; +# +# 6. A row split in two by an update over part of its period. Such +# an update writes the pieces of the old row that the period does +# not cover as new rows, and it writes them by putting the saved +# copy of the row back into the buffer the checks read. Anything +# said about what stood there before is about the value the update +# wrote, not the one just restored, so it goes with the row it was +# about. +# +# A trigger is what makes the difference visible: it writes the +# document column while the piece is being made, so the check has +# something to catch. +# +CREATE TABLE t1 (id INT, +j VARCHAR(64) CHECK (JSON_VALID(j)), +s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +INSERT INTO t1 (id, j, s, e) +VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), +(2, '[1,2]', '1999-01-01', '2018-12-12'); +UPDATE t1 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t1; +id j ty s e +1 {"a":1} OBJECT 1999-01-01 2000-01-01 +1 {"a":1} OBJECT 2018-01-01 2018-12-12 +2 [1,2] ARRAY 1999-01-01 2000-01-01 +2 [1,2] ARRAY 2018-01-01 2018-12-12 +6 {"a":1} OBJECT 2000-01-01 2018-01-01 +7 [1,2] ARRAY 2000-01-01 2018-01-01 +# the same split with the pieces given documents of their own +CREATE TABLE t2 (id INT, +j VARCHAR(64) CHECK (JSON_VALID(j)), +s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t2_bi BEFORE INSERT ON t2 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t2 (id, j, s, e) +VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), +(2, '{"b":2}', '1999-01-01', '2018-12-12'); +UPDATE t2 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t2; +id j ty s e +1 {"a": 1, "piece": 1} OBJECT 1999-01-01 2000-01-01 +1 {"a": 1, "piece": 1} OBJECT 2018-01-01 2018-12-12 +2 {"b": 2, "piece": 2} OBJECT 1999-01-01 2000-01-01 +2 {"b": 2, "piece": 2} OBJECT 2018-01-01 2018-12-12 +6 {"a": 1, "piece": 1} OBJECT 2000-01-01 2018-01-01 +7 {"b": 2, "piece": 2} OBJECT 2000-01-01 2018-01-01 +# the row the update itself writes is checked as any other row is, +# and it is written after the pieces have been, so what the pieces +# left behind must not be believed of it: a portion set to +# something that is not a document is refused +CREATE TABLE t3 (id INT, +j VARCHAR(64) CHECK (JSON_VALID(j)), +s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t3_bi BEFORE INSERT ON t3 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t3 (id, j, s, e) +VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), +(2, '{"b":2}', '1999-01-01', '2018-12-12'); +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= 'not a document'; +ERROR 23000: CONSTRAINT `t3.j` failed for `test`.`t3` +SELECT id, j, s, e FROM t3; +id j s e +1 {"a": 1, "piece": 1} 1999-01-01 2018-12-12 +2 {"b": 2, "piece": 2} 1999-01-01 2018-12-12 +# and one set to a document goes in +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= JSON_OBJECT('portion', id); +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t3; +id j ty s e +1 {"a": 1, "piece": 1} OBJECT 1999-01-01 2000-01-01 +1 {"a": 1, "piece": 1} OBJECT 2018-01-01 2018-12-12 +1 {"portion": 1} OBJECT 2000-01-01 2018-01-01 +2 {"b": 2, "piece": 2} OBJECT 1999-01-01 2000-01-01 +2 {"b": 2, "piece": 2} OBJECT 2018-01-01 2018-12-12 +2 {"portion": 2} OBJECT 2000-01-01 2018-01-01 +DROP TABLE t3, t2, t1; diff --git a/mysql-test/main/func_json_check_store.test b/mysql-test/main/func_json_check_store.test new file mode 100644 index 0000000000000..6449a1affcf98 --- /dev/null +++ b/mysql-test/main/func_json_check_store.test @@ -0,0 +1,249 @@ +--echo # +--echo # What a check constraint has to read for itself, whatever wrote the +--echo # value. +--echo # +--echo # A value that arrived from something that attested to it is only +--echo # the value the check will read if TWO things hold: the store put +--echo # those same characters in the row and nothing has written over them +--echo # since. Every case below is one where one of the two fails, and +--echo # the check has to read the row for itself. +--echo # +--echo # These answers are what a released server gives. A value getting +--echo # in here that reads back as not a document is a value that got past +--echo # the constraint declaring it must be one. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Stores that do not put down the characters they were given. +--echo # + +--echo # BINARY pads with NUL, and the padding is part of what is stored +CREATE TABLE t1 (b BINARY(20) CHECK (JSON_VALID(b))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # CHAR pads too, with spaces, which are not read back +CREATE TABLE t1 (c CHAR(20) CHECK (JSON_VALID(c))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES ('not a document'); +SELECT c, JSON_VALID(c) AS reads_as_a_document FROM t1; +DROP TABLE t1; + +--echo # ENUM keeps the member it matched, not the text it was given +CREATE TABLE t1 (e ENUM('True','banana') CHECK (JSON_VALID(e))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_COMPACT('2')); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # SET the same +CREATE TABLE t1 (s SET('True','1') CHECK (JSON_VALID(s))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # a column that DOES put down what it was given takes the document +CREATE TABLE t1 (v VARCHAR(64) CHECK (JSON_VALID(v)), + t LONGTEXT CHECK (JSON_VALID(t))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_ARRAY(1, 2)); +SELECT v, t FROM t1; +DROP TABLE t1; + +--echo # +--echo # 2. A stored generated column is written twice: once from what was +--echo # given, and again from its own expression, which is what stays. +--echo # + +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; +CREATE TABLE t1 (a VARCHAR(20), j LONGTEXT AS (a) STORED CHECK (JSON_VALID(j))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 (a, j) VALUES ('not a document', JSON_OBJECT('k', 1)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # and the value the expression does produce is taken +INSERT INTO t1 (a, j) VALUES ('{"k":1}', JSON_OBJECT('k', 1)); +SELECT a, j FROM t1; +--echo # the same on update +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET a = 'not a document'; +SELECT a, j FROM t1; +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; + +--echo # +--echo # 3. A row whose writing ends before the check is reached leaves +--echo # nothing behind for the NEXT row to be read with. +--echo # + +--echo # an update where one row comes out exactly as it went in +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('k', 1)); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (2, 0, 'not a document'); +SET @@check_constraint_checks = ON; +DELIMITER |; +CREATE TRIGGER t1t BEFORE UPDATE ON t1 FOR EACH ROW +BEGIN + IF OLD.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +DELIMITER ;| +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET b = b + (id = 2); +SELECT id, b, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # an insert through a view whose own check refuses the row first +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, + j VARCHAR(64) DEFAULT 'not a document' + CHECK (JSON_VALID(j))); +CREATE VIEW v1 AS SELECT id, b, j FROM t1 WHERE b > 0 WITH CHECK OPTION; +DELIMITER |; +CREATE TRIGGER t1t BEFORE INSERT ON t1 FOR EACH ROW +BEGIN + IF NEW.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +DELIMITER ;| +INSERT IGNORE INTO v1 (id, b) VALUES (1, 0), (2, 1); +SELECT id, b, j FROM t1 ORDER BY id; +DROP VIEW v1; +DROP TABLE t1; + +--echo # +--echo # 4. A row written twice over, where the SECOND writing does not +--echo # touch the column at all. +--echo # +--echo # The first writing puts a document there and attests to it; the +--echo # old row's bytes are then put back, and the update clause leaves +--echo # the column alone. What the check reads is the old bytes, which +--echo # nothing has attested. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, a INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 0, 'not a document'); +SET @@check_constraint_checks = ON; +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('x', 1)) + ON DUPLICATE KEY UPDATE a = 1; +SELECT id, a, j FROM t1 ORDER BY id; +--echo # and where the update clause DOES write the column, it is read the +--echo # ordinary way +INSERT INTO t1 VALUES (1, 2, JSON_OBJECT('x', 1)) + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('y', 2); +SELECT id, a, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 5. Stores that put down every byte they were given and call them +--echo # by another name. +--echo # +--echo # A binary column takes the bytes of a document written in some +--echo # other character set exactly as they stand, and a column in some +--echo # other set takes the bytes of a binary one the same way. Neither +--echo # writes the characters again, so what the row holds is not the +--echo # characters that were attested, and the check has to read it. +--echo # + +--echo # a document written in a wide set, into a binary column +CREATE TABLE t1 (j BLOB CHECK (JSON_VALID(j))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (CONVERT(JSON_OBJECT('a', 1) USING ucs2)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (CONVERT(JSON_ARRAY(1, 2) USING utf16)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # the same documents in a set the column can take go in +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +DROP TABLE t1; + +--echo # a binary document, into a column written in a wide set +CREATE TABLE t1 (j TEXT CHARACTER SET ucs2 CHECK (JSON_VALID(j))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(_binary'ab')); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT(_binary'k', 1)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # a store that does write the characters again, in the column's +--echo # own set, keeps the document +CREATE TABLE t1 (j TEXT CHARACTER SET latin1 CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +DROP TABLE t1; + +--echo # +--echo # 6. A row split in two by an update over part of its period. Such +--echo # an update writes the pieces of the old row that the period does +--echo # not cover as new rows, and it writes them by putting the saved +--echo # copy of the row back into the buffer the checks read. Anything +--echo # said about what stood there before is about the value the update +--echo # wrote, not the one just restored, so it goes with the row it was +--echo # about. +--echo # +--echo # A trigger is what makes the difference visible: it writes the +--echo # document column while the piece is being made, so the check has +--echo # something to catch. +--echo # + +CREATE TABLE t1 (id INT, + j VARCHAR(64) CHECK (JSON_VALID(j)), + s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +INSERT INTO t1 (id, j, s, e) + VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), + (2, '[1,2]', '1999-01-01', '2018-12-12'); + +UPDATE t1 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +--sorted_result +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t1; + +--echo # the same split with the pieces given documents of their own +CREATE TABLE t2 (id INT, + j VARCHAR(64) CHECK (JSON_VALID(j)), + s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t2_bi BEFORE INSERT ON t2 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t2 (id, j, s, e) + VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), + (2, '{"b":2}', '1999-01-01', '2018-12-12'); +UPDATE t2 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +--sorted_result +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t2; + +--echo # the row the update itself writes is checked as any other row is, +--echo # and it is written after the pieces have been, so what the pieces +--echo # left behind must not be believed of it: a portion set to +--echo # something that is not a document is refused +CREATE TABLE t3 (id INT, + j VARCHAR(64) CHECK (JSON_VALID(j)), + s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t3_bi BEFORE INSERT ON t3 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t3 (id, j, s, e) + VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), + (2, '{"b":2}', '1999-01-01', '2018-12-12'); +--error ER_CONSTRAINT_FAILED +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= 'not a document'; +--sorted_result +SELECT id, j, s, e FROM t3; + +--echo # and one set to a document goes in +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= JSON_OBJECT('portion', id); +--sorted_result +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t3; + +DROP TABLE t3, t2, t1; diff --git a/sql/field.cc b/sql/field.cc index 57a08128e6989..5df0e93ed2494 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -2687,6 +2687,111 @@ Field *Field::clone(MEM_ROOT *root, TABLE *new_table, my_ptrdiff_t diff) } +/* + Whether a store into this field that answered 0 puts down exactly the + characters it was handed. + + Only such a field can carry what an Item answered about those characters, + JSON being written in characters. CHAR and BINARY pad what they are + given and the padding is part of what is stored; ENUM and SET keep the + member they matched rather than the text that matched it; GEOMETRY and + the compressed types put down something else entirely; and the rest + convert. Every one of those answers 0 while doing it. + + Asked of real_type() rather than answered by a virtual, so that a field + type has to be NAMED here to be trusted. A field type nobody has looked + at is untrusted, and a new subclass of a trusted one does not inherit the + trust - which is the way round that costs a check that was going to pass, + rather than the way round that admits a value that is not a document. + + That holds for every subclass that says what it is: Field_geom derives + from Field_blob and answers MYSQL_TYPE_GEOMETRY, so naming the blobs + below does not name it. The compressed classes are the exception and + the reason for the question below the comment: they derive from a named + type and go on answering its real_type(), saying nothing anywhere that + a switch could read. They are asked about their compression instead, + which is the one thing they do say. + + A virtual is safe THAT way round. It is asked in order to REFUSE, so a + field type that has never heard of it is left where it started - refused + unless something below names it - and the inheritance that costs the + trust cannot grant any. +*/ + +bool Field::is_character_preserving() const +{ + if (compression_method()) + return false; + + switch (real_type()) { + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + return true; + default: + return false; + } +} + + +/* + The one rule every store site applies, asked in one place. + + A store that returned 0 into a field that puts down what it is given + mapped every character across without putting anything in its place. + JSON is written in characters, so a document that arrived that way is + still a document; anything else leaves the caller with nothing to say. + + Mapped across, and not merely carried across. A store into or out of + a binary field keeps the bytes and calls them by the other set's name, + which is a different string of characters - so a document written in + ucs2 and put into a BLOB arrives as bytes that read back as nothing at + all. conversion_keeps_characters() is the question, rather than the + plain my_charset_same() the two standing grants ask: those are said + once about values not yet made, where a store that would convert has + not happened yet and cannot be counted on, and this one is said with + the store already done and its answer in hand. + + The item is asked before the field type is, the two being in the order + that refuses soonest. Every value a table is given arrives here, and + one virtual answering no for everything that is not a document costs + less than the two virtuals and the switch that work out what this + field would have done with it. +*/ + +bool Field::is_attestation_preserved(const Item *item, int store_rc) const +{ + CHARSET_INFO *from= item->collation.collation; + + return store_rc == 0 && item->is_valid_json() && + is_character_preserving() && + String_copier::conversion_keeps_characters(charset(), from) && + !table->in_use->is_error(); +} + + +void Field::set_is_valid_json(const Item *item, int store_rc) +{ + /* + The list of places that set a mark is closed, and every one of them + asks first whether any column of this table carries a check that + could ever read one. Said here as well as at each of them, because + the closed list is what the debug reading in TABLE rests on: a site + added later that sets a mark on a table nobody asked about is a site + that will go on setting marks nothing polices. + */ + DBUG_ASSERT(table->has_own_json_valid_check); + + if (is_attestation_preserved(item, store_rc)) + bitmap_set_bit(&table->is_valid_json_set, field_index); + else + bitmap_clear_bit(&table->is_valid_json_set, field_index); +} + + int Field::set_default() { if (default_value) @@ -11825,6 +11930,12 @@ Virtual_column_info* Virtual_column_info::clone(THD *thd) Virtual_column_info* dst= new (thd->mem_root) Virtual_column_info(*this); if (!dst) return NULL; + /* + The copy's expression reads other Item_fields than the ones this was + worked out against, so it has to be worked out again when the table + the copy belongs to is opened. + */ + dst->json_valid_field_index= NO_JSON_VALID_FIELD; if (expr) { dst->expr= expr->deep_copy_with_checks(thd); diff --git a/sql/field.h b/sql/field.h index 8ff4d90f38a57..4250886781d72 100644 --- a/sql/field.h +++ b/sql/field.h @@ -581,6 +581,9 @@ static inline const char *vcol_type_name(enum_vcol_info_type type) - whether the field is used in a partitioning expression */ +/* A check constraint that is not one column's JSON_VALID() over itself */ +#define NO_JSON_VALID_FIELD ((field_index_t) ~0U) + class Virtual_column_info: public Sql_alloc, private Type_handler_hybrid_field_type { @@ -603,12 +606,21 @@ class Virtual_column_info: public Sql_alloc, Lex_ident name; /* Name of constraint */ /* see VCOL_* (VCOL_FIELD_REF, ...) */ uint flags; + /* + Worked out once at open: for a column's own check constraint that asks + nothing but whether that very column holds a document, the index of the + column. A check over another column, one wrapped in anything else, and + a check belonging to the table rather than a column all leave this + NO_JSON_VALID_FIELD and are always run. See TABLE::is_valid_json_set. + */ + field_index_t json_valid_field_index; Virtual_column_info() :Type_handler_hybrid_field_type(&type_handler_null), vcol_type((enum_vcol_info_type)VCOL_TYPE_NONE), in_partitioning_expr(FALSE), stored_in_db(FALSE), - utf8(TRUE), automatic_name(FALSE), expr(NULL), flags(0) + utf8(TRUE), automatic_name(FALSE), expr(NULL), flags(0), + json_valid_field_index(NO_JSON_VALID_FIELD) { name.str= NULL; name.length= 0; @@ -1217,6 +1229,30 @@ class Field: public Value_source bitmap_clear_bit(&table->has_value_set, field_index); } + /* + Attest to the value just stored here, by the one rule every store site + applies: the Item said it was a document, the store kept every + character of it, and nothing went wrong while it happened. Anything + else clears the mark - a store that failed leaves bytes behind too. + See TABLE::is_valid_json_set. + + Written here and read nowhere here. The one place that reads a mark + is TABLE::verify_constraints, which has the bitmap in front of it and + asks it by name; giving this class the short question as well would + put "does this field attest to its bytes" within reach of anything + that wanted it, and this is not that question. It is true of one row + image between a store and the next check and false everywhere a + reader would think to ask, which is a thing to be read by the one + caller that knows that and by nobody who does not. + */ + bool is_character_preserving() const; + bool is_attestation_preserved(const Item *item, int store_rc) const; + void set_is_valid_json(const Item *item, int store_rc); + void clear_is_valid_json() + { + bitmap_clear_bit(&table->is_valid_json_set, field_index); + } + virtual my_time_t get_timestamp(const uchar *pos, ulong *sec_part) const { DBUG_ASSERT(0); return 0; } my_time_t get_timestamp(ulong *sec_part) const diff --git a/sql/item.cc b/sql/item.cc index b2b58d5e76898..126c8719cce4e 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -10452,6 +10452,8 @@ bool Item_trigger_field::set_value(THD *thd, sp_rcontext * /*ctx*/, Item **it) field->table->copy_blobs= copy_blobs_saved; field->set_has_explicit_value(); + if (field->table->has_own_json_valid_check) + field->set_is_valid_json(item, err_code); return err_code < 0; } diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 803d9fcc224fb..489beedc04f3b 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -76,6 +76,17 @@ static struct Json_scan_count_hook { Json_scan_count_hook() { json_scan_start_hook= json_count_scan; } } json_scan_count_hook; + + +Json_scans_unbilled::Json_scans_unbilled(THD *thd) + :m_thd(thd), m_scans(thd->status_var.json_scans) +{ } + + +Json_scans_unbilled::~Json_scans_unbilled() +{ + m_thd->status_var.json_scans= m_scans; +} #endif @@ -1334,6 +1345,20 @@ bool Item_func_json_valid::val_bool() if ((null_value= args[0]->null_value)) return 0; + /* + What this function asks is what a value answers about itself, so + where the value has answered there is nothing left to find out. + + It is the same reading the check constraint leaves unrun at the + field boundary, left unrun here at the most direct site there is, + and the two cannot disagree: a mark says the characters read back + as a document, which is exactly what the walk below would go and + see. Nothing is said either way, the answer being true, and the + reading is the only thing that could have said anything. + */ + if (args[0]->is_valid_json()) + return true; + thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); je.killed_ptr= (uint32_t *) &thd->killed; diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 5217a201b3c46..2f310f7b402e1 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -26,6 +26,28 @@ #include "item_sum.h" #include "sql_type_json.h" +#ifndef DBUG_OFF +/* + Holds the reading count still across a reading the released server does + not do. + + The count is of the work a server does, and a debug build's reading back + of a value to check what was claimed about it is not that work: counting + it would move a number kept to watch queries for reasons no query has. + Where the reading is one call the count is taken straight back off it; + where it is a whole expression, whose readings the caller cannot count, + the figure is put back as it stood. +*/ +class Json_scans_unbilled +{ + THD *m_thd; + ulong m_scans; +public: + Json_scans_unbilled(THD *thd); + ~Json_scans_unbilled(); +}; +#endif + class json_path_with_flags { public: diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 160d7d4776d60..34b52b0badb20 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -9155,6 +9155,17 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, if (fields.elements) table_arg->auto_increment_field_not_null= FALSE; + /* + The marks are about the row image built here and no other. Clearing + them as the image is begun, rather than trusting something later to + clear them once it is done with, is what keeps them from being read + with the NEXT row: a row whose writing ends before anything reads what + was written leaves its marks standing - an update that finds nothing to + change never reaches the reading, and a view whose own check refuses + the row returns before it. + */ + table_arg->clear_is_valid_json_marks(); + while ((fld= f++)) { field= fld->field_for_view_update(); @@ -9186,12 +9197,15 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, { if (!skip_sys_field) { - if (value->save_in_field(rfield, 0) < 0 && !ignore_errors) + int store_rc= value->save_in_field(rfield, 0); + if (store_rc < 0 && !ignore_errors) { my_message(ER_UNKNOWN_ERROR, ER_THD(thd, ER_UNKNOWN_ERROR), MYF(0)); goto err_unwind_fields; } rfield->set_has_explicit_value(); + if (table->has_own_json_valid_check) + rfield->set_is_valid_json(value, store_rc); } /* In sql MODE_SIMULTANEOUS_ASSIGNMENT, @@ -9434,6 +9448,10 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List &values, only one row. */ table->auto_increment_field_not_null= FALSE; + + /* See the same call in the other fill_record() above. */ + table->clear_is_valid_json_marks(); + while ((field = *ptr++) && ! thd->is_error()) { /* Ensure that all fields are from the same table */ @@ -9469,10 +9487,23 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List &values, continue; if (use_value) + { + /* + Nothing said how the store went, so nothing can be attested - + and there is nothing to take back either. The marks were cleared + where this row image was begun, a few lines above, and this route + sets none. + */ value->save_val(field); + } else - if (value->save_in_field(field, 0) < 0) + { + int store_rc= value->save_in_field(field, 0); + if (store_rc < 0) goto err; + if (table->has_own_json_valid_check) + field->set_is_valid_json(value, store_rc); + } field->set_has_explicit_value(); } /* Update virtual fields if there wasn't any errors */ diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 8df98b09a7eea..aba2b41a4c9b4 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -3119,6 +3119,16 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) bitmaps_used*share->column_bitmap_size), share->fields); } + /* + Unlike has_value_set above this one is always initialised: it is read + for every row that is written, and the TABLE this was memcpy'd from + would otherwise leave it pointing at the client thread's bitmap. + The fourth block of the four allocated above is its own. + */ + my_bitmap_init(©->is_valid_json_set, + (my_bitmap_map*) (bitmap + + (bitmaps_used + 1)*share->column_bitmap_size), + share->fields); copy->tmp_set.bitmap= 0; // To catch errors bzero((char*) bitmap, share->column_bitmap_size * bitmaps_used); copy->read_set= ©->def_read_set; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index f0ef090d25059..6edfcc72b17ee 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -20781,6 +20781,9 @@ setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps, uint field_count) bitmaps+= bitmap_size; my_bitmap_init(&table->has_value_set, (my_bitmap_map*) bitmaps, field_count); + bitmaps+= bitmap_size; + my_bitmap_init(&table->is_valid_json_set, + (my_bitmap_map*) bitmaps, field_count); /* write_set and all_set are copies of read_set */ table->def_write_set= table->def_read_set; table->s->all_set= table->def_read_set; diff --git a/sql/sql_string.cc b/sql/sql_string.cc index f5edb1a5414d9..0e761fdf9c897 100644 --- a/sql/sql_string.cc +++ b/sql/sql_string.cc @@ -1118,10 +1118,7 @@ String_copier::well_formed_copy(CHARSET_INFO *to_cs, const char *from, size_t from_length, size_t nchars) { - if ((to_cs == &my_charset_bin) || - (from_cs == &my_charset_bin) || - (to_cs == from_cs) || - my_charset_same(from_cs, to_cs)) + if (conversion_copies_bytes(to_cs, from_cs)) { m_cannot_convert_error_pos= NULL; return (uint) to_cs->copy_fix(to, to_length, from, from_length, diff --git a/sql/sql_string.h b/sql/sql_string.h index f0977dc0dff9a..3ddfa4d32107f 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -101,6 +101,38 @@ class String_copier: public String_copy_status, protected MY_STRCONV_STATUS { public: + /* + Whether a copy between these two character sets moves the bytes + across as they stand, rather than writing the characters again in + the destination's formatting. + + This is the branch well_formed_copy() takes, said where a caller can + read it as well. Sets that are the same are copied unchanged, and so + is anything with a binary set at either end - the bytes go across + untouched and are then read as whatever the other set makes of them. + */ + static bool conversion_copies_bytes(CHARSET_INFO *to_cs, + CHARSET_INFO *from_cs) + { + return to_cs == &my_charset_bin || from_cs == &my_charset_bin || + to_cs == from_cs || my_charset_same(from_cs, to_cs); + } + /* + Whether such a copy still puts down the characters it was given. + + Sets that are the same put them down unchanged and different sets + convert them, both of which keep the characters. A binary set at + either end is neither: the bytes are kept and called by the other + set's name, which is a different string of characters and, for a + document, usually not one at all. That is the one case the branch + above takes without converting, which is what this asks. + */ + static bool conversion_keeps_characters(CHARSET_INFO *to_cs, + CHARSET_INFO *from_cs) + { + return my_charset_same(from_cs, to_cs) || + !conversion_copies_bytes(to_cs, from_cs); + } const char *cannot_convert_error_pos() const { return m_cannot_convert_error_pos; } const char *most_important_error_pos() const diff --git a/sql/sql_type_json.cc b/sql/sql_type_json.cc index 27072de2d559c..31767d68503ec 100644 --- a/sql/sql_type_json.cc +++ b/sql/sql_type_json.cc @@ -117,6 +117,38 @@ bool Type_handler_json_common::has_json_valid_constraint(const Field *field) } +/** + Whether a check constraint asks nothing but whether one particular + column holds a document, that column being the one the constraint + belongs to. + + has_json_valid_constraint() above deliberately does not ask WHICH column + the constraint reads: CHECK(json_valid(other_column)) types this column + as JSON, and has done for as long as the type has existed. Leaving the + check unrun needs the stronger question, because what was written into + this column says nothing about what is in another one, and that question + is the one asked here. +*/ + +bool Type_handler_json_common::is_json_valid_of_field(Virtual_column_info *check, + const Field *field) +{ + Item_func *func; + Item *arg; + + if (!check || check->get_vcol_type() != VCOL_CHECK_FIELD || + !check->expr || check->expr->type() != Item::FUNC_ITEM) + return false; + func= static_cast(check->expr); + if (func->functype() != Item_func::JSON_VALID_FUNC || + func->argument_count() != 1) + return false; + arg= func->arguments()[0]->real_item(); + return arg->type() == Item::FIELD_ITEM && + static_cast(arg)->field == field; +} + + /** Create JSON_VALID(field_name) expression */ diff --git a/sql/sql_type_json.h b/sql/sql_type_json.h index b7fe5c8aa64a3..2161f091a62d6 100644 --- a/sql/sql_type_json.h +++ b/sql/sql_type_json.h @@ -38,6 +38,8 @@ class Type_handler_json_common static const Type_handler *json_type_handler_sum(const Item_sum *sum); static const Type_handler *json_type_handler_from_generic(const Type_handler *th); static bool has_json_valid_constraint(const Field *field); + static bool is_json_valid_of_field(Virtual_column_info *check, + const Field *field); static const Type_collection *type_collection(); static bool is_json_type_handler(const Type_handler *handler) { diff --git a/sql/sql_update.cc b/sql/sql_update.cc index a50b52456474a..19b406b2c9c2e 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -3015,6 +3015,15 @@ int multi_update::do_updates() table->status|= STATUS_UPDATED; store_record(table,record[1]); + /* + A new row image begins here. Nothing on this pass goes through + fill_record() - the columns arrive by Copy_field, below - so this is + where the marks of the row before have to go; a trigger on this + table can set them, and a row that turns out to need no update + never reaches the reading that would clear them. + */ + table->clear_is_valid_json_marks(); + /* Copy data from temporary table to current table */ for (copy_field_ptr=copy_field; copy_field_ptr != copy_field_end; diff --git a/sql/table.cc b/sql/table.cc index f52f0590dea47..1b873c11d8ebd 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -42,6 +42,8 @@ #include "sql_view.h" #include "rpl_filter.h" #include "sql_cte.h" +#include "sql_type_json.h" // Type_handler_json_common +#include "item_jsonfunc.h" // Json_scans_unbilled #include "ha_sequence.h" #include "sql_show.h" #include "opt_trace.h" @@ -1256,6 +1258,17 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table, &((*field_ptr)->check_constraint), error_reported); *check_constraint_ptr++= (*field_ptr)->check_constraint; + /* + Work out once, here, whether this check asks anything besides + whether its own column holds a document. TABLE::verify_constraints + reads the answer per row and cannot afford to work it out there. + */ + if (vcol && + Type_handler_json_common::is_json_valid_of_field(vcol, *field_ptr)) + { + vcol->json_valid_field_index= (*field_ptr)->field_index; + table->has_own_json_valid_check= true; + } break; case VCOL_CHECK_TABLE: vcol= unpack_vcol_info_from_frm(thd, table, &expr_str, @@ -4461,7 +4474,7 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, /* Allocate bitmaps */ bitmap_size= share->column_bitmap_size; - bitmap_count= 7; + bitmap_count= 8; if (share->virtual_fields) bitmap_count++; @@ -4490,6 +4503,9 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, bitmaps+= bitmap_size; my_bitmap_init(&outparam->def_rpl_write_set, (my_bitmap_map*) bitmaps, share->fields); + bitmaps+= bitmap_size; + my_bitmap_init(&outparam->is_valid_json_set, + (my_bitmap_map*) bitmaps, share->fields); outparam->default_column_bitmaps(); outparam->cond_selectivity= 1.0; @@ -5861,6 +5877,12 @@ void TABLE::init(THD *thd, TABLE_LIST *tl) pos_in_table_list= tl; clear_column_bitmaps(); + /* + A cached TABLE, or one whose statement gave up between a write and the + reading of what was written, can arrive here with marks still on it. + They are about a row image that is over. + */ + clear_is_valid_json_marks(); for (Field **f_ptr= field ; *f_ptr ; f_ptr++) { (*f_ptr)->next_equal_field= NULL; @@ -6528,6 +6550,66 @@ int TABLE_LIST::view_check_option(THD *thd, bool ignore_failure) int TABLE::verify_constraints(bool ignore_failure) +{ + int rc= run_check_constraints(ignore_failure); + /* + The marks are about the bytes that are in record[0] right now, and this + is where the row image they belong to is done with. Draining them here + rather than at the callers is what makes them safe: whatever replaces + those bytes next - restore_record() between the two readings of + INSERT ... ON DUPLICATE KEY UPDATE is the sharp case - finds nothing + left over to be believed. + */ + clear_is_valid_json_marks(); + return rc; +} + + +#ifndef DBUG_OFF +/* + Reads back what a mark claimed, and stops a debug server if it was wrong. + + The mark says this column holds a document, so the check would pass and + is being left unrun. Nothing downstream ever finds out otherwise: a + wrong mark is a constraint that quietly stops being enforced, and the row + it lets through is indistinguishable from a row that passed. This is the + last place that can still tell the difference, so a debug build spends + the reading here to tell it. + + It is worth the cost because of the shape of the thing it polices. The + marks are SET at three places, which is a closed list, but they are + CLEARED at every place that writes record[0] without going through those + three - and that list is open. A site added later that forgets to clear + produces no wrong answer, no warning and no failing test: it produces a + check that silently stops running. With this here it produces an abort + on the first row that reaches it. + + The reading is taken back off Json_scans afterwards. It is the debug + build's work rather than the server's, and counting it would move a + number kept to watch the work a released server does. +*/ + +void TABLE::check_json_valid_mark(Virtual_column_info *check) +{ + bool held; + + /* + An error already standing makes the reading say nothing about the + mark, only about the error. + */ + if (in_use->is_error()) + return; + + { + Json_scans_unbilled unbilled(in_use); + held= check->expr->val_bool() || check->expr->null_value; + } + DBUG_ASSERT(held); +} +#endif + + +int TABLE::run_check_constraints(bool ignore_failure) { /* We have to check is_error() first as we are checking it for each @@ -6546,6 +6628,19 @@ int TABLE::verify_constraints(bool ignore_failure) StringBuffer field_error(system_charset_info); for (Virtual_column_info **chk= check_constraints ; *chk ; chk++) { + /* + A check that asks nothing but whether its own column holds a + document has nothing to find out where that column was written + from an item that already attested to the value. + */ + if ((*chk)->json_valid_field_index != NO_JSON_VALID_FIELD && + bitmap_is_set(&is_valid_json_set, (*chk)->json_valid_field_index)) + { +#ifndef DBUG_OFF + check_json_valid_mark(*chk); +#endif + continue; + } /* yes! NULL is ok. see 4.23.3.4 Table check constraints, part 2, SQL:2016 @@ -9179,6 +9274,13 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) # endif vcol_info->expr->save_in_field(vf, 0); DBUG_RESTORE_WRITE_SET(vf); + /* + Whatever was put into this field before, what stands there now came + from the column's own expression and nobody has attested to it. + A value assigned to a stored generated column is taken and then + written over exactly here. + */ + vf->clear_is_valid_json(); DBUG_PRINT("info", ("field '%s' - updated error: %d", vf->field_name.str, field_error)); if (swap_values && (vf->flags & BLOB_FLAG)) @@ -9238,6 +9340,8 @@ int TABLE::update_virtual_field(Field *vf, bool ignore_warnings) DBUG_FIX_WRITE_SET(vf); vf->vcol_info->expr->save_in_field(vf, 0); DBUG_RESTORE_WRITE_SET(vf); + /* See the same call in update_virtual_fields() above. */ + vf->clear_is_valid_json(); in_use->restore_active_arena(expr_arena, &backup_arena); in_use->pop_internal_handler(); if (ignore_warnings) diff --git a/sql/table.h b/sql/table.h index 6b8ebae10dcb6..75c985f2278f1 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1355,6 +1355,31 @@ struct TABLE MY_BITMAP *read_set, *write_set, *rpl_write_set; /* On INSERT: fields that the user specified a value for */ MY_BITMAP has_value_set; + /* + Fields whose bytes in record[0] were written from an Item that answered + is_valid_json(), by a store that kept every character of it. A + json_valid() check over such a field can only find out what is already + known, so verify_constraints() leaves it unread. + + The bits last one row image and no longer. verify_constraints() reads + them and clears the whole bitmap on every way out of itself, and + TABLE::init() clears it at the start of a statement. Whatever writes + record[0] without setting a bit therefore leaves it clear, and the + check runs - which is what every writer other than the three store + sites does. + */ + MY_BITMAP is_valid_json_set; + /* + Whether any column of this table carries a check that asks nothing but + whether that same column holds a document - which is the only thing the + bitmap above is ever read for. + + Worked out once, where the checks are worked out, because the writers + would otherwise ask a field about every column of every row on the + chance that one of them was constrained. Where no column is, no bit + can ever be set and nothing reads one. + */ + bool has_own_json_valid_check; /* The ID of the query that opened and is using this table. Has different @@ -1623,6 +1648,45 @@ struct TABLE void mark_columns_used_by_virtual_fields(void); void mark_check_constraint_columns_for_read(void); int verify_constraints(bool ignore_failure); + /* + The constraint loop alone. Call verify_constraints() instead: it is + what clears is_valid_json_set afterwards, and every way out of this + one leaves the marks standing. + */ + int run_check_constraints(bool ignore_failure); + /* + Take back whatever was said about the bytes standing in record[0]. + + Asked only of a table some column of which carries a check that could + read a mark. Where none does no bit can ever be set, and the writers + below run on every row of every table in the instance: the guard is + what keeps a bitmap nothing ever reads from being walked by all of + them. + */ + void clear_is_valid_json_marks() + { + if (has_own_json_valid_check) + bitmap_clear_all(&is_valid_json_set); + } + /* + Put a whole row image into record[0]. This is what restore_record() + does, and the reason it is a method rather than the memcpy it reads + as. + + Whatever replaces a row image wholesale leaves nothing that was said + about the one before it still true, and the list of places that do it + is open where the list of places that say something is closed. Naming + them one at a time is how one gets missed, and a missed one is a check + that quietly stops being run. + */ + void restore_record_image(const uchar *from) + { + memcpy(record[0], from, (size_t) s->reclength); + clear_is_valid_json_marks(); + } +#ifndef DBUG_OFF + void check_json_valid_mark(Virtual_column_info *check); +#endif void free_engine_stats(); void update_engine_independent_stats(); inline void column_bitmaps_set(MY_BITMAP *read_set_arg) diff --git a/sql/unireg.h b/sql/unireg.h index fa657a267c2cb..d8369a2a32aa4 100644 --- a/sql/unireg.h +++ b/sql/unireg.h @@ -72,7 +72,7 @@ /* Extern defines */ #define store_record(A,B) memcpy((A)->B,(A)->record[0],(size_t) (A)->s->reclength) -#define restore_record(A,B) memcpy((A)->record[0],(A)->B,(size_t) (A)->s->reclength) +#define restore_record(A,B) (A)->restore_record_image((A)->B) #define cmp_record(A,B) memcmp((A)->record[0],(A)->B,(size_t) (A)->s->reclength) #define empty_record(A) { \ restore_record((A),s->default_values); \ From 762c7a87ad12343252c3fbc51c69085da4fd8cd5 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Wed, 5 Aug 2026 17:01:38 -0400 Subject: [PATCH 14/30] MDEV-40641 Ask for the room a document takes once it is written A JSON function has to name a length for its result before it has seen a value, so it asks for one out of what the arguments say about themselves. What was asked for did not cover what writing the result out comes to. A result that does not fit what was asked for is cut where it stands, and what is left of a document is not one. Writing an internal temporary table does not raise `count_cuted_fields`, so the store that cuts it answers 0 and warns nobody; the width of that column was never chosen by anyone but the function itself. ## Reading the arguments two at a time and one at a time `Item_func_json_insert::fix_length_and_dec()`, which works out the length for `JSON_SET`, `JSON_INSERT` and `JSON_REPLACE`, steps `n_arg` by two but indexes `args[n_arg/2]` and `args[n_arg/2+1]`, which step by one. A document and k pairs is 2k+1 arguments and only the first k+1 are ever read. So the document is counted a second time, the value of the last pair is not counted at all, and an earlier value, once the loop reaches it, is added in the term meant for a path. `Item_func_json_array_append::fix_length_and_dec()` alongside it reads `args[n_arg+1]` and is right. Both were right after MDEV-29264. The up-merge of MDEV-32454, written on a branch that fix had never reached, put the older form back in one of the two, which is why the correct form has been sitting next to the wrong one since. ## Room for what the writing makes of a value Correcting the indices takes away the slack that counting the document twice had been providing, so the room the value really needs has to be asked for instead. A value that is not already a document is written as a JSON string, and twice its characters is what is asked for that. A string carries no separators of its own, so the spacing below has nothing to add to it. Twice does not cover an escaping, and this does not undertake to. A character the document's character set cannot carry is written as the hex of its UTF-16 form, which `my_uni_utf16()` gives as one unit for a character of the first plane and two for any other, each unit taking an escape of its own - six characters, or twelve where the pair is needed. Charging that needs both character sets to have a say, since a character is escaped only where the set being written in cannot carry it and the pair is only reached where the value's own set goes past the first plane; and charging it widens what these functions declare by enough to move a result out of a memory temporary table onto disk. It is left to be done with the work that stops a declared width deciding where a temporary table lives, and is reported on its own. What that leaves standing is recorded rather than left to be found. `Item_func_json_quote::fix_length_and_dec()` is the other way about: it asked for twelve, its comment reading that a character can turn into a pair of escapes. A pair is only ever needed for a character the set being written into cannot carry, and this one writes into utf8mb4, which carries every character there is. So it never escapes one for want of somewhere to put it, what is left is the handful JSON refuses literally, and six is the whole of its cost. The other six per character is room asked for that nothing can take, and a wide enough argument passes the width at which a result is given a blob to live in on the strength of it. The rule is written once now, in `json_value_reserve()`, and asked by all three functions that work out a length this way, so that they cannot drift apart again. A value that is already a document goes in as it stands, and is written out again with everything around it. One written as text and not a document is written as a JSON string. A boolean is written as one of the two words `true` and `false`. What is left is a number, written as itself. One that is not there at all is still written, as `null`, so nothing is ever shorter than four characters. Which of those a value is has to be the writer's question and not one of its own, so it asks `is_json_type()`, which looks through the wrapper that aggregating character sets puts round whichever argument has to move to get into the set the result is written in. Asking the argument's own type handler instead prices a document that reaches the writing as a document, and goes in as it stands, as though it were going to be quoted - two characters for a pair of quotes that never arrives: CREATE TABLE t (v VARCHAR(20) CHARACTER SET latin1, u VARCHAR(10) CHARACTER SET utf8mb4); CREATE TABLE w AS SELECT JSON_ARRAY(JSON_QUERY(v, '$'), u) FROM t; declares two characters more that way than the same two widths in one set do, which is 74 against 72. Nothing comes out wrong by it - this error is always room asked for and not taken, never a value cut short - but it is the same two characters however wide the document is, and it grows to six of them apiece once an escaping is charged for, where a wide enough result passes the width at which a column is given a blob to live in and the asking has decided the type. Only the two that BUILD a document can meet a wrapper at all, those being the two that aggregate the sets of their arguments; `JSON_ARRAY_APPEND` and the three that edit take the set of the document they were handed and aggregate nothing. The question is settled where it is asked rather than at the callers, so that a function which comes to aggregate later inherits the right one. The question is only put to an argument that could answer yes. Every type handler in the JSON collection is built over a string one, so an argument of any other result type is not a document and its own type has said so already; looking through the wrappers round it would be a walk that can only come back no, and the arguments that are not documents are most of them. Seven of the SQL functions are served by those three: `JSON_ARRAY` and `JSON_OBJECT`, `JSON_ARRAY_APPEND` and `JSON_ARRAY_INSERT`, `JSON_SET`, `JSON_INSERT` and `JSON_REPLACE`. ## Room for the spacing a document is written with The document a function is handed is written out again, with a space after every separator that is copied. So what comes back can be longer than what went in whether something was taken out of it, put into it, or neither, and the room for the document has to cover the writing and not only the reading. `JSON_REMOVE` shows it plainest, having nothing to add: a dense array of thirty ones, 61 characters held in a `VARCHAR(64)`, comes back as 87, and a path that matches nothing at all comes back as 90 with nothing taken out. The same spacing arrives when a document is edited rather than cut down, and again when one is spliced into another: the same array comes back from `JSON_SET` as 93, and merged with itself as 180. None of them allowed for it. For `JSON_SET` and its two the accident above had been covering it, and correcting the indices takes that away as well, so what was covered by mistake is now asked for on purpose. `JSON_ARRAY_APPEND` and `JSON_ARRAY_INSERT` never had it, and neither did a value that is already a document, which is what `JSON_MERGE` and `JSON_MERGE_PATCH` are counted with. All of them ask for twice the document now. Twice is a true bound rather than a guess: a separator has a value on either side of it and the shortest value is one character, so at worst every second character gains one. It is what `JSON_LOOSE` asks for, adding the same spacing, and `JSON_COMPACT`, which only ever takes spacing away, asks for exactly the argument. `JSON_QUERY` and `JSON_KEYS` also ask for exactly the argument and are right to. `JSON_QUERY` returns a piece of the document as it stands without respacing it, and every key `JSON_KEYS` lists costs less to list than it cost to declare. ## What this moves The length asked for is a ceiling, and ten recorded lines move, each of them a declared column width or the length reported in the metadata of a result. Five grow, being the ones that were asked for too little: two in `func_json`, one in `func_json_agg_limits`, one in `func_json_charset` and one in `type_json`. Five shrink, and all five are ceilings `JSON_QUOTE` names: three in `func_json`, one in `func_json_charset` and one in `type_json`. They shrink by six for every character of the argument, that being what separates the twelve asked for from the six a result written in utf8mb4 can ever spend. No line changes the type it declares, no value moves, and no warning or note appears or goes away. ## The test `func_json_value_reserve` reads the ceiling off the result itself rather than off a column the result was put in. A result wide enough to be given a blob to live in is held by the blob whatever was asked for, so a case shaped that way passes whether the arithmetic is right or wrong, and the width above which that happens is not a fixed one. Read straight off the result, the answer is the arithmetic and nothing else: unfixed, one line of it says the ceiling is 33 and the value that same statement produced is 47, and another says 64 against 87. The spacing is asked about the other way round, through a document packed tight enough that the spacing alone outgrows it, put through each of the functions that write one out again and read back off the column it lands in. Unfixed, the array of thirty ones comes back from `JSON_SET` as 83 characters of a 93 character document, and `JSON_VALID` says 0 of what is left. What a character costs `JSON_QUOTE` is asked as a width, a value of 45 characters being the room for 45 of them and the two quotes, and the constructor beside it is asked for twice its argument instead. What twice does not cover is recorded rather than left out, under a heading that says it is still broken. The same value of twenty characters that each need an escape is written two ways: into a table of its own, where the store says `ER_DATA_TOO_LONG` and the statement stops, and into an internal temporary table, where nothing is said at all - 137 characters produced, 75 kept, and `JSON_VALID` answering 0 of what is left. Those two lines are what the work this defers will turn over. The wrapper is priced by declaring the same two columns twice over, once within one character set and once across two, for the array and for the object: the two declare the same width in both worlds and hold the same answer. Beside them stands a function that EDITS, declaring the same width in both worlds for the other reason, which is what says the two above move because of the wrapping and not because two sets are in play. --- mysql-test/main/func_json.result | 10 +- mysql-test/main/func_json_agg_limits.result | 2 +- mysql-test/main/func_json_charset.result | 4 +- .../main/func_json_value_reserve.result | 653 ++++++++++++++++++ mysql-test/main/func_json_value_reserve.test | 437 ++++++++++++ mysql-test/main/type_json.result | 4 +- sql/item_jsonfunc.cc | 132 +++- 7 files changed, 1205 insertions(+), 37 deletions(-) create mode 100644 mysql-test/main/func_json_value_reserve.result create mode 100644 mysql-test/main/func_json_value_reserve.test diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index 6020e407a7184..80f7f753ac0c9 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -280,7 +280,7 @@ create table t1 as select json_object('id', 87, 'name', 'carrot') as f; show create table t1; Table Create Table t1 CREATE TABLE `t1` ( - `f` varchar(46) DEFAULT NULL + `f` varchar(52) DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci select * from t1; f @@ -305,7 +305,7 @@ json_quote('foo') show create table t1; Table Create Table t1 CREATE TABLE `t1` ( - `json_quote('foo')` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + `json_quote('foo')` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table t1; select json_merge('string'); @@ -770,8 +770,8 @@ JSON_QUOTE(_utf8'foo') AS c2; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `c1` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, - `c2` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + `c1` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `c2` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci DROP TABLE t1; # @@ -836,7 +836,7 @@ CREATE TABLE t2 SELECT JSON_ARRAY_INSERT(fld, '$.[0]', '0') FROM t1; SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( - `JSON_ARRAY_INSERT(fld, '$.[0]', '0')` varchar(21) DEFAULT NULL + `JSON_ARRAY_INSERT(fld, '$.[0]', '0')` varchar(40) DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci DROP TABLE t1, t2; SET sql_mode=default; diff --git a/mysql-test/main/func_json_agg_limits.result b/mysql-test/main/func_json_agg_limits.result index 196247ab7a4b8..d81e928259324 100644 --- a/mysql-test/main/func_json_agg_limits.result +++ b/mysql-test/main/func_json_agg_limits.result @@ -377,7 +377,7 @@ v [1] SELECT JSON_SET('{}', '$.a', 1) AS v; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def v 253 (format=json) 68 8 Y 0 39 45 +def v 253 (format=json) 84 8 Y 0 39 45 v {"a": 1} DROP TABLE t1; diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result index a2ad055dac99f..aa56e5763a119 100644 --- a/mysql-test/main/func_json_charset.result +++ b/mysql-test/main/func_json_charset.result @@ -480,7 +480,7 @@ v {"a": 1} SELECT JSON_SET('{}','$.a',1) AS v; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def v 253 (format=json) 68 8 Y 0 39 45 +def v 253 (format=json) 84 8 Y 0 39 45 v {"a": 1} SELECT JSON_EXTRACT('{"a":1}','$.a') AS v; @@ -495,7 +495,7 @@ v a SELECT JSON_QUOTE('a') AS v; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def v 253 56 3 Y 128 39 45 +def v 253 32 3 Y 128 39 45 v "a" SELECT JSON_KEYS('{"a":1}') AS v; diff --git a/mysql-test/main/func_json_value_reserve.result b/mysql-test/main/func_json_value_reserve.result new file mode 100644 index 0000000000000..d06ac662375a3 --- /dev/null +++ b/mysql-test/main/func_json_value_reserve.result @@ -0,0 +1,653 @@ +SET @save_optimizer_switch= @@optimizer_switch; +CREATE TABLE t1 (v VARCHAR(200)); +INSERT INTO t1 VALUES ('{"x":1}'), ('{"x":2}'); +CREATE TABLE t2 (v VARCHAR(10), s VARCHAR(20)); +INSERT INTO t2 VALUES ('{"x":1}', REPEAT(CHAR(1),20)), +('{"x":2}', REPEAT(CHAR(2),20)); +CREATE TABLE t3 (v VARCHAR(200)); +INSERT INTO t3 VALUES (CONCAT('{"x":"', REPEAT('a',190), '"}')), +(CONCAT('{"x":"', REPEAT('b',190), '"}')); +# +# What is asked for, taken off the result itself rather than off +# a column it was put in. Nothing about where the result goes can +# reach this number, so it says what the arithmetic came to and +# nothing else. The Length column is the one to read. +# +SELECT JSON_SET(v, '$.p', 'z') AS one_pair FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def one_pair 253 (format=json) 417 18 Y 0 39 8 +one_pair +{"x": 1, "p": "z"} +{"x": 2, "p": "z"} +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS two_pairs FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def two_pairs 253 (format=json) 434 28 Y 0 39 8 +two_pairs +{"x": 1, "p": "z", "q": "w"} +{"x": 2, "p": "z", "q": "w"} +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS three_pairs +FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def three_pairs 253 (format=json) 451 38 Y 0 39 8 +three_pairs +{"x": 1, "p": "z", "q": "w", "s": "y"} +{"x": 2, "p": "z", "q": "w", "s": "y"} +SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS plain_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def plain_value 253 (format=json) 95 47 Y 0 39 8 +plain_value +{"x": 1, "p": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +{"x": 2, "p": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS escaped_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def escaped_value 253 (format=json) 75 57 Y 0 39 8 +escaped_value +{"x": 1, "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +{"x": 2, "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +SELECT JSON_SET(v, '$.p', s) AS wide_escape_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def wide_escape_value 253 (format=json) 75 137 Y 0 39 8 +wide_escape_value +{"x": 1, "p": "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"} +{"x": 2, "p": "\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"} +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS document_fills_column FROM t3; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def document_fills_column 253 (format=json) 455 248 Y 0 39 8 +document_fills_column +{"x": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +{"x": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +# +# The same numbers again, read off the column a result is put in. +# The document, then a path and a value for each pair. +# +CREATE TABLE d1 AS SELECT JSON_SET(v, '$.p', 'z') AS r FROM t1; +SHOW CREATE TABLE d1; +Table Create Table +d1 CREATE TABLE `d1` ( + `r` varchar(417) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d1; +CREATE TABLE d2 AS +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS r FROM t1; +SHOW CREATE TABLE d2; +Table Create Table +d2 CREATE TABLE `d2` ( + `r` varchar(434) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d2; +CREATE TABLE d3 AS +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS r FROM t1; +SHOW CREATE TABLE d3; +Table Create Table +d3 CREATE TABLE `d3` ( + `r` varchar(451) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d3; +SET @@optimizer_switch='derived_merge=off'; +# +# A value longer than the document it goes into. The result is +# put in a column here, so what is asked for is what is kept. +# +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +produced +5017 +5017 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5017 1 +5017 1 +# +# A small document, so that what is asked for stays under the +# width above which a result is given a blob to live in. The +# column stays a VARCHAR, so the arithmetic is what decides the +# answer rather than the change of type. +# +CREATE TABLE d4 AS SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2; +SHOW CREATE TABLE d4; +Table Create Table +d4 CREATE TABLE `d4` ( + `r` varchar(95) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d4; +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',30))) AS produced FROM t2; +produced +47 +47 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2) AS d; +kept valid +47 1 +47 1 +# +# A value whose characters are written with an escape apiece, so +# what goes in is twice what was passed. +# +CREATE TABLE d5 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2; +SHOW CREATE TABLE d5; +Table Create Table +d5 CREATE TABLE `d5` ( + `r` varchar(75) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d5; +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t2; +produced +57 +57 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2) AS d; +kept valid +57 1 +57 1 +# +# A value whose characters cannot be written as they stand and are +# escaped out instead, six characters for each one. Twice the +# characters does not cover that, and this is the room that is +# still asked for short. +# +# STILL BROKEN AND KNOWN TO BE. Pricing an escaping properly needs +# both character sets to have a say, and it widens what these +# functions declare by enough to move a result out of a memory +# temporary table onto disk. It is left until a declared width no +# longer decides where a temporary table lives, and what is +# recorded below is what that leaves standing. +# +# Written into a table of its own, the store says so and the +# statement stops. +# +CREATE TABLE d6 AS SELECT JSON_SET(v, '$.p', s) AS r FROM t2; +ERROR 22001: Data too long for column 'r' at row 1 +SELECT LENGTH(s) AS value_length FROM t2; +value_length +20 +20 +SELECT LENGTH(JSON_SET(v, '$.p', s)) AS produced FROM t2; +produced +137 +137 +# +# Written into an internal temporary table, nothing says anything: +# that store does not raise count_cuted_fields, so the document is +# cut where it stands and what is left of it is not one. 'valid' +# reading 0 here is the same defect going unremarked. +# +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', s) AS r FROM t2) AS d; +kept valid +75 0 +75 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# A document that fills the column it is held in, with a short +# value that has to be written with escapes. The document is +# allowed twice its own width and no more, so whatever the +# number below comes to over that is the room for the path and +# the value or it is nothing at all. +# +CREATE TABLE d7 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3; +SHOW CREATE TABLE d7; +Table Create Table +d7 CREATE TABLE `d7` ( + `r` varchar(455) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d7; +SELECT LENGTH(v) AS document FROM t3; +document +198 +198 +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t3; +produced +248 +248 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3) AS d; +kept valid +248 1 +248 1 +# +# The long value in the last pair of three, which is the one the +# loop reaches last. +# +SELECT LENGTH(JSON_SET(v, '$.p', 'x', '$.q', 'y', +'$.s', REPEAT('z',5000))) AS produced FROM t1; +produced +5037 +5037 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', 'x', '$.q', 'y', +'$.s', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5037 1 +5037 1 +# +# JSON_INSERT and JSON_REPLACE, whose length the same function +# works out. +# +SELECT LENGTH(JSON_INSERT(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +produced +5017 +5017 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_INSERT(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5017 1 +5017 1 +SELECT LENGTH(JSON_REPLACE(v, '$.x', REPEAT('z',5000))) AS produced FROM t1; +produced +5009 +5009 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REPLACE(v, '$.x', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5009 1 +5009 1 +# +# A path long enough to be worth counting. The name it adds is +# what goes into the result, so the room for it is asked for once. +# +SELECT LENGTH(JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z')) AS produced +FROM t1; +produced +317 +317 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z') AS r +FROM t1) AS d; +kept valid +317 1 +317 1 +# +# The same room asked for where it is checked rather than cut. +# +CREATE TABLE d8 AS SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid FROM d8; +kept valid +5017 1 +5017 1 +DROP TABLE d8; +# +# A document held in a column wide enough that nothing it is asked +# to hold can overflow it. +# +CREATE TABLE t4 (j JSON); +INSERT INTO t4 VALUES ('{"x":1}'), ('{"x":2}'); +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(j, '$.p', REPEAT('z',5000)) AS r FROM t4) AS d; +kept valid +5017 1 +5017 1 +DROP TABLE t4; +# +# The same room, asked for by the functions that build a document +# rather than edit one, and by the two that append to an array. +# All four work it out the same way and all four are given a value +# whose characters have to be escaped. +# +SELECT JSON_ARRAY(s) AS array_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def array_value 253 (format=json) 48 124 Y 0 39 8 +array_value +["\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"] +["\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"] +SELECT JSON_OBJECT('k', s) AS object_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def object_value 253 (format=json) 56 129 Y 0 39 8 +object_value +{"k": "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"} +{"k": "\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"} +SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS appended FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def appended 253 (format=json) 66 134 Y 0 39 8 +appended +{"x": [1, "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"]} +{"x": [2, "\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"]} +SELECT JSON_ARRAY_INSERT(JSON_SET(v, '$.x', JSON_ARRAY(1)), '$.x[0]', s) +AS inserted FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def inserted 253 (format=json) 152 134 Y 0 39 8 +inserted +{"x": ["\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001", 1]} +{"x": ["\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002", 1]} +SELECT LENGTH(JSON_ARRAY(s)) AS produced FROM t2; +produced +124 +124 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY(s) AS r FROM t2) AS d; +kept valid +48 0 +48 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT LENGTH(JSON_OBJECT('k', s)) AS produced FROM t2; +produced +129 +129 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_OBJECT('k', s) AS r FROM t2) AS d; +kept valid +56 0 +56 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$.x', s)) AS produced FROM t2; +produced +134 +134 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS r FROM t2) AS d; +kept valid +66 0 +66 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# What an escape costs where it is JSON_QUOTE doing the writing. +# A character that cannot be written as it stands is escaped as a +# backslash, a 'u' and the hex of its UTF-16 form, which is four +# figures for a character of the first plane and a second escape +# after it for any other. +# +# The pair is only ever reached for a character the set being +# written into cannot carry, and this function writes into +# utf8mb4, which carries everything there is. So six apiece is +# the whole of it, and every width over that is room asked for +# that nothing can take. +# +# The constructor below is asked for twice its argument instead, +# which is the room that does not cover an escaping - see above. +# +CREATE TABLE t7 (s VARCHAR(45)); +INSERT INTO t7 VALUES (REPEAT('a',45)), (REPEAT('b',45)); +SELECT JSON_QUOTE(s) AS quoted FROM t7; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def quoted 253 272 47 Y 128 39 8 +quoted +"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +CREATE TABLE d9 AS SELECT JSON_ARRAY(s) AS r FROM t7; +SHOW CREATE TABLE d9; +Table Create Table +d9 CREATE TABLE `d9` ( + `r` varchar(98) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d9; +DROP TABLE t7; +# +# A value that is already a document goes in as it stands, and it +# is written out again with the spacing, so the room for it is +# asked for twice over and no quotes are asked for round it. +# +SELECT JSON_ARRAY(JSON_QUERY(v, '$')) AS document_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def document_value 253 (format=json) 26 9 Y 0 39 8 +document_value +[{"x":1}] +[{"x":2}] +# +# A function that takes something out and writes what is left out +# again, a space arriving after every separator it copies. One +# element of two characters goes and twenty-eight spaces come, so +# the room asked for has to cover the writing here as well. +# +CREATE TABLE t5 (v VARCHAR(64)) CHARSET utf8mb4; +INSERT INTO t5 VALUES +('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), +('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); +SELECT JSON_REMOVE(v, '$[0]') AS removed FROM t5; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def removed 253 (format=json) 128 87 Y 0 39 8 +removed +[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] +[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] +SELECT LENGTH(v) AS document FROM t5; +document +61 +61 +SELECT LENGTH(JSON_REMOVE(v, '$[0]')) AS produced FROM t5; +produced +87 +87 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REMOVE(v, '$[0]') AS r FROM t5) AS d; +kept valid +87 1 +87 1 +# +# A path that matches nothing, where nothing is taken out at all +# and only the spacing arrives. +# +SELECT LENGTH(JSON_REMOVE(v, '$.nothing')) AS produced FROM t5; +produced +90 +90 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REMOVE(v, '$.nothing') AS r FROM t5) AS d; +kept valid +90 1 +90 1 +# +# The neighbour that adds the same spacing without taking anything +# out, and the one that only ever takes spacing away. +# +SELECT LENGTH(JSON_LOOSE(v)) AS produced FROM t5; +produced +90 +90 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_LOOSE(v) AS r FROM t5) AS d; +kept valid +90 1 +90 1 +SELECT LENGTH(JSON_COMPACT(v)) AS produced FROM t5; +produced +61 +61 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_COMPACT(v) AS r FROM t5) AS d; +kept valid +61 1 +61 1 +# +# The same spacing arrives when a document is edited rather than +# cut down, the whole of it being written out again around whatever +# was put in. A document packed tight enough that the spacing +# alone outgrows it is the shape that says whether the room for the +# document covers the writing or only the reading. +# +SELECT LENGTH(JSON_SET(v, '$[30]', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$[30]', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +SELECT LENGTH(JSON_INSERT(v, '$[30]', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_INSERT(v, '$[30]', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +SELECT LENGTH(JSON_REPLACE(v, '$[0]', 9)) AS produced FROM t5; +produced +90 +90 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REPLACE(v, '$[0]', 9) AS r FROM t5) AS d; +kept valid +90 1 +90 1 +# +# The two that append to an array, which write the document out +# again in the same way. +# +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY_APPEND(v, '$', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +SELECT LENGTH(JSON_ARRAY_INSERT(v, '$[0]', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY_INSERT(v, '$[0]', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +# +# A value that is already a document goes in as it stands, but what +# goes in is written out again with the rest, so the room for it +# has to cover the spacing too. +# +SELECT LENGTH(JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$'))) AS produced +FROM t5; +produced +180 +180 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$')) AS r +FROM t5) AS d; +kept valid +180 1 +180 1 +DROP TABLE t5; +# +# The same, patched rather than merged, over two documents that +# have no name in common so that what comes back holds all of both. +# The columns are no wider than what they hold, so the room asked +# for is what the two of them say and nothing is over from the +# declaring. +# +CREATE TABLE t6 (v1 VARCHAR(37), v2 VARCHAR(37)) CHARSET utf8mb4; +INSERT INTO t6 VALUES +('{"a":1,"b":1,"c":1,"d":1,"e":1,"f":1}', +'{"g":1,"h":1,"i":1,"j":1,"k":1,"l":1}'), +('{"m":2,"n":2,"o":2,"p":2,"q":2,"r":2}', +'{"s":2,"t":2,"u":2,"v":2,"w":2,"x":2}'); +SELECT LENGTH(v1) AS document FROM t6; +document +37 +37 +SELECT LENGTH(JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), +JSON_QUERY(v2, '$'))) AS produced FROM t6; +produced +96 +96 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), +JSON_QUERY(v2, '$')) AS r FROM t6) AS d; +kept valid +96 1 +96 1 +DROP TABLE t6; +# +# The same document priced twice: once with every argument already in +# the character set the result is written in, and once with one of them +# having to be converted to get there. +# +# Aggregating the sets wraps the argument that has to move, and what is +# asked for has to see through that wrapper the same way the writing +# does. A document goes in as it stands whether it arrived wrapped or +# not, so it costs what a document costs; read as a plain string it +# would be charged for the two quote characters that the writing is +# never going to put round it. Both columns below are declared the +# same width, +# and hold the same answer - the miscounting was only ever room asked +# for and not taken, so no value was ever cut short by it. +# +CREATE TABLE t4 (v VARCHAR(20) CHARACTER SET latin1, +u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t4 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); +CREATE TABLE t7 (v VARCHAR(20) CHARACTER SET utf8mb4, +u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t7 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); +CREATE TABLE m1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t4; +SHOW CREATE TABLE m1; +Table Create Table +m1 CREATE TABLE `m1` ( + `r` varchar(72) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE TABLE s1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t7; +SHOW CREATE TABLE s1; +Table Create Table +s1 CREATE TABLE `s1` ( + `r` varchar(72) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT r FROM m1 ORDER BY r; +r +[{"x":1}, "{\"y\":1}"] +[{"x":2}, "{\"y\":2}"] +SELECT r FROM s1 ORDER BY r; +r +[{"x":1}, "{\"y\":1}"] +[{"x":2}, "{\"y\":2}"] +DROP TABLE m1, s1; +# +# An object is built by the same arithmetic and moves the same way. +# +CREATE TABLE m2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r +FROM t4; +SHOW CREATE TABLE m2; +Table Create Table +m2 CREATE TABLE `m2` ( + `r` varchar(88) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE TABLE s2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r +FROM t7; +SHOW CREATE TABLE s2; +Table Create Table +s2 CREATE TABLE `s2` ( + `r` varchar(88) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE m2, s2; +# +# THE CONTROL: the same two columns through a function that EDITS a +# document. Those take the set of the document they were given rather +# than aggregating over their arguments, so no wrapper is built and +# there is nothing to see through. Both are declared the same width in +# both worlds, which is what says the widths above move because of the +# wrapping and not because two sets are in play. +# +CREATE TABLE m3 AS +SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r +FROM t4; +SHOW CREATE TABLE m3; +Table Create Table +m3 CREATE TABLE `m3` ( + `r` varchar(73) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE TABLE s3 AS +SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r +FROM t7; +SHOW CREATE TABLE s3; +Table Create Table +s3 CREATE TABLE `s3` ( + `r` varchar(73) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT r FROM m3 ORDER BY r; +r +{"y": 1, "k": {"x": 1}} +{"y": 2, "k": {"x": 2}} +SELECT r FROM s3 ORDER BY r; +r +{"y": 1, "k": {"x": 1}} +{"y": 2, "k": {"x": 2}} +DROP TABLE m3, s3; +DROP TABLE t4, t7; +SET @@optimizer_switch= @save_optimizer_switch; +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/func_json_value_reserve.test b/mysql-test/main/func_json_value_reserve.test new file mode 100644 index 0000000000000..c9e903b04cd60 --- /dev/null +++ b/mysql-test/main/func_json_value_reserve.test @@ -0,0 +1,437 @@ +# +# The room a function asks for the document it is going to return. +# +# The length has to be named before any value has been seen, so it is +# asked for out of what the arguments say about themselves. A result +# that does not fit what was asked for is cut where it stands, and what +# is left of a document is not one. +# +# A value that is not already a document goes in as a JSON string, so +# what is asked for has to cover what the writing makes of it and not +# what was passed. Twice its characters is what is asked, and a value +# that is not there at all is still written as four. +# +# Twice does not cover an escaping - that costs six for a character, or +# twelve where the set being written into cannot carry it - and pricing +# it does not stand here yet. Where that leaves a value escaped +# throughout is recorded below, under the heading that says so. +# +# The document a function is given has the same to attest to. It is +# written out again with a space after every separator that is copied, +# so what comes back can be longer than what went in whether something +# was taken out of it, put into it, or neither. +# +SET @save_optimizer_switch= @@optimizer_switch; + +CREATE TABLE t1 (v VARCHAR(200)); +INSERT INTO t1 VALUES ('{"x":1}'), ('{"x":2}'); + +CREATE TABLE t2 (v VARCHAR(10), s VARCHAR(20)); +INSERT INTO t2 VALUES ('{"x":1}', REPEAT(CHAR(1),20)), + ('{"x":2}', REPEAT(CHAR(2),20)); + +CREATE TABLE t3 (v VARCHAR(200)); +INSERT INTO t3 VALUES (CONCAT('{"x":"', REPEAT('a',190), '"}')), + (CONCAT('{"x":"', REPEAT('b',190), '"}')); + +--echo # +--echo # What is asked for, taken off the result itself rather than off +--echo # a column it was put in. Nothing about where the result goes can +--echo # reach this number, so it says what the arithmetic came to and +--echo # nothing else. The Length column is the one to read. +--echo # +--enable_metadata +SELECT JSON_SET(v, '$.p', 'z') AS one_pair FROM t1; +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS two_pairs FROM t1; +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS three_pairs + FROM t1; +SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS plain_value FROM t2; +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS escaped_value FROM t2; +SELECT JSON_SET(v, '$.p', s) AS wide_escape_value FROM t2; +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS document_fills_column FROM t3; +--disable_metadata + +--echo # +--echo # The same numbers again, read off the column a result is put in. +--echo # The document, then a path and a value for each pair. +--echo # +CREATE TABLE d1 AS SELECT JSON_SET(v, '$.p', 'z') AS r FROM t1; +SHOW CREATE TABLE d1; +DROP TABLE d1; + +CREATE TABLE d2 AS + SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS r FROM t1; +SHOW CREATE TABLE d2; +DROP TABLE d2; + +CREATE TABLE d3 AS + SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS r FROM t1; +SHOW CREATE TABLE d3; +DROP TABLE d3; + +SET @@optimizer_switch='derived_merge=off'; + +--echo # +--echo # A value longer than the document it goes into. The result is +--echo # put in a column here, so what is asked for is what is kept. +--echo # +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; + +--echo # +--echo # A small document, so that what is asked for stays under the +--echo # width above which a result is given a blob to live in. The +--echo # column stays a VARCHAR, so the arithmetic is what decides the +--echo # answer rather than the change of type. +--echo # +CREATE TABLE d4 AS SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2; +SHOW CREATE TABLE d4; +DROP TABLE d4; + +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',30))) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2) AS d; + +--echo # +--echo # A value whose characters are written with an escape apiece, so +--echo # what goes in is twice what was passed. +--echo # +CREATE TABLE d5 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2; +SHOW CREATE TABLE d5; +DROP TABLE d5; + +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2) AS d; + +--echo # +--echo # A value whose characters cannot be written as they stand and are +--echo # escaped out instead, six characters for each one. Twice the +--echo # characters does not cover that, and this is the room that is +--echo # still asked for short. +--echo # +--echo # STILL BROKEN AND KNOWN TO BE. Pricing an escaping properly needs +--echo # both character sets to have a say, and it widens what these +--echo # functions declare by enough to move a result out of a memory +--echo # temporary table onto disk. It is left until a declared width no +--echo # longer decides where a temporary table lives, and what is +--echo # recorded below is what that leaves standing. +--echo # +--echo # Written into a table of its own, the store says so and the +--echo # statement stops. +--echo # +--error ER_DATA_TOO_LONG +CREATE TABLE d6 AS SELECT JSON_SET(v, '$.p', s) AS r FROM t2; + +SELECT LENGTH(s) AS value_length FROM t2; +SELECT LENGTH(JSON_SET(v, '$.p', s)) AS produced FROM t2; + +--echo # +--echo # Written into an internal temporary table, nothing says anything: +--echo # that store does not raise count_cuted_fields, so the document is +--echo # cut where it stands and what is left of it is not one. 'valid' +--echo # reading 0 here is the same defect going unremarked. +--echo # +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', s) AS r FROM t2) AS d; + +--echo # +--echo # A document that fills the column it is held in, with a short +--echo # value that has to be written with escapes. The document is +--echo # allowed twice its own width and no more, so whatever the +--echo # number below comes to over that is the room for the path and +--echo # the value or it is nothing at all. +--echo # +CREATE TABLE d7 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3; +SHOW CREATE TABLE d7; +DROP TABLE d7; + +SELECT LENGTH(v) AS document FROM t3; +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t3; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3) AS d; + +--echo # +--echo # The long value in the last pair of three, which is the one the +--echo # loop reaches last. +--echo # +SELECT LENGTH(JSON_SET(v, '$.p', 'x', '$.q', 'y', + '$.s', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', 'x', '$.q', 'y', + '$.s', REPEAT('z',5000)) AS r FROM t1) AS d; + +--echo # +--echo # JSON_INSERT and JSON_REPLACE, whose length the same function +--echo # works out. +--echo # +SELECT LENGTH(JSON_INSERT(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_INSERT(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; + +SELECT LENGTH(JSON_REPLACE(v, '$.x', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REPLACE(v, '$.x', REPEAT('z',5000)) AS r FROM t1) AS d; + +--echo # +--echo # A path long enough to be worth counting. The name it adds is +--echo # what goes into the result, so the room for it is asked for once. +--echo # +SELECT LENGTH(JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z')) AS produced + FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z') AS r + FROM t1) AS d; + +--echo # +--echo # The same room asked for where it is checked rather than cut. +--echo # +CREATE TABLE d8 AS SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid FROM d8; +DROP TABLE d8; + +--echo # +--echo # A document held in a column wide enough that nothing it is asked +--echo # to hold can overflow it. +--echo # +CREATE TABLE t4 (j JSON); +INSERT INTO t4 VALUES ('{"x":1}'), ('{"x":2}'); +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(j, '$.p', REPEAT('z',5000)) AS r FROM t4) AS d; +DROP TABLE t4; + +--echo # +--echo # The same room, asked for by the functions that build a document +--echo # rather than edit one, and by the two that append to an array. +--echo # All four work it out the same way and all four are given a value +--echo # whose characters have to be escaped. +--echo # +--enable_metadata +SELECT JSON_ARRAY(s) AS array_value FROM t2; +SELECT JSON_OBJECT('k', s) AS object_value FROM t2; +SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS appended FROM t2; +SELECT JSON_ARRAY_INSERT(JSON_SET(v, '$.x', JSON_ARRAY(1)), '$.x[0]', s) + AS inserted FROM t2; +--disable_metadata + +SELECT LENGTH(JSON_ARRAY(s)) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY(s) AS r FROM t2) AS d; + +SELECT LENGTH(JSON_OBJECT('k', s)) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_OBJECT('k', s) AS r FROM t2) AS d; + +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$.x', s)) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS r FROM t2) AS d; + +--echo # +--echo # What an escape costs where it is JSON_QUOTE doing the writing. +--echo # A character that cannot be written as it stands is escaped as a +--echo # backslash, a 'u' and the hex of its UTF-16 form, which is four +--echo # figures for a character of the first plane and a second escape +--echo # after it for any other. +--echo # +--echo # The pair is only ever reached for a character the set being +--echo # written into cannot carry, and this function writes into +--echo # utf8mb4, which carries everything there is. So six apiece is +--echo # the whole of it, and every width over that is room asked for +--echo # that nothing can take. +--echo # +--echo # The constructor below is asked for twice its argument instead, +--echo # which is the room that does not cover an escaping - see above. +--echo # +CREATE TABLE t7 (s VARCHAR(45)); +INSERT INTO t7 VALUES (REPEAT('a',45)), (REPEAT('b',45)); + +--enable_metadata +SELECT JSON_QUOTE(s) AS quoted FROM t7; +--disable_metadata + +CREATE TABLE d9 AS SELECT JSON_ARRAY(s) AS r FROM t7; +SHOW CREATE TABLE d9; +DROP TABLE d9; +DROP TABLE t7; + +--echo # +--echo # A value that is already a document goes in as it stands, and it +--echo # is written out again with the spacing, so the room for it is +--echo # asked for twice over and no quotes are asked for round it. +--echo # +--enable_metadata +SELECT JSON_ARRAY(JSON_QUERY(v, '$')) AS document_value FROM t2; +--disable_metadata + +--echo # +--echo # A function that takes something out and writes what is left out +--echo # again, a space arriving after every separator it copies. One +--echo # element of two characters goes and twenty-eight spaces come, so +--echo # the room asked for has to cover the writing here as well. +--echo # +CREATE TABLE t5 (v VARCHAR(64)) CHARSET utf8mb4; +INSERT INTO t5 VALUES + ('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), + ('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); + +--enable_metadata +SELECT JSON_REMOVE(v, '$[0]') AS removed FROM t5; +--disable_metadata + +SELECT LENGTH(v) AS document FROM t5; +SELECT LENGTH(JSON_REMOVE(v, '$[0]')) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REMOVE(v, '$[0]') AS r FROM t5) AS d; + +--echo # +--echo # A path that matches nothing, where nothing is taken out at all +--echo # and only the spacing arrives. +--echo # +SELECT LENGTH(JSON_REMOVE(v, '$.nothing')) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REMOVE(v, '$.nothing') AS r FROM t5) AS d; + +--echo # +--echo # The neighbour that adds the same spacing without taking anything +--echo # out, and the one that only ever takes spacing away. +--echo # +SELECT LENGTH(JSON_LOOSE(v)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_LOOSE(v) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_COMPACT(v)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_COMPACT(v) AS r FROM t5) AS d; + +--echo # +--echo # The same spacing arrives when a document is edited rather than +--echo # cut down, the whole of it being written out again around whatever +--echo # was put in. A document packed tight enough that the spacing +--echo # alone outgrows it is the shape that says whether the room for the +--echo # document covers the writing or only the reading. +--echo # +SELECT LENGTH(JSON_SET(v, '$[30]', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$[30]', 1) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_INSERT(v, '$[30]', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_INSERT(v, '$[30]', 1) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_REPLACE(v, '$[0]', 9)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REPLACE(v, '$[0]', 9) AS r FROM t5) AS d; + +--echo # +--echo # The two that append to an array, which write the document out +--echo # again in the same way. +--echo # +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY_APPEND(v, '$', 1) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_ARRAY_INSERT(v, '$[0]', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY_INSERT(v, '$[0]', 1) AS r FROM t5) AS d; + +--echo # +--echo # A value that is already a document goes in as it stands, but what +--echo # goes in is written out again with the rest, so the room for it +--echo # has to cover the spacing too. +--echo # +SELECT LENGTH(JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$'))) AS produced + FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$')) AS r + FROM t5) AS d; + +DROP TABLE t5; + +--echo # +--echo # The same, patched rather than merged, over two documents that +--echo # have no name in common so that what comes back holds all of both. +--echo # The columns are no wider than what they hold, so the room asked +--echo # for is what the two of them say and nothing is over from the +--echo # declaring. +--echo # +CREATE TABLE t6 (v1 VARCHAR(37), v2 VARCHAR(37)) CHARSET utf8mb4; +INSERT INTO t6 VALUES + ('{"a":1,"b":1,"c":1,"d":1,"e":1,"f":1}', + '{"g":1,"h":1,"i":1,"j":1,"k":1,"l":1}'), + ('{"m":2,"n":2,"o":2,"p":2,"q":2,"r":2}', + '{"s":2,"t":2,"u":2,"v":2,"w":2,"x":2}'); + +SELECT LENGTH(v1) AS document FROM t6; +SELECT LENGTH(JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), + JSON_QUERY(v2, '$'))) AS produced FROM t6; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), + JSON_QUERY(v2, '$')) AS r FROM t6) AS d; + +DROP TABLE t6; + +--echo # +--echo # The same document priced twice: once with every argument already in +--echo # the character set the result is written in, and once with one of them +--echo # having to be converted to get there. +--echo # +--echo # Aggregating the sets wraps the argument that has to move, and what is +--echo # asked for has to see through that wrapper the same way the writing +--echo # does. A document goes in as it stands whether it arrived wrapped or +--echo # not, so it costs what a document costs; read as a plain string it +--echo # would be charged for the two quote characters that the writing is +--echo # never going to put round it. Both columns below are declared the +--echo # same width, +--echo # and hold the same answer - the miscounting was only ever room asked +--echo # for and not taken, so no value was ever cut short by it. +--echo # +CREATE TABLE t4 (v VARCHAR(20) CHARACTER SET latin1, + u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t4 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); +CREATE TABLE t7 (v VARCHAR(20) CHARACTER SET utf8mb4, + u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t7 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); + +CREATE TABLE m1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t4; +SHOW CREATE TABLE m1; +CREATE TABLE s1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t7; +SHOW CREATE TABLE s1; +SELECT r FROM m1 ORDER BY r; +SELECT r FROM s1 ORDER BY r; +DROP TABLE m1, s1; + +--echo # +--echo # An object is built by the same arithmetic and moves the same way. +--echo # +CREATE TABLE m2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r + FROM t4; +SHOW CREATE TABLE m2; +CREATE TABLE s2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r + FROM t7; +SHOW CREATE TABLE s2; +DROP TABLE m2, s2; + +--echo # +--echo # THE CONTROL: the same two columns through a function that EDITS a +--echo # document. Those take the set of the document they were given rather +--echo # than aggregating over their arguments, so no wrapper is built and +--echo # there is nothing to see through. Both are declared the same width in +--echo # both worlds, which is what says the widths above move because of the +--echo # wrapping and not because two sets are in play. +--echo # +CREATE TABLE m3 AS + SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r + FROM t4; +SHOW CREATE TABLE m3; +CREATE TABLE s3 AS + SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r + FROM t7; +SHOW CREATE TABLE s3; +SELECT r FROM m3 ORDER BY r; +SELECT r FROM s3 ORDER BY r; +DROP TABLE m3, s3; +DROP TABLE t4, t7; + +SET @@optimizer_switch= @save_optimizer_switch; +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/type_json.result b/mysql-test/main/type_json.result index 5c7e4a9718366..362355f222169 100644 --- a/mysql-test/main/type_json.result +++ b/mysql-test/main/type_json.result @@ -57,7 +57,7 @@ show create table t2; Table Create Table t2 CREATE TABLE `t2` ( `a` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`a`)), - `t` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + `t` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci show create table t3; Table Create Table @@ -155,7 +155,7 @@ DROP TABLE t1; # SELECT json_object('a', (SELECT json_objectagg(b, c) FROM (SELECT 'b','c') d)) AS j FROM DUAL; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def j 250 (format=json) 9437310 16 Y 0 39 33 +def j 250 (format=json) 18874494 16 Y 0 39 33 j {"a": {"b":"c"}} # diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 489beedc04f3b..bf96f9d1fe2a3 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -1659,10 +1659,23 @@ bool Item_func_json_quote::fix_length_and_dec(THD *thd) { collation.set(&my_charset_utf8mb4_bin); /* - Odd but realistic worst case is when all characters - of the argument turn into '\uXXXX\uXXXX', which is 12. + Odd but realistic worst case is when every character of the argument + has to be escaped, and six is what one of them costs. + + A character is written as the hex of its UTF-16 form, one unit for a + character of the first plane and two for any other, each unit taking + a `\uXXXX` of its own - so twelve where a pair is needed. A pair is + only ever needed for a character the set being written into cannot + carry, and the set here is utf8mb4, which carries every character + there is. So nothing is escaped for want of somewhere to put it and + what is left is the handful JSON will not have literally: a quote, a + backslash, and the control characters, all of them inside the first + plane and none of them costing more than six. + + The two for the quotes around the whole of it are this function's + own. */ - fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 12 + 2); + fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 6 + 2); return FALSE; } @@ -3538,6 +3551,68 @@ static json_append_result append_json_keyname(String *str, Item *item, } +/* + How much room a value needs in a document being built, before any + value has been seen and so out of what the argument says about itself. + + A value that is already a document goes in as it stands, but it is + written out again with the rest and a space arrives after every + separator it brought with it, so the room for it covers the writing + the same way JSON_REMOVE asks for it to. One written as text and not + a document is written as a JSON string, and twice its characters is + what is asked for that. + + Twice does not cover an escaping: a character written out as the hex + of its UTF-16 form costs six, and twelve where the character needs a + pair of them, so a value escaped throughout is asked for too little + and is cut. What it costs turns on the character set being written + into as well as the value's own, neither of them alone, and pricing + it properly widens what these functions declare enough to move a + result from a memory temporary table onto disk. That is left to be + done together with the work that stops a declared width deciding + where a temporary table lives. + + A string carries no separators of its own, so the spacing has nothing + to add to it. A boolean is written as one of the two words 'true' and + 'false', the longer of which is five characters. What is left is a + number, which is written as itself. A value that is not there at all + is written as 'null', so nothing is ever shorter than four characters. + + The punctuation that goes around it is the caller's to ask for. + + Whether it IS a document has to be the question the writing asks, and + not a shorter one that happens to agree most of the time. Aggregating + the arguments into one character set wraps whichever of them has to + move, and the wrapper attests to itself when asked for a type handler + - so a document that arrived wrapped would be priced as a string and + charged for quotes that the writing, which looks through the wrapper, + is never going to put round it. is_json_type() is that same look. +*/ +static ulonglong json_value_reserve(Item *arg) +{ + /* + A document is a string: every type handler in the JSON collection is + built over a string one. An argument of any other result type is + not one and its own type has said so, so the look through the + wrappers round it is only made where it can come back yes. + */ + const bool is_string= arg->result_type() == STRING_RESULT; + const bool is_document= is_string && is_json_type(arg); + ulonglong length; + + if (is_string && !is_document) + length= static_cast(arg->max_char_length()) * 2 + 2; + else if (arg->type_handler()->is_bool_type()) + length= 5; + else if (is_document) + length= static_cast(arg->max_char_length()) * 2; + else + length= arg->max_char_length(); + + return length < 4 ? 4 : length; +} + + bool Item_func_json_array::fix_length_and_dec(THD *thd) { ulonglong char_length= 2; @@ -3559,23 +3634,7 @@ bool Item_func_json_array::fix_length_and_dec(THD *thd) return TRUE; for (n_arg=0 ; n_arg < arg_count ; n_arg++) - { - ulonglong arg_length; - Item *arg= args[n_arg]; - - if (arg->result_type() == STRING_RESULT && - !Type_handler_json_common::is_json_type_handler(arg->type_handler())) - arg_length= arg->max_char_length() * 2; /*escaping possible */ - else if (arg->type_handler()->is_bool_type()) - arg_length= 5; - else - arg_length= arg->max_char_length(); - - if (arg_length < 4) - arg_length= 4; /* can be 'null' */ - - char_length+= arg_length + 4; - } + char_length+= json_value_reserve(args[n_arg]) + 4; fix_char_length_ulonglong(char_length); tmp_val.set_charset(collation.collation); @@ -3657,13 +3716,18 @@ bool Item_func_json_array_append::fix_length_and_dec(THD *thd) ulonglong char_length; collation.set(args[0]->collation); - char_length= args[0]->max_char_length(); + /* + The document is written out again around what is added to it, a + space arriving after every separator that is copied, so the room for + it has to cover the writing and not only the reading - the same + allowance JSON_REMOVE asks for, adding the same spacing. + */ + char_length= static_cast(args[0]->max_char_length()) * 2; for (n_arg= 1; n_arg < arg_count; n_arg+= 2) { paths[n_arg/2].set_constant_flag(args[n_arg]->const_item()); - char_length+= - static_cast(args[n_arg+1]->max_char_length()) + 4; + char_length+= json_value_reserve(args[n_arg+1]) + 4; } fix_char_length_ulonglong(char_length); @@ -5420,7 +5484,13 @@ bool Item_func_json_insert::fix_length_and_dec(THD *thd) JSON_DO_PAUSE_EXECUTION(thd, 0.0002); collation.set(args[0]->collation); - char_length= args[0]->max_char_length(); + /* + The document is written out again around what is put into it, a + space arriving after every separator that is copied, so the room for + it has to cover the writing and not only the reading - the same + allowance JSON_REMOVE asks for, adding the same spacing. + */ + char_length= static_cast(args[0]->max_char_length()) * 2; for (n_arg= 1; n_arg < arg_count; n_arg+= 2) { @@ -5429,8 +5499,8 @@ bool Item_func_json_insert::fix_length_and_dec(THD *thd) In the resulting JSON we can insert the property name from the path, and the value itself. */ - char_length+= args[n_arg/2]->max_char_length() + 6; - char_length+= args[n_arg/2+1]->max_char_length() + 4; + char_length+= static_cast(args[n_arg]->max_char_length()) + 6; + char_length+= json_value_reserve(args[n_arg+1]) + 4; } fix_char_length_ulonglong(char_length); @@ -5885,7 +5955,15 @@ String *Item_func_json_insert::val_str(String *str) bool Item_func_json_remove::fix_length_and_dec(THD *thd) { collation.set(args[0]->collation); - max_length= args[0]->max_length; + /* + What is left of the document is written out again, with a space after + every separator that is copied, so what comes back can be longer than + what went in even though something was taken out of it. A separator + has a value on either side of it and the shortest value is one + character, so at worst every second character gains one - which is + the allowance JSON_LOOSE asks for, adding the same spacing. + */ + fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 2); mark_constant_paths(paths, args+1, arg_count-1); set_maybe_null(); From bce094403a24851cb5748cb54abb576037f0a9af Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Thu, 6 Aug 2026 19:07:16 -0400 Subject: [PATCH 15/30] MDEV-40659 Start a Copy_field with no fields rather than with rubbish `Copy_field` has two `set()` overloads and only one of them records the fields it works between. `set(Field*, Field*, bool)` records both, but returns early for a `MYSQL_TYPE_NULL` destination without recording either; `set(uchar*, Field*)`, which copies a field into a string buffer, has no destination field to record and records neither. Entries of every kind go in the same array, and callers walk that array with a single loop. So anything that asks an entry which fields it works between reads whatever the memory the array was built on happened to hold, and `Copy_field() = default` left `from_field` and `to_field` indeterminate for it to read. Give the class a constructor that starts the two absent, so that an entry that was never told about a field says so. Nothing in the server asks today - the only readers are the copying functions that the field to field overload installs, and those are reached only for the entries that overload filled in - so this changes no behaviour. The unit test builds an array the way the server builds one, on memory known not to be zero, and reads the two members. --- sql/field.h | 9 ++- unittest/sql/CMakeLists.txt | 4 ++ unittest/sql/copy_field-t.cc | 110 +++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 unittest/sql/copy_field-t.cc diff --git a/sql/field.h b/sql/field.h index 4250886781d72..1ac6d4b6a7cba 100644 --- a/sql/field.h +++ b/sql/field.h @@ -5943,7 +5943,14 @@ class Copy_field :public Sql_alloc { Field *from_field,*to_field; String tmp; // For items - Copy_field() = default; + /* + set(uchar*, Field*) has no destination field to record and records + neither field, and set(Field*, Field*, bool) returns early for a + MYSQL_TYPE_NULL destination without recording them either. Both kinds + of entry go in the same array, so start them absent rather than + leaving them to read as whatever the memory held. + */ + Copy_field() : from_field(NULL), to_field(NULL) {} ~Copy_field() = default; void set(Field *to,Field *from,bool save); // Field to field void set(uchar *to,Field *from); // Field to string diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index b8682de74c304..6f216a4dfbad6 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -34,3 +34,7 @@ MY_ADD_TEST(mf_iocache) ADD_EXECUTABLE(my_json_writer-t my_json_writer-t.cc dummy_builtins.cc) TARGET_LINK_LIBRARIES(my_json_writer-t sql mytap) MY_ADD_TEST(my_json_writer) + +ADD_EXECUTABLE(copy_field-t copy_field-t.cc dummy_builtins.cc) +TARGET_LINK_LIBRARIES(copy_field-t sql mytap) +MY_ADD_TEST(copy_field) diff --git a/unittest/sql/copy_field-t.cc b/unittest/sql/copy_field-t.cc new file mode 100644 index 0000000000000..7bd46722f0c2b --- /dev/null +++ b/unittest/sql/copy_field-t.cc @@ -0,0 +1,110 @@ +/* + Copyright (c) 2026, MariaDB Foundation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +/** + Unit test for the state a Copy_field starts life in. + + Copy_field has two set() overloads. The field to field one records the + two fields it works between; the field to string one has no destination + field to record and records neither, and set(Field*, Field*, bool) + returns early for a MYSQL_TYPE_NULL destination without recording them + either. Entries of both kinds are put in one array, which callers walk + with a single loop, so from_field and to_field have to read as absent + where they were never filled in rather than as whatever the memory the + array was built on happened to hold. +*/ + +#include +#include +#include +#include +#include + +/* + Any byte but zero: memory the constructor leaves alone must be + distinguishable from memory it cleared. +*/ +static const int POISON= 0xAB; + + +/** + Construct over memory known to be dirty. +*/ + +static void test_construction_over_dirty_memory() +{ + union + { + char bytes[2 * sizeof(Copy_field)]; + longlong align; + } storage; + Copy_field *first, *second; + + memset(storage.bytes, POISON, sizeof(storage.bytes)); + first= ::new ((void*) storage.bytes) Copy_field; + second= ::new ((void*) (storage.bytes + sizeof(Copy_field))) Copy_field; + + ok(first->from_field == NULL, "first from_field starts absent"); + ok(first->to_field == NULL, "first to_field starts absent"); + ok(second->from_field == NULL, "second from_field starts absent"); + ok(second->to_field == NULL, "second to_field starts absent"); + + second->~Copy_field(); + first->~Copy_field(); +} + + +/** + Build an array the way the server builds one, on a MEM_ROOT. + + The root is dirtied and its blocks marked free rather than released, so + the array is built on bytes that are known not to be zero. +*/ + +static void test_array_on_mem_root() +{ + const uint count= 3; + const size_t dirty= count * sizeof(Copy_field) + 64; + MEM_ROOT mem_root; + Copy_field *copy; + uint i; + + init_alloc_root(PSI_NOT_INSTRUMENTED, &mem_root, 4096, 0, MYF(0)); + memset(alloc_root(&mem_root, dirty), POISON, dirty); + free_root(&mem_root, MYF(MY_MARK_BLOCKS_FREE)); + + copy= new (&mem_root) Copy_field[count]; + for (i= 0; i < count; i++) + { + ok(copy[i].from_field == NULL, "copy[%u] from_field starts absent", i); + ok(copy[i].to_field == NULL, "copy[%u] to_field starts absent", i); + } + free_root(&mem_root, MYF(0)); +} + + +int main(int argc __attribute__((unused)), char *argv[]) +{ + MY_INIT(argv[0]); + plan(10); + + test_construction_over_dirty_memory(); + test_array_on_mem_root(); + + my_end(0); + return exit_status(); +} From 870cfc532d1460d979a8ea3fad93d2657a467d48 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Thu, 6 Aug 2026 19:07:33 -0400 Subject: [PATCH 16/30] MDEV-40660 Type a column JSON only on a check about that column Whether a string column is a JSON column is decided by whether it carries a `JSON_VALID` check constraint. The test did not ask what that constraint reads, so `CHECK (json_valid(other_column))` typed this column, and so did `CHECK (json_valid('{}'))`, which is a constant true and constrains nothing whatsoever. The column the constraint does protect is left an ordinary column, so the type lands on the other one of the pair. Being typed JSON is what puts a column's contents into a document verbatim instead of quoted and escaped. A column that keeps no promise about its contents was therefore spliced, and SELECT JSON_SET('{}', '$.k', a) FROM t1; returned NULL for text that is not a document where an ordinary column would have given `{"k": "not json at all"}`. Ask the question the callers need: does this column carry a check constraint that says THIS column holds a document. The predicate that asks it already existed for deciding which checks a written document lets stand; the two now share one test of the expression, which the frm reader wraps in the further question of whether the check belongs to the column rather than to the table. THIS COLUMN is same table and same position, and not the same `Field` object, because a table has more than one `Field` for one column. The row images a trigger reads through `OLD.` and `NEW.` are reached through a second and a third set of them, made by `Field::make_new_field()`, which memdups the field and points the copy at the other record buffer; the copy carries the same `check_constraint` pointer, to an expression still naming the field it was fixed against. Asked of the object, such a copy answers that its check is about some other column and stops being a JSON column, so CREATE TABLE t (id INT, j JSON); CREATE TRIGGER tr BEFORE UPDATE ON t FOR EACH ROW INSERT INTO lg VALUES (JSON_SET('{}', '$.k', OLD.j)); logs `{"k": "{\"a\":1}"}` - the column's contents quoted into the document - where a released server logs `{"k": {"a": 1}}`. Which copies exist is not symmetric, so the cases that would break are not the ones a reader would guess: the `OLD.` buffer is built for any update or delete trigger and copies every column, while the `NEW.` buffer is built only where some column of the table cannot be null and copies only the columns that cannot, and `OLD.` in a delete trigger reads the table's own fields. Two integers give nothing away. A column's check can only read columns of its own table, so a check naming another position is a check about another column - which is the thing this exists to refuse, and which a check over a neighbouring column still gets - and one naming the same position in another table is a copy that was put somewhere else, where the constraint it brought along says nothing about where it landed. Rebinding the constraint on the copy would answer it too, and is not done: that needs a fresh `Virtual_column_info` and a fresh item tree for every copied field, on a path every temporary table goes through, to say what two integers already say. The declared type was decided a second time and separately, in `Field_longstr::make_send_field()`, which asked the constraint expression to name a format through a virtual of its own. That walk understood a conjunction, so `CHECK (length(js) > 0 AND json_valid(js))` named `format=json` where the question that types the column answers no - the client being told a column holds documents that the server quotes into them. It asks `has_json_valid_constraint()`, which is the predicate `Field_string::type_handler()` just below already asks, and the virtual goes with it: the declaration on `Item`, the descent `Item_cond_and` made through a conjunction's parts, and the answer `Item_func_json_valid` gave. What a client is told about a column is what the column is. Every column that stops being reported as a document column is one the server already warns about where it is defined, in the warning that says the check asks more than the call of it, so the protocol says to the client what the writer of the table was told to their face. That is a change to how a released server behaves rather than to anything introduced here, and it is worth saying so plainly: a column with a check of that shape has been advertised as a document column for years, and a connector reading the metadata to decide whether to parse a value will stop parsing values out of it. The other producer of the format is untouched - a real `JSON` column and every expression result are named by the type handler and never came this way - which three recorded rows say: the shape with the call alone keeps the format beside the shape that loses it, and the file of expression results does not move at all. Both questions about a `JSON_VALID` call - the one that compares a `Field` and the one that compares a name - are that call over a single column argument, plus a way of naming the column. The shape is `json_valid_single_field_arg()` and is asked once. Whether the argument is followed through `Item::real_item()` is a flag it takes rather than the difference between two copies of a walk: it is followed where the expression has been fixed, and where it has not there is nothing yet to follow. The walk over a conjunction's parts that is left is not the one that went. It decides which of two warnings a refused constraint is given, and it stops at a conjunction on purpose: one that descended everywhere would call a constraint that can pass without the call holding a constraint that requires it. Writing `JSON_VALID` in a column's check constraint is how a JSON column is asked for, so a constraint that mentions it and leaves the column ordinary is worth saying out loud rather than leaving to be found when the column reaches a document. Three warnings, at the time the column is defined, where the expression is not fixed yet and the column can only be recognised by its name: 4269 the call reads something other than this column 4270 the constraint can pass without the call holding, as under OR 4271 the call is about this column and holds, and something stands beside it, as under AND The numbers start above every one the later series have given out. A number is a promise to whoever reads it - `mysql_errno()` is what a connector keys off - and numbering here is positional, an error's number being how many entries stand before it, so an entry appended to a series whose tail was frozen at release takes a number a later series has already spent. A second `start-error-number` is how this file already opts out of position-derived numbering, and it starts these three at 4269, one above the highest given out anywhere. Which of the three is said is decided by the same question the typing asks, so a check that mentions the call and is refused all the same is always told about. The last of them is the commonest shape of the three and is the one the promise is kept in: a conjunction cannot pass unless every part of it does, so `CHECK (json_valid(j) AND id > 0)` really does require the call, and the column is still ordinary text. A check constraint the server writes itself, for a `JSON` column or for the columns of a temporary table, reads the column it belongs to, so those stay JSON and say nothing. A table level constraint never typed a column and still does not. The checking itself is unchanged: a constraint naming another column is evaluated and enforced against that column exactly as written. A column that was typed by a check over another column had its contents spliced into documents; they are quoted now. Where the column held text that is not a document that turns NULL into the quoted text, and where it held a well formed document that turns a nested document into a string. Both follow from the column never having been a JSON column, and the warnings say which columns those are. The constraint shapes that are not a bare check of one column are recorded with what each of them accepts and refuses, the nested one among them: an `AND` whose parts must all hold, but whose call sits inside an `OR` one level down, so the constraint can pass without the call holding and the column stays ordinary text. `func_json_valid_constraint` reads a JSON column through `OLD.` and `NEW.` in update and insert triggers, nullable and not-null, with the delete trigger and the two direct reads beside them to say which of the seven go through a copy at all. Nothing in the suite had read a JSON column that way. Every answer recorded there is the one a released 10.11 server gives, taken off that server rather than off this one. --- .../main/func_json_check_scan_count.result | 4 + mysql-test/main/func_json_check_skip.result | 4 + mysql-test/main/func_json_columns.result | 28 +- mysql-test/main/func_json_columns.test | 13 +- .../main/func_json_valid_constraint.result | 271 ++++++++++++++++++ .../main/func_json_valid_constraint.test | 216 ++++++++++++++ mysql-test/main/mysql-metadata.result | 5 +- mysql-test/main/type_json.result | 5 +- sql/field.cc | 35 ++- sql/item.h | 15 +- sql/item_cmpfunc.cc | 15 - sql/item_cmpfunc.h | 2 - sql/item_jsonfunc.h | 7 +- sql/share/errmsg-utf8.txt | 10 + sql/sql_type_json.cc | 217 ++++++++++++-- sql/sql_type_json.h | 7 + 16 files changed, 783 insertions(+), 71 deletions(-) create mode 100644 mysql-test/main/func_json_valid_constraint.result create mode 100644 mysql-test/main/func_json_valid_constraint.test diff --git a/mysql-test/main/func_json_check_scan_count.result b/mysql-test/main/func_json_check_scan_count.result index 37322d92ffdaf..bc6dbbbba4423 100644 --- a/mysql-test/main/func_json_check_scan_count.result +++ b/mysql-test/main/func_json_check_scan_count.result @@ -140,6 +140,8 @@ DROP TABLE t1; # the check keeps reading: one reading, and it stays. # CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column FLUSH STATUS; INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)); SHOW STATUS LIKE 'Json_scans'; @@ -152,6 +154,8 @@ DROP TABLE t1; # alone - one per row, and both stay. # CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 40)); +Warnings: +Warning 4271 CHECK constraint of column 'j' asks more than JSON_VALID() of it; the column is not a JSON column FLUSH STATUS; INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_OBJECT('b', 2)); SHOW STATUS LIKE 'Json_scans'; diff --git a/mysql-test/main/func_json_check_skip.result b/mysql-test/main/func_json_check_skip.result index 95b587a4e71d9..05c90d9c2c721 100644 --- a/mysql-test/main/func_json_check_skip.result +++ b/mysql-test/main/func_json_check_skip.result @@ -137,6 +137,8 @@ DROP TABLE t1; # check, however much it looks like one. # CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column # a is given a document, b is not; the check reads b and refuses INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), 'not a document'); ERROR 23000: CONSTRAINT `t1.a` failed for `test`.`t1` @@ -155,6 +157,8 @@ DROP TABLE t1; # document keeps asking it. # CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 12)); +Warnings: +Warning 4271 CHECK constraint of column 'j' asks more than JSON_VALID() of it; the column is not a JSON column INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); INSERT INTO t1 VALUES (JSON_OBJECT('aaaaaaaaaa', 1)); ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` diff --git a/mysql-test/main/func_json_columns.result b/mysql-test/main/func_json_columns.result index 17ed0d5b50039..ebdfb8e2019ab 100644 --- a/mysql-test/main/func_json_columns.result +++ b/mysql-test/main/func_json_columns.result @@ -54,6 +54,8 @@ t3 CREATE TABLE `t3` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci # a constraint that mentions the column inside a larger expression CREATE TABLE t4 (id INT, j LONGTEXT CHECK (json_valid(j) AND id > 0)); +Warnings: +Warning 4271 CHECK constraint of column 'j' asks more than JSON_VALID() of it; the column is not a JSON column SHOW CREATE TABLE t4; Table Create Table t4 CREATE TABLE `t4` ( @@ -116,6 +118,8 @@ id j k 1 {"a":1} {"b":2} # an OR, where the other side can let an invalid document through CREATE TABLE t5b (id INT, j LONGTEXT CHECK (json_valid(j) OR id > 0)); +Warnings: +Warning 4270 CHECK constraint of column 'j' can pass without JSON_VALID() holding; the column is not a JSON column INSERT INTO t5b VALUES (1, '{'); Warnings: Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' @@ -126,8 +130,30 @@ id j valid 1 { 0 Warnings: Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# an AND whose parts all have to hold, but where none of them is +# the call: the promise is inside an OR one level down, so the +# constraint can pass without it and the column stays ordinary +CREATE TABLE t5d (id INT, +j LONGTEXT CHECK (LENGTH(j) > 0 AND +(json_valid(j) OR id > 0))); +Warnings: +Warning 4270 CHECK constraint of column 'j' can pass without JSON_VALID() holding; the column is not a JSON column +INSERT INTO t5d VALUES (1, '{'); +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +INSERT INTO t5d VALUES (0, '{'); +ERROR 23000: CONSTRAINT `t5d.j` failed for `test`.`t5d` +INSERT INTO t5d VALUES (0, '{"a":1}'); +SELECT id, j, JSON_VALID(j) AS valid FROM t5d ORDER BY id, j; +id j valid +0 {"a":1} 1 +1 { 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' # a constraint attached to one column that tests a different column CREATE TABLE t5c (a LONGTEXT, b LONGTEXT CHECK (json_valid(a))); +Warnings: +Warning 4269 CHECK constraint of column 'b' calls JSON_VALID() on something other than 'b'; the column is not a JSON column INSERT INTO t5c VALUES ('{"a":1}', '{'); INSERT INTO t5c VALUES ('{', '{"a":1}'); ERROR 23000: CONSTRAINT `t5c.b` failed for `test`.`t5c` @@ -157,7 +183,7 @@ FROM t5e ORDER BY j; v still_valid ["{"] 1 ["{\"a\":1}"] 1 -DROP TABLE t5b, t5c, t5e; +DROP TABLE t5b, t5c, t5d, t5e; # # 3. Switching the check off, which is what makes a stored document # unverifiable afterwards. diff --git a/mysql-test/main/func_json_columns.test b/mysql-test/main/func_json_columns.test index a8ca999345499..9b8e1b54537b4 100644 --- a/mysql-test/main/func_json_columns.test +++ b/mysql-test/main/func_json_columns.test @@ -87,6 +87,17 @@ INSERT INTO t5b VALUES (1, '{'); --error ER_CONSTRAINT_FAILED INSERT INTO t5b VALUES (0, '{'); SELECT id, j, JSON_VALID(j) AS valid FROM t5b; +--echo # an AND whose parts all have to hold, but where none of them is +--echo # the call: the promise is inside an OR one level down, so the +--echo # constraint can pass without it and the column stays ordinary +CREATE TABLE t5d (id INT, + j LONGTEXT CHECK (LENGTH(j) > 0 AND + (json_valid(j) OR id > 0))); +INSERT INTO t5d VALUES (1, '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5d VALUES (0, '{'); +INSERT INTO t5d VALUES (0, '{"a":1}'); +SELECT id, j, JSON_VALID(j) AS valid FROM t5d ORDER BY id, j; --echo # a constraint attached to one column that tests a different column CREATE TABLE t5c (a LONGTEXT, b LONGTEXT CHECK (json_valid(a))); INSERT INTO t5c VALUES ('{"a":1}', '{'); @@ -104,7 +115,7 @@ INSERT INTO t5e VALUES (''); SELECT j, JSON_VALID(j) AS valid FROM t5e ORDER BY j; SELECT JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t5e ORDER BY j; -DROP TABLE t5b, t5c, t5e; +DROP TABLE t5b, t5c, t5d, t5e; --echo # --echo # 3. Switching the check off, which is what makes a stored document diff --git a/mysql-test/main/func_json_valid_constraint.result b/mysql-test/main/func_json_valid_constraint.result new file mode 100644 index 0000000000000..b3075dc614aea --- /dev/null +++ b/mysql-test/main/func_json_valid_constraint.result @@ -0,0 +1,271 @@ +# +# Which column a JSON_VALID check constraint makes a JSON column +# +# A string column is a JSON column when it carries a check constraint +# that says that column holds a document. A constraint that reads +# another column, or no column at all, says nothing about this one. +# +SET NAMES utf8mb4; +# +# A constraint reading another column +# +# d is the ordinary usage and e an ordinary JSON column, both controls. +CREATE TABLE t1 ( +a VARCHAR(100) CHECK (JSON_VALID(b)), +b VARCHAR(100), +d VARCHAR(100) CHECK (JSON_VALID(d)), +e JSON +); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(100) DEFAULT NULL CHECK (json_valid(`b`)), + `b` varchar(100) DEFAULT NULL, + `d` varchar(100) DEFAULT NULL CHECK (json_valid(`d`)), + `e` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`e`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# what the server tells a client the columns are +SELECT a, b, d, e FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t1 t1 a a 253 400 0 Y 0 0 45 +def test t1 t1 b b 253 400 0 Y 0 0 45 +def test t1 t1 d d 253 (format=json) 400 0 Y 0 0 45 +def test t1 t1 e e 252 (format=json) 4294967295 0 Y 144 0 45 +a b d e +SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1' ORDER BY COLUMN_NAME; +COLUMN_NAME DATA_TYPE +a varchar +b varchar +d varchar +e longtext +# +# a is unconstrained, so it can hold text that is not a document +# +INSERT INTO t1 VALUES ('not json at all', '{"x":1}', '{"y":2}', '{"z":3}'); +SELECT a, JSON_VALID(a) AS a_is_a_document FROM t1; +a a_is_a_document +not json at all 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# a is quoted like the ordinary column it is, not spliced +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t1; +from_a +{"k": "not json at all"} +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t1; +from_b +{"k": "{\"x\":1}"} +# the controls are spliced +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t1; +from_d +{"k": {"y": 2}} +SELECT JSON_SET('{}', '$.k', e) AS from_e FROM t1; +from_e +{"k": {"z": 3}} +DROP TABLE t1; +# +# The same column holding a document that IS well formed +# +# Whether a is spliced or quoted decides whether the document nests +# or arrives as a string, and it must not depend on b's constraint. +# +CREATE TABLE t2 ( +a VARCHAR(100) CHECK (JSON_VALID(b)), +b VARCHAR(100), +d VARCHAR(100) CHECK (JSON_VALID(d)) +); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +INSERT INTO t2 VALUES ('{"p":1}', '{"x":1}', '{"p":1}'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t2; +from_a +{"k": "{\"p\":1}"} +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t2; +from_b +{"k": "{\"x\":1}"} +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t2; +from_d +{"k": {"p": 1}} +DROP TABLE t2; +# +# A constraint that constrains nothing +# +# JSON_VALID('{}') is a constant true, so the column keeps no +# promise about its contents whatsoever. +# +CREATE TABLE t3 (a VARCHAR(100) CHECK (JSON_VALID('{}'))); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +SELECT a FROM t3; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t3 t3 a a 253 400 0 Y 0 0 45 +a +INSERT INTO t3 VALUES ('not json at all'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t3; +from_a +{"k": "not json at all"} +DROP TABLE t3; +# +# The constraint is still enforced, on the column it names +# +CREATE TABLE t4 (a VARCHAR(100) CHECK (JSON_VALID(b)), b VARCHAR(100)); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +INSERT INTO t4 VALUES ('anything', 'not json'); +ERROR 23000: CONSTRAINT `t4.a` failed for `test`.`t4` +INSERT INTO t4 VALUES ('anything', '{"x":1}'); +SELECT a, b FROM t4; +a b +anything {"x":1} +DROP TABLE t4; +# +# A table level constraint types neither column, as before +# +CREATE TABLE t5 ( +a VARCHAR(100), +b VARCHAR(100), +CONSTRAINT c CHECK (JSON_VALID(b)) +); +SELECT a, b FROM t5; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t5 t5 a a 253 400 0 Y 0 0 45 +def test t5 t5 b b 253 400 0 Y 0 0 45 +a b +DROP TABLE t5; +# +# A check that asks more than whether the column holds a document +# +# What types a column is JSON_VALID() of it and nothing else. An +# AND keeps the promise and an OR does not, but a promise kept is +# not a column typed, and neither shape is typed. Each is told so, +# in the words that fit it. +# +CREATE TABLE t7 ( +a TEXT CHECK (LENGTH(a) > 0 AND JSON_VALID(a)), +b TEXT CHECK (LENGTH(b) > 0 OR JSON_VALID(b)), +c TEXT CHECK (LENGTH(d) > 0 AND JSON_VALID(d)), +d TEXT +); +Warnings: +Warning 4271 CHECK constraint of column 'a' asks more than JSON_VALID() of it; the column is not a JSON column +Warning 4270 CHECK constraint of column 'b' can pass without JSON_VALID() holding; the column is not a JSON column +Warning 4269 CHECK constraint of column 'c' calls JSON_VALID() on something other than 'c'; the column is not a JSON column +CREATE TABLE t8 (d TEXT CHECK (JSON_VALID(d))); +# the protocol tells the client what the warnings told the writer: +# none of the three is a JSON column, and a column whose check is +# the call alone still is +SELECT a, b, c, d FROM t7; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t7 t7 a a 252 262140 0 Y 16 0 45 +def test t7 t7 b b 252 262140 0 Y 16 0 45 +def test t7 t7 c c 252 262140 0 Y 16 0 45 +def test t7 t7 d d 252 262140 0 Y 16 0 45 +a b c d +SELECT d FROM t8; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t8 t8 d d 252 (format=json) 262140 0 Y 16 0 45 +d +# and what each does with a document says the same: quoted, where a +# column whose check is the call alone is spliced +INSERT INTO t7 VALUES ('{"p":1}', '{"p":1}', '{"p":1}', '{"p":1}'), +('[1,2]', '[1,2]', '[1,2]', '[1,2]'); +INSERT INTO t8 VALUES ('{"p":1}'), ('[1,2]'); +SELECT JSON_SET('{}', '$.k', a) AS from_and_shape FROM t7; +from_and_shape +{"k": "{\"p\":1}"} +{"k": "[1,2]"} +SELECT JSON_SET('{}', '$.k', d) AS from_call_alone FROM t8; +from_call_alone +{"k": {"p": 1}} +{"k": [1, 2]} +DROP TABLE t7, t8; +# +# A column the server itself wrote the constraint for stays JSON +# +# Both a declared JSON column and the columns of a temporary table +# built for grouping carry a constraint over themselves. +# +CREATE TABLE t6 (j JSON, s VARCHAR(100) CHECK (JSON_VALID(s))); +INSERT INTO t6 VALUES ('{"x":1}', '{"y":2}'), ('{"x":3}', '{"y":4}'); +SELECT j, s FROM t6 GROUP BY j, s; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t6 t6 j j 252 (format=json) 4294967295 7 Y 32912 0 45 +def test t6 t6 s s 253 (format=json) 400 7 Y 32768 0 45 +j s +{"x":1} {"y":2} +{"x":3} {"y":4} +SELECT JSON_SET('{}', '$.k', j) AS from_j, +JSON_SET('{}', '$.k', s) AS from_s +FROM (SELECT j, s FROM t6 GROUP BY j, s) AS grouped; +from_j from_s +{"k": {"x": 1}} {"k": {"y": 2}} +{"k": {"x": 3}} {"k": {"y": 4}} +SELECT JSON_ARRAYAGG(j) AS agg_j, JSON_ARRAYAGG(s) AS agg_s FROM t6; +agg_j agg_s +[{"x":1},{"x":3}] [{"y":2},{"y":4}] +DROP TABLE t6; +# +# A row image a trigger reads through OLD. or NEW. is reached +# through a SECOND set of field objects, copied from the table's +# own so that they can point at the other record buffer. A copy is +# the same column of the same table and carries the same check, so +# it is the same JSON column, and what it holds goes into a +# document as a document. +# +# Which columns get copied is not the same on the two sides. The +# OLD. buffer is built whenever there is an update or delete +# trigger and copies every column; the NEW. buffer is built only +# where some column of the table cannot be null, and copies only +# the columns that cannot be. So the five cases below that read a +# copy are not the five a reader would guess, and the two that +# read the table's own fields are here to say which is which. +# +# Every answer below is what a released server gives. +# +CREATE TABLE lg (tag VARCHAR(24), v TEXT); +CREATE TABLE t7 (id INT, j JSON); +CREATE TABLE t8 (id INT, j JSON NOT NULL); +INSERT INTO t7 VALUES (1, '{"a":1}'); +INSERT INTO t8 VALUES (1, '{"a":1}'); +CREATE TRIGGER tr1 BEFORE UPDATE ON t7 FOR EACH ROW +INSERT INTO lg VALUES ('nullable OLD upd', JSON_SET('{}','$.k',OLD.j)), +('nullable NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr2 BEFORE UPDATE ON t8 FOR EACH ROW +INSERT INTO lg VALUES ('notnull OLD upd', JSON_SET('{}','$.k',OLD.j)), +('notnull NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr3 BEFORE INSERT ON t7 FOR EACH ROW +INSERT INTO lg VALUES ('nullable NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr4 BEFORE INSERT ON t8 FOR EACH ROW +INSERT INTO lg VALUES ('notnull NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr5 BEFORE DELETE ON t7 FOR EACH ROW +INSERT INTO lg VALUES ('nullable OLD del', JSON_SET('{}','$.k',OLD.j)); +UPDATE t7 SET id = 2; +UPDATE t8 SET id = 2; +INSERT INTO t7 VALUES (3, '{"b":2}'); +INSERT INTO t8 VALUES (3, '{"b":2}'); +DELETE FROM t7 WHERE id = 3; +SELECT tag, v FROM lg ORDER BY tag, v; +tag v +notnull NEW ins {"k": {"b": 2}} +notnull NEW upd {"k": {"a": 1}} +notnull OLD upd {"k": {"a": 1}} +nullable NEW ins {"k": {"b": 2}} +nullable NEW upd {"k": {"a": 1}} +nullable OLD del {"k": {"b": 2}} +nullable OLD upd {"k": {"a": 1}} +# +# THE CONTROLS: the same two columns read through the table's own +# fields, where there is no copy in the way. These were never in +# question, and they say that the answers above are about the +# copying and not about the columns. +# +SELECT JSON_SET('{}','$.k',j) AS direct_nullable FROM t7 ORDER BY 1; +direct_nullable +{"k": {"a": 1}} +SELECT JSON_SET('{}','$.k',j) AS direct_notnull FROM t8 ORDER BY 1; +direct_notnull +{"k": {"a": 1}} +{"k": {"b": 2}} +DROP TABLE lg, t7, t8; diff --git a/mysql-test/main/func_json_valid_constraint.test b/mysql-test/main/func_json_valid_constraint.test new file mode 100644 index 0000000000000..ac076cb71f17c --- /dev/null +++ b/mysql-test/main/func_json_valid_constraint.test @@ -0,0 +1,216 @@ +--echo # +--echo # Which column a JSON_VALID check constraint makes a JSON column +--echo # +--echo # A string column is a JSON column when it carries a check constraint +--echo # that says that column holds a document. A constraint that reads +--echo # another column, or no column at all, says nothing about this one. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # A constraint reading another column +--echo # +--echo # d is the ordinary usage and e an ordinary JSON column, both controls. + +CREATE TABLE t1 ( + a VARCHAR(100) CHECK (JSON_VALID(b)), + b VARCHAR(100), + d VARCHAR(100) CHECK (JSON_VALID(d)), + e JSON +); +SHOW CREATE TABLE t1; + +--echo # what the server tells a client the columns are +--enable_metadata +SELECT a, b, d, e FROM t1; +--disable_metadata + +SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1' ORDER BY COLUMN_NAME; + +--echo # +--echo # a is unconstrained, so it can hold text that is not a document +--echo # +INSERT INTO t1 VALUES ('not json at all', '{"x":1}', '{"y":2}', '{"z":3}'); +SELECT a, JSON_VALID(a) AS a_is_a_document FROM t1; + +--echo # a is quoted like the ordinary column it is, not spliced +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t1; +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t1; + +--echo # the controls are spliced +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t1; +SELECT JSON_SET('{}', '$.k', e) AS from_e FROM t1; + +DROP TABLE t1; + +--echo # +--echo # The same column holding a document that IS well formed +--echo # +--echo # Whether a is spliced or quoted decides whether the document nests +--echo # or arrives as a string, and it must not depend on b's constraint. +--echo # + +CREATE TABLE t2 ( + a VARCHAR(100) CHECK (JSON_VALID(b)), + b VARCHAR(100), + d VARCHAR(100) CHECK (JSON_VALID(d)) +); +INSERT INTO t2 VALUES ('{"p":1}', '{"x":1}', '{"p":1}'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t2; +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t2; +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t2; +DROP TABLE t2; + +--echo # +--echo # A constraint that constrains nothing +--echo # +--echo # JSON_VALID('{}') is a constant true, so the column keeps no +--echo # promise about its contents whatsoever. +--echo # + +CREATE TABLE t3 (a VARCHAR(100) CHECK (JSON_VALID('{}'))); +--enable_metadata +SELECT a FROM t3; +--disable_metadata +INSERT INTO t3 VALUES ('not json at all'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t3; +DROP TABLE t3; + +--echo # +--echo # The constraint is still enforced, on the column it names +--echo # + +CREATE TABLE t4 (a VARCHAR(100) CHECK (JSON_VALID(b)), b VARCHAR(100)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t4 VALUES ('anything', 'not json'); +INSERT INTO t4 VALUES ('anything', '{"x":1}'); +SELECT a, b FROM t4; +DROP TABLE t4; + +--echo # +--echo # A table level constraint types neither column, as before +--echo # + +CREATE TABLE t5 ( + a VARCHAR(100), + b VARCHAR(100), + CONSTRAINT c CHECK (JSON_VALID(b)) +); +--enable_metadata +SELECT a, b FROM t5; +--disable_metadata +DROP TABLE t5; + +--echo # +--echo # A check that asks more than whether the column holds a document +--echo # +--echo # What types a column is JSON_VALID() of it and nothing else. An +--echo # AND keeps the promise and an OR does not, but a promise kept is +--echo # not a column typed, and neither shape is typed. Each is told so, +--echo # in the words that fit it. +--echo # + +CREATE TABLE t7 ( + a TEXT CHECK (LENGTH(a) > 0 AND JSON_VALID(a)), + b TEXT CHECK (LENGTH(b) > 0 OR JSON_VALID(b)), + c TEXT CHECK (LENGTH(d) > 0 AND JSON_VALID(d)), + d TEXT +); +CREATE TABLE t8 (d TEXT CHECK (JSON_VALID(d))); + +--echo # the protocol tells the client what the warnings told the writer: +--echo # none of the three is a JSON column, and a column whose check is +--echo # the call alone still is +--enable_metadata +SELECT a, b, c, d FROM t7; +SELECT d FROM t8; +--disable_metadata + +--echo # and what each does with a document says the same: quoted, where a +--echo # column whose check is the call alone is spliced +INSERT INTO t7 VALUES ('{"p":1}', '{"p":1}', '{"p":1}', '{"p":1}'), + ('[1,2]', '[1,2]', '[1,2]', '[1,2]'); +INSERT INTO t8 VALUES ('{"p":1}'), ('[1,2]'); +SELECT JSON_SET('{}', '$.k', a) AS from_and_shape FROM t7; +SELECT JSON_SET('{}', '$.k', d) AS from_call_alone FROM t8; +DROP TABLE t7, t8; + +--echo # +--echo # A column the server itself wrote the constraint for stays JSON +--echo # +--echo # Both a declared JSON column and the columns of a temporary table +--echo # built for grouping carry a constraint over themselves. +--echo # + +CREATE TABLE t6 (j JSON, s VARCHAR(100) CHECK (JSON_VALID(s))); +INSERT INTO t6 VALUES ('{"x":1}', '{"y":2}'), ('{"x":3}', '{"y":4}'); + +--enable_metadata +SELECT j, s FROM t6 GROUP BY j, s; +--disable_metadata + +SELECT JSON_SET('{}', '$.k', j) AS from_j, + JSON_SET('{}', '$.k', s) AS from_s +FROM (SELECT j, s FROM t6 GROUP BY j, s) AS grouped; + +SELECT JSON_ARRAYAGG(j) AS agg_j, JSON_ARRAYAGG(s) AS agg_s FROM t6; + +DROP TABLE t6; + +--echo # +--echo # A row image a trigger reads through OLD. or NEW. is reached +--echo # through a SECOND set of field objects, copied from the table's +--echo # own so that they can point at the other record buffer. A copy is +--echo # the same column of the same table and carries the same check, so +--echo # it is the same JSON column, and what it holds goes into a +--echo # document as a document. +--echo # +--echo # Which columns get copied is not the same on the two sides. The +--echo # OLD. buffer is built whenever there is an update or delete +--echo # trigger and copies every column; the NEW. buffer is built only +--echo # where some column of the table cannot be null, and copies only +--echo # the columns that cannot be. So the five cases below that read a +--echo # copy are not the five a reader would guess, and the two that +--echo # read the table's own fields are here to say which is which. +--echo # +--echo # Every answer below is what a released server gives. +--echo # +CREATE TABLE lg (tag VARCHAR(24), v TEXT); +CREATE TABLE t7 (id INT, j JSON); +CREATE TABLE t8 (id INT, j JSON NOT NULL); +INSERT INTO t7 VALUES (1, '{"a":1}'); +INSERT INTO t8 VALUES (1, '{"a":1}'); + +CREATE TRIGGER tr1 BEFORE UPDATE ON t7 FOR EACH ROW + INSERT INTO lg VALUES ('nullable OLD upd', JSON_SET('{}','$.k',OLD.j)), + ('nullable NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr2 BEFORE UPDATE ON t8 FOR EACH ROW + INSERT INTO lg VALUES ('notnull OLD upd', JSON_SET('{}','$.k',OLD.j)), + ('notnull NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr3 BEFORE INSERT ON t7 FOR EACH ROW + INSERT INTO lg VALUES ('nullable NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr4 BEFORE INSERT ON t8 FOR EACH ROW + INSERT INTO lg VALUES ('notnull NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr5 BEFORE DELETE ON t7 FOR EACH ROW + INSERT INTO lg VALUES ('nullable OLD del', JSON_SET('{}','$.k',OLD.j)); + +UPDATE t7 SET id = 2; +UPDATE t8 SET id = 2; +INSERT INTO t7 VALUES (3, '{"b":2}'); +INSERT INTO t8 VALUES (3, '{"b":2}'); +DELETE FROM t7 WHERE id = 3; + +SELECT tag, v FROM lg ORDER BY tag, v; + +--echo # +--echo # THE CONTROLS: the same two columns read through the table's own +--echo # fields, where there is no copy in the way. These were never in +--echo # question, and they say that the answers above are about the +--echo # copying and not about the columns. +--echo # +SELECT JSON_SET('{}','$.k',j) AS direct_nullable FROM t7 ORDER BY 1; +SELECT JSON_SET('{}','$.k',j) AS direct_notnull FROM t8 ORDER BY 1; + +DROP TABLE lg, t7, t8; diff --git a/mysql-test/main/mysql-metadata.result b/mysql-test/main/mysql-metadata.result index 1530465eaf8bd..09687f8b088c2 100644 --- a/mysql-test/main/mysql-metadata.result +++ b/mysql-test/main/mysql-metadata.result @@ -8,6 +8,9 @@ js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +Warnings: +Warning 4271 CHECK constraint of column 'js2' asks more than JSON_VALID() of it; the column is not a JSON column +Warning 4269 CHECK constraint of column 'js3' calls JSON_VALID() on something other than 'js3'; the column is not a JSON column -------------- SELECT * FROM t1 -------------- @@ -44,7 +47,7 @@ Catalog: `def` Database: `test` Table: `t1` Org_table: `t1` -Type: BLOB (format=json) +Type: BLOB Collation: latin1_swedish_ci (8) Length: 65535 Max_length: 0 diff --git a/mysql-test/main/type_json.result b/mysql-test/main/type_json.result index 362355f222169..8e0b7e44a923f 100644 --- a/mysql-test/main/type_json.result +++ b/mysql-test/main/type_json.result @@ -111,11 +111,14 @@ js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +Warnings: +Warning 4271 CHECK constraint of column 'js2' asks more than JSON_VALID() of it; the column is not a JSON column +Warning 4269 CHECK constraint of column 'js3' calls JSON_VALID() on something other than 'js3'; the column is not a JSON column SELECT * FROM t1; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr def test t1 t1 js0 js0 252 (format=json) 4294967295 0 Y 144 0 33 def test t1 t1 js1 js1 252 (format=json) 196605 0 Y 16 0 33 -def test t1 t1 js2 js2 252 (format=json) 196605 0 Y 16 0 33 +def test t1 t1 js2 js2 252 196605 0 Y 16 0 33 def test t1 t1 js3 js3 252 196605 0 Y 16 0 33 js0 js1 js2 js3 SELECT js0, JSON_COMPACT(js0), JSON_COMPACT('{}') FROM t1; diff --git a/sql/field.cc b/sql/field.cc index 5df0e93ed2494..f16f0d952586d 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7502,17 +7502,22 @@ Field_longstr::report_if_important_data(const char *pstr, const char *end, void Field_longstr::make_send_field(Send_field *field) { Field_str::make_send_field(field); - if (check_constraint) - { - /* - Append the format that is implicitly implied by the CHECK CONSTRAINT. - For example: - CREATE TABLE t1 (js longtext DEFAULT NULL CHECK (json_valid(a))); - SELECT j FROM t1; - will add "format=json" to the extended type info metadata for t1.js. - */ - check_constraint->expr->set_format_by_check_constraint(field); - } + /* + Append the format that is implicitly implied by the CHECK CONSTRAINT. + For example: + CREATE TABLE t1 (js longtext DEFAULT NULL CHECK (json_valid(js))); + SELECT js FROM t1; + will add "format=json" to the extended type info metadata for t1.js. + + What is asked is what types the column, which is the same question + Field_string::type_handler() below asks and the same answer. A column + the client is told is a document is a column whose values it may pass + to a parser without quoting them, and that is what being typed JSON + means here; a check the column cannot pass without holding a document + is a weaker thing and does not type it. + */ + if (Type_handler_json_common::has_json_valid_constraint(this)) + Type_handler_json_common::set_format_name(field); } @@ -10950,8 +10955,12 @@ bool Column_definition::fix_attributes_temporal_with_time(uint int_part_length) bool Column_definition::validate_check_constraint(THD *thd) { - return check_constraint && - check_expression(check_constraint, &field_name, VCOL_CHECK_FIELD); + if (!check_constraint) + return false; + Type_handler_json_common::warn_if_json_valid_does_not_type(thd, + check_constraint, + field_name); + return check_expression(check_constraint, &field_name, VCOL_CHECK_FIELD); } diff --git a/sql/item.h b/sql/item.h index 85d86fc8c1c51..2213f8eb9171a 100644 --- a/sql/item.h +++ b/sql/item.h @@ -2278,15 +2278,6 @@ class Item :public Value_source, /* This is to handle printing of default values */ virtual bool need_parentheses_in_default() { return false; } virtual void save_in_result_field(bool no_conversions) {} - /* - Data type format implied by the CHECK CONSTRAINT, - to be sent to the client in the result set metadata. - */ - virtual bool set_format_by_check_constraint(Send_field_extended_metadata *) - const - { - return false; - } /* set value of aggregate function in case of no rows for grouping were found */ @@ -2431,6 +2422,12 @@ class Item :public Value_source, virtual bool switch_to_nullable_fields_processor(void *arg) { return 0; } virtual bool find_function_processor (void *arg) { return 0; } + /* + Whether this is a JSON_VALID() call reading the column named by arg. + Used while a table is being defined, where the expression is not fixed + yet and a column can only be recognised by its name. + */ + virtual bool json_valid_of_column_processor (void *) { return 0; } /* Check if a partition function is allowed SYNOPSIS diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index cfae1bae1da8c..3c0878a8f06f5 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -6761,21 +6761,6 @@ Item *Item_cond_and::neg_transformer(THD *thd) /* NOT(a AND b AND ...) -> */ } -bool -Item_cond_and::set_format_by_check_constraint( - Send_field_extended_metadata *to) const -{ - List_iterator_fast li(const_cast&>(list)); - Item *item; - while ((item= li++)) - { - if (item->set_format_by_check_constraint(to)) - return true; - } - return false; -} - - Item *Item_cond_or::neg_transformer(THD *thd) /* NOT(a OR b OR ...) -> */ /* NOT a AND NOT b AND ... */ { diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index a75b1eef510f9..a661c695068df 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -3695,8 +3695,6 @@ class Item_cond_and final :public Item_cond COND *build_equal_items(THD *thd, COND_EQUAL *inherited, bool link_item_fields, COND_EQUAL **cond_equal_ref) override; - bool set_format_by_check_constraint(Send_field_extended_metadata *to) const - override; void add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) override; diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 2f310f7b402e1..4868a53a12ffc 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -148,11 +148,10 @@ class Item_func_json_valid: public Item_bool_func set_maybe_null(); return FALSE; } - bool set_format_by_check_constraint(Send_field_extended_metadata *to) const - override + bool json_valid_of_column_processor(void *arg) override { - static const Lex_cstring fmt(STRING_WITH_LEN("json")); - return to->set_format_name(fmt); + return Type_handler_json_common:: + is_json_valid_of_name(this, *(const LEX_CSTRING *) arg); } enum Functype functype() const override { return JSON_VALID_FUNC; } diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 9f1dcec69a067..27be865b78ced 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -10758,3 +10758,13 @@ ER_CM_OPTION_MISSING_REQUIREMENT eng "CHANGE MASTER TO option '%s=%s' is missing requirement %s" ER_SLAVE_STATEMENT_TIMEOUT 70100 eng "Slave log event execution was interrupted (slave_max_statement_time exceeded)" +# A number a later series has already given out would mean one number +# with two meanings, so the next one starts above every number assigned +# in any of them +start-error-number 4269 +ER_WARN_JSON_VALID_OTHER_COLUMN + eng "CHECK constraint of column '%-.192s' calls JSON_VALID() on something other than '%-.192s'; the column is not a JSON column" +ER_WARN_JSON_VALID_NOT_REQUIRED + eng "CHECK constraint of column '%-.192s' can pass without JSON_VALID() holding; the column is not a JSON column" +ER_WARN_JSON_VALID_NOT_ALONE + eng "CHECK constraint of column '%-.192s' asks more than JSON_VALID() of it; the column is not a JSON column" diff --git a/sql/sql_type_json.cc b/sql/sql_type_json.cc index 31767d68503ec..885f9cc6d3754 100644 --- a/sql/sql_type_json.cc +++ b/sql/sql_type_json.cc @@ -107,45 +107,214 @@ Type_handler_json_common::json_type_handler_sum(const Item_sum *item) } +/** + The column a JSON_VALID() call is about, where the expression is that + call and nothing else. + + NULL for anything else at all: another function, the call with more + than the one argument it takes, or an argument that is not a column. + Both questions below are that shape plus a way of naming the column, + so the shape is asked once. + + RESOLVED says whether the expression has been fixed. It has, wherever + there is a Field to compare against, and the argument is then reached + through Item::real_item() so that a reference standing in for the + column is followed to it; it has not while a table is being defined, + where following would be reaching through a pointer nothing has set. +*/ + +static const Item_field *json_valid_single_field_arg(const Item *expr, + bool resolved) +{ + const Item_func *func; + const Item *arg; + + if (!expr || expr->type() != Item::FUNC_ITEM) + return NULL; + func= static_cast(expr); + if (func->functype() != Item_func::JSON_VALID_FUNC || + func->argument_count() != 1) + return NULL; + arg= func->arguments()[0]; + if (resolved) + arg= arg->real_item(); + if (arg->type() != Item::FIELD_ITEM) + return NULL; + return static_cast(arg); +} + + +/** + Whether an expression asks nothing but whether one particular column + holds a document. + + The COLUMN, and not the field object standing for it. A table can + have more than one of those for the same column: the second and third + row images a trigger reads through OLD. and NEW. are reached through + copies made by Field::make_new_field(), which memdups the field and + points it at the other buffer. A copy carries the same check + constraint - the same pointer, to an expression that still names the + field it was fixed against - so comparing the objects makes every + copy of a JSON column stop being one, and what it holds is quoted + into a document instead of going in as a document. + + Same table and same position is the whole of the question and gives + nothing away. A column check can only read columns of its own table, + so a check that names a different position is a check about another + column - which is the thing this asks in order to refuse - and one + that names the same position in another table is a copy that was put + somewhere else, where the constraint it brought along says nothing. +*/ + +bool Type_handler_json_common::is_json_valid_of_field_expr(const Item *expr, + const Field *field) +{ + const Item_field *arg= json_valid_single_field_arg(expr, true); + + return arg && arg->field->table == field->table && + arg->field->field_index == field->field_index; +} + + +/** + Whether a column carries a check constraint that says that column holds + a document, which is what makes it a JSON column. + + The question has to be about this column: a constraint reading another + column, or none at all, promises nothing about what is in this one, and + a column typed JSON on such a promise has its contents put into + documents verbatim rather than quoted. + + A constraint the server wrote for a temporary table column carries no + vcol type, so there is nothing to ask about it beyond the expression; + a constraint belonging to the table rather than to a column never + reaches a column's check_constraint in the first place. +*/ + bool Type_handler_json_common::has_json_valid_constraint(const Field *field) { return field->check_constraint && - field->check_constraint->expr && - field->check_constraint->expr->type() == Item::FUNC_ITEM && - static_cast(field->check_constraint->expr)-> - functype() == Item_func::JSON_VALID_FUNC; + is_json_valid_of_field_expr(field->check_constraint->expr, field); } /** - Whether a check constraint asks nothing but whether one particular - column holds a document, that column being the one the constraint - belongs to. - - has_json_valid_constraint() above deliberately does not ask WHICH column - the constraint reads: CHECK(json_valid(other_column)) types this column - as JSON, and has done for as long as the type has existed. Leaving the - check unrun needs the stronger question, because what was written into - this column says nothing about what is in another one, and that question - is the one asked here. + The same question about a check constraint read from a table + definition, where it also has to be the column's own check rather than + the table's before its result can be trusted. */ bool Type_handler_json_common::is_json_valid_of_field(Virtual_column_info *check, const Field *field) { - Item_func *func; - Item *arg; + return check && check->get_vcol_type() == VCOL_CHECK_FIELD && + is_json_valid_of_field_expr(check->expr, field); +} + + +/** + The same question asked while a table is being defined. + + There the expression has not been fixed yet, so there is no Field to + compare against and the column is recognised by name instead. Do not + fold this into is_json_valid_of_field_expr(): that one runs on an open + table, where names have already been resolved and the identity of the + Field is the stronger answer. +*/ + +bool Type_handler_json_common::is_json_valid_of_name(const Item *expr, + const LEX_CSTRING &name) +{ + const Item_field *arg= json_valid_single_field_arg(expr, false); + + return arg && !my_strcasecmp(system_charset_info, arg->field_name.str, + name.str); +} + + +/** + Whether a check constraint cannot pass without the named column holding + a document. - if (!check || check->get_vcol_type() != VCOL_CHECK_FIELD || - !check->expr || check->expr->type() != Item::FUNC_ITEM) + A conjunction has to hold in full, so a JSON_VALID() anywhere among its + parts is required; anywhere else in an expression it may be what the + constraint passes without. +*/ + +static bool json_valid_of_name_required(Item *expr, const LEX_CSTRING &name) +{ + Item_cond *cond; + List_iterator_fast li; + Item *item; + + if (Type_handler_json_common::is_json_valid_of_name(expr, name)) + return true; + if (!expr || expr->type() != Item::COND_ITEM) return false; - func= static_cast(check->expr); - if (func->functype() != Item_func::JSON_VALID_FUNC || - func->argument_count() != 1) + cond= static_cast(expr); + if (cond->functype() != Item_func::COND_AND_FUNC) return false; - arg= func->arguments()[0]->real_item(); - return arg->type() == Item::FIELD_ITEM && - static_cast(arg)->field == field; + li.init(*cond->argument_list()); + while ((item= li++)) + { + if (json_valid_of_name_required(item, name)) + return true; + } + return false; +} + + +/** + Say so when a check constraint mentions JSON_VALID() but leaves the + column an ordinary one. + + Writing JSON_VALID() in a column's check constraint is how a JSON column + is asked for, so a constraint that mentions it and still does not make + the column a JSON column is worth reporting: it reads something other + than this column, or it can be satisfied without the call holding, or it + asks something else as well. All three are quiet, and until the column + is used in a document none of them shows. + + What is asked here is what types the column and nothing weaker. A + conjunction cannot pass unless every part of it does, so JSON_VALID() + inside one is a promise that really is kept - but a promise kept is not + the same thing as a column typed, and has_json_valid_constraint() wants + the call and nothing else. Asking whether the promise is kept, which is + what this used to ask, left the commonest shape of all - JSON_VALID(c) + AND something - typed as text, warned about not at all, and reported to + the client as a JSON column by the protocol. +*/ + +void Type_handler_json_common::warn_if_json_valid_does_not_type( + THD *thd, + Virtual_column_info *check, + const LEX_CSTRING &name) +{ + Item_func::Functype json_valid= Item_func::JSON_VALID_FUNC; + + if (!check || !check->expr || is_json_valid_of_name(check->expr, name)) + return; + + if (check->expr->walk(&Item::json_valid_of_column_processor, 0, + const_cast(&name))) + { + /* + The call is there and it is about this column, so what stands in the + way is one of two things, and they are worth telling apart: the + constraint can be satisfied without the call holding, or it cannot + and the constraint is still not the call alone. Only the second is + a constraint that does what its writer meant and is refused anyway. + */ + uint code= json_valid_of_name_required(check->expr, name) ? + ER_WARN_JSON_VALID_NOT_ALONE : ER_WARN_JSON_VALID_NOT_REQUIRED; + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, code, + ER_THD(thd, code), name.str); + } + else if (check->expr->walk(&Item::find_function_processor, 0, &json_valid)) + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_WARN_JSON_VALID_OTHER_COLUMN, + ER_THD(thd, ER_WARN_JSON_VALID_OTHER_COLUMN), + name.str, name.str); } diff --git a/sql/sql_type_json.h b/sql/sql_type_json.h index 2161f091a62d6..77aae18542ef3 100644 --- a/sql/sql_type_json.h +++ b/sql/sql_type_json.h @@ -37,6 +37,13 @@ class Type_handler_json_common static const Type_handler *json_blob_type_handler_by_length_bytes(uint len); static const Type_handler *json_type_handler_sum(const Item_sum *sum); static const Type_handler *json_type_handler_from_generic(const Type_handler *th); + static bool is_json_valid_of_field_expr(const Item *expr, + const Field *field); + static bool is_json_valid_of_name(const Item *expr, + const LEX_CSTRING &name); + static void warn_if_json_valid_does_not_type(THD *thd, + Virtual_column_info *check, + const LEX_CSTRING &name); static bool has_json_valid_constraint(const Field *field); static bool is_json_valid_of_field(Virtual_column_info *check, const Field *field); From 006eefb390b6c058e84d8fd777f8ff60a0be7e5c Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Wed, 5 Aug 2026 21:53:57 -0400 Subject: [PATCH 17/30] Let a column the server filled itself attest to its values A value read out of a record is read again to find out whether it is a document, because a record says nothing about how what is in it was written. For a temporary table the server built for itself, something does. Such a table is written, read and thrown away inside one query, and no statement can name it. Each of its columns has one producer, known while the column is being made, so what will be written into it can be settled then and there - once, rather than once per row - and a reader of it spared finding out again what was already known. `TABLE::is_valid_json_static_set` is a bitmap pointer, and the pointer is the whole of the discriminator. `Create_tmp_table::start()` is the only place that leaves one, so a base table and a table the user asked for answer no by having nowhere to keep a yes. Nothing has to remember to say no: `open_table_from_share()` zeroes the `TABLE` it fills in, and the one place that copies a `TABLE` wholesale says it outright through `TABLE::set_filled_by_engine()`, the call the two ways in already use, so what giving them up amounts to is written down once. One table is built here and filled elsewhere, and it returns the bitmap: a select pushed down whole to an engine borrows this machinery for its columns and then has its rows written by that engine, so no store of ours ever sees what goes into them. How well the item a column was made from attests to the values IT makes says nothing about values it never made. Both ways in drop the attestations - the arm of `select_handler::prepare()` that builds such a table and the arm that takes one the engine brought with it - so that every way in has done it and not merely the one anybody walks, and `select_handler::execute()`, where the engine's rows are taken, asserts that some way in did. Unlike `TABLE::is_valid_json_set` alongside it, these bits say nothing about one row image. They are written while the table is being built and they hold for every row the column will ever be given, which is why a reader can still believe one long after the row it was written in is gone. Every row but one. A row that came out SQL NULL has no characters under it to have been attested, and the bit stands over it like any other, so `Field::answers_is_valid_json()` asks whether the field is holding a value before it reads the bit at all. That is where the question belongs rather than at the store that acts on the answer: an answer wrong where it is given is wrong for every reader of it, and the store is one of four. Without it, a NULL arriving at a column that cannot hold one - which puts the empty string there instead - is written down as a document, and the check constraint saying it must be one is left unrun over the value it exists to refuse. A grouping or a union is enough to build the table the bit is about: CREATE TABLE t (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))); INSERT INTO t (j) SELECT JSON_EXTRACT(d, '$.nosuch') FROM src GROUP BY d; `func_json_check_store` has both shapes, in the two modes that decide whether the row is refused or carried past, and beside them a column that can hold a NULL, which holds the NULL itself and is not asked to hold a document. `Field::set_is_valid_json_static()` is told, at each of the two places a field of such a table is added, the item that will fill it. It asks for an item that gives back a document every time it is evaluated rather than the time somebody happened to look, and for a field that puts down the characters it is handed. It does not ask whether the character sets keep them, and says why it does not: the branches are asked what they always answer, which is not what the column's store will keep of the value, and the set the column will be in is not settled until the line below the loop. The question belongs at the store, and a reader of the grant would otherwise have to find its answer at the far end. `Field::confirm_is_valid_json_static()` runs after every store into a marked field, and is where that question is asked. It asks how much was kept, and it asks `String_copier::conversion_keeps_characters()` beside it - the caller passing the set the value was written in - because the length alone attests to a store that converts nothing. A store between a wide set and the binary one keeps every byte and calls them by the other set's name, so the length comes back unchanged over characters that are no longer the ones that were written: a `ucs2` document arrives as bytes beginning `00 7B`, which nothing reads as a document. The grant cannot ask this and the confirm says so where it does: the field is made from a type holder whose own set is the aggregated one and so is always the field's, which makes the question answer itself there and decide nothing. What is left of a confirm once the answer itself has survived it - the formatting and the depth, which are said about the values rather than about the column - is one body that both confirms end at. Two copies of a check are two chances for one of them to stop matching what it is checking, which is the reason the debug reading below it is written once as well. Going wrong is not reported here: `Field_longstr::report_if_important_data()` answers 0 for a truncation unless `count_cuted_fields` is raised above `CHECK_FIELD_EXPRESSION`, and writing a temporary table does not raise it. A store that dropped the tail of a document would return success and warn nobody. A mark lost that way is lost for good, one field of one row being enough. `Item_field::is_valid_json()` returns the bit, and `append_json_value_from_field()` asks the field where it used to assume nothing - which is where an aggregate reads back the elements it gathered. A column of one of these tables can be written into a column of the next, and the second is built before either has a row - so what the first says at that moment is not what it will say once the row comes. It is more than it will say: the bit is set in one place, reached only while such a table is being built, and everything that touches it afterwards only ever clears. What the second column is given is therefore an upper bound on what the first can still say, and asking the first again as each row is put across is what turns the bound into the answer. Both tables are alive at that moment by construction, and the first has already stored the row being asked about, so nothing here rests on the order two tables are written in. A value reaches such a field off an item or off another field, and those are two different questions - the first is asked of the store, the second of the source. A field filled from another field is confirmed in `field_conv()`, after the dispatch it makes and before it returns, so a route that reaches it is covered wherever it is written rather than by being on a list that has to be kept complete. Two routes reach what it dispatches to without going through it - the fast copier a grouping key is written by, and the `copy_fields()` loop - and those two ask for themselves. The copier is asked although no column carrying an answer is known to arrive there, which fields a route is handed being a fact about the queries people have written rather than about the route. Whether the loop has anything to ask about is settled where its entry is made rather than once a row. The bit it would read is set only while a temporary table is being built, which is before any entry here is made, and afterwards it is only ever cleared; so a yes recorded at the entry is an upper bound, and the confirm the loop makes reads the bit again and turns the bound into the answer. An entry that records neither field answers no by starting so, both kinds of entry sharing one array. The unit test over that constructor holds all three of its members to it, and reads the answer as a byte rather than as a value: a bool holding neither 0 nor 1 has no comparison the language defines, so asking one whether it equals false over the poison the test is built on returns the poison unchanged and passes. What the source is asked is what the store is asked - whether it attests to its values, and whether every character survived - with the character sets asked about as well, a length in bytes being able to agree across two of them while every character disagrees. A column is only ever attested where they match, so that decides nothing today; it is there so the byte count does not quietly become the wrong question at a fill site the grant never looked at. A grouping that needs no temporary table of its own still has to hold a value across the group it belongs to, and holds it in a buffer instead. A HAVING or an ORDER BY reads what it reads out of that buffer, so what arrives there is a value nobody had attested. A buffer is filled beside the item that made the value and holds one value at a time, so what that item says goes away with the value and is about it still when it is read back. That is more than a column can be told. A column is made before any of its rows exist, so only a producer that attests to every value it will ever make can be recorded in one; a buffer is filled afterwards and takes the answer about the value in it, which is why a function attesting to what it has just written is read here and not there. A buffer that could not find the room for what it was handed keeps what it held before and returns that, as it always has. What is in it then is a value from another row that nothing here asked about, so the buffer attests to nothing. The room is asked for by the copy rather than reported by it, which is the same question a store into a column is asked, and it is arranged for the same way. Whether a value held for a group is one a splice puts into a document is settled by the type, and the type is settled where the buffer is set up. It is worked out there and kept, the row path having otherwise asked two virtuals per copied field per row about something that was true before the query began. What reads a buffer is a reference, and a reference returns what it reads from the result side rather than from the value side. For an item that keeps a result field those are two different runs of bytes - the value it made, and a copy of it sitting in a record - and only the first is what anything said about; for a buffer there is one. Which of the two a reference reads is not a property of what it refers to. Two references over the same item read differently, so the question can only be put to the reference, and it is one the reference already attests to its own bytes. The three answers therefore grow a result side beside the value side, virtual bool is_valid_json_result() const; virtual bool is_nice_json_result() const; virtual uint last_depth_result() const; each class saying there whatever its own `str_result()` reads: `Item_field` out of its result field, `Item_ref` by the same runtime condition its `val_str()` uses, `Item_direct_ref` off the value side, `Item_direct_view_ref` by passing on. A reference then asks the side it is going to read from and carries what it finds, where before it had to decide from the referent alone whether carrying anything was safe - and a class-wide answer cannot settle a per-reference question. The standing answer gets no result-side twin and could not use one. It is asked from `Field::set_is_valid_json_static()`, which runs after `create_tmp_field_ex()` has already pointed the result field at the very column being granted, so a twin would have read the bit it was about to set. A UNION writes one column from a producer per branch, and that column is read as one column - nothing about a row afterwards saying which branch it came out of. So what can be said about it is what every one of those producers says, and one that says nothing is the whole column saying nothing. They are all in reach in one place: the branches are walked item by item at the same position while the types are being agreed, and each position gathers there the one item each branch put at it. That walk already aggregates in this direction, taking a column to be nullable if any branch's is; taking it to be attested if every branch's is is the same walk read the other way round. Nobody gathered is a no, an answer about no producers being about nothing rather than a yes about an empty set. Asking at that moment is sound in a way asking a column is not. What is asked of a producer is a property of the kind of item it is rather than of anything it has done, so it holds before a row exists as much as after; a branch whose producer is a column answers no, which is what keeps one such table read into another out of this and on the footing it already has. A recursive CTE is where they are NOT all in reach: the column is built once the parts that do not go round have agreed on a type, the parts that do go round not being walked at all. There the ones that were walked are not allowed to answer in their place - and this is the same place, and the same reason, as the nullability that is forced on there already. Rows written one list at a time rather than one branch at a time gather in the same holders through the same walk, and are read the same way. A document spliced into another goes in as it stands, so a caller whose own answer has to come out spaced a particular way cannot take one that is not without reading it - and the functions that EDIT a document are all such callers, each of them reading back everything it has written to settle the spacing. A value that arrived spaced that way already saves them that reading, and until now nothing in a record could say so. This is a second thing a column can attest to, and it is not come by the way the first is. Being a document is promised by the producer before a row exists, so it can be written down once and believed; how a value is FORMATTED is settled one value at a time, and there is nobody to ask about a value nobody has made. So it is granted along with the first answer and spent afterwards: every value that arrives written another way takes it back, for good, and what the answer amounts to when it is read is that nothing put there so far was. That is what a reader with a row in front of it is asking, the row having been put there before the reading. It is spent at the places the first answer is already asked about, both of them: the store, which has the item that has just written the value and can ask it, and the fill from another field, which has only the field the characters were copied from and asks that. Nothing new was added to reach them. A column of a table the user asked for still answers neither, and by the same construction - the formatting is only ever read through the answer above, so the one place that leaves a bitmap is the one place either can be kept. A value spliced into a document is read to find out how deep it nests, because how deep it ends up is what decides whether the answer can be read back at all. A value something else has already attested is not read, so that measurement is not made either, and the only thing standing in for it was how long the value is - nothing nests deeper than half its length. That is exact for a run of brackets and says nothing whatever about a long shallow document, which was read as before however often it had been read already. `Item::last_depth()` is a third answer beside is_valid and is_nice, asked under the same rules and about the same value. A function that walked the value while writing it knows how deep it went and says so; everything else answers `JSON_DEPTH_UNKNOWN`, which costs a reading and nothing else. The two answers are bounds on the same thing, so the smaller is taken and either alone would do - which is what leaves a value nobody counted treated exactly as it was. The counting comes from walks that were happening anyway. `json_nice()` kept a depth to know how far to indent and now returns it, which attests to every function that reads its own result. `JSON_EXTRACT` already tracked the deepest value it wrote. The functions that BUILD a document collect it from what went in, and the ones that EDIT one take the deeper of what went in and what the document said of itself. Merging takes one level more than its arguments: two arrays laid together stay where they were, but an array merged with anything else makes that thing a member of it, which is one level in. It folds that in one argument at a time, beside the two marks that are already taken that way, because what an argument answers is about the value it has just passed: an argument that reads a variable answers out of what is in the variable now, and a later argument can assign to it. Where a value's own answer goes is taken by reference rather than by pointer, there being no caller with nowhere to put one: every function that composes a document keeps a set of marks, whether or not it ends up believing what they say, so the absent case ten guards stood for cannot arise - and a guard that cannot fire is a reader's question about when it does. What a caller says as it starts is the depth it is reckoning from and nothing else, the marks taking that in a constructor of their own, and where a function returns its document rather than reading it the depth it worked out goes with it. The bound on how deep a value may go is worked out inside the arm that uses it, past the questions that would make it moot, rather than ahead of all of them. And a record can say this too, but not the way it says the other two. Those are answers; this is a figure, and the figure a column can offer is the deepest of the rows written into it so far rather than the depth of the row being read. That is never less than the row needs, which is all a reader wants of it - nothing acts on the figure except to decide that a reading can be skipped, so one larger than the truth costs a reading and one smaller would let a document be composed that nothing can read back. It runs the other way from the two beside it, and for the same reason they run the way they do. They start as a yes and are taken back; this starts at nothing, that being the deepest of no rows, and is raised by each value that arrives. A value nobody counted arrives as the largest figure there is, so it takes the column there and nothing brings it back - which is a column that has stopped saying, written as arithmetic rather than as a rule. It is asked through the answer above like the formatting is, so a column that has given that up has given this up with it. An aggregate that gathers its elements through a table therefore reads them no longer, which is where the sister that writes its pairs straight out has been all along. No answer. The readings that are left out could only have found out what was already known, so no value changes and no diagnostic appears or goes away. Three files say so for the column: one puts a document through the shapes it can travel in and shows the answers standing, another counts the readings through `Json_scans`, and the third stands where the server gives a table it built to an engine to fill. That is three gatherings pushed down whole - one sent straight out, one held in a temporary table on the way, and two ordered so the rows are read back out of the table they were held in - each with the `EXPLAIN` that says the engine took it, since a case the engine declined would run down the ordinary road and pass without going near what it is there for. An engine writes `record[0]` through `ha_write_tmp_row()` with no `Field::store()` anywhere on the way in, so nothing there could confirm a promise or take one back; they are dropped where the table is given to the engine rather than at the fill, a table built out of this one reading them while it is being built. A fourth file counts the same readings where the values are paths rather than documents somebody wrote: `func_json_search_typed` splices what `JSON_SEARCH` built out of such a table, and runs the same query with the column's answer taken away beside it. Nineteen of the forty counts fall: a document through a derived table, through a GROUP BY, gathered by an aggregate, gathered by one whose arguments were buffered first, the two that read one such table into another - a chain of them, and the table an aggregate gathers in, which is built out of a column rather than out of the function that filled that column - four held for a group in a buffer: read back from a HAVING and from an ORDER BY, filled from a column rather than from an item, and filled by a producer that attests to the value it has just made, which is the one that reaches nothing read at all - two written by more than one producer at once, a UNION whose branches all answer and a column written one list of values at a time - and three where the formatting is what is attested rather than the reading: a document edited out of such a column, the same out of the second of two such tables, and the same where the row crossed by the copier that fills a grouping. Four of the nineteen are the depth rather than the reading, and they are the only long documents in the file: every other count in it is on a value short enough that half its length was already inside the limit, so nothing there was ever read to find out how deep it went. A long document out of such a column goes in unread, and so do one carried into a second such table, one an aggregate gathers, and one out of a column two counting producers wrote. The other twenty-one stand, among them every shape that must not be believed: a UNION one of whose branches attests per evaluation and one whose branch is a column, the same for a list of values, a recursive CTE, a table the user asked for, and a producer that attests per evaluation into a column, which is the same producer the buffer above takes and the difference between them is the point. Three of them are that same editing over a column filled by a producer that returns its document formatted some other way, at each of the three places the formatting can be dropped. Three more are the depth not being attested: the same long document out of a base column, out of a column whose producer copied what it returns rather than writing it, and out of one where a single producer among several counted nothing - the figure being the deepest of all the rows, one unknown row is an unknown column. Five of those twenty-one are a place that gave its answer up, and between them they name every place one can be given up: at the store, at the store into the first table of a chain, at the row that crosses between two tables, at the store into a buffer an aggregate gathers from, and at a buffer that could not take what it was handed. Each runs the query of the case that believes, so its count is what that case would have come to with nothing said, and each writes the same value. A store that comes up short can no longer be reached through any of these functions, each of them asking for the room it goes on to use, so they are arranged for with debug hooks. The counts for the depth move where the length was answering and now is not: a long document put inside another was read at every step out and is read once, so two nested arrays over two rows go from 6 to 2. The exactness the count has to keep is pinned beside the length's own - a chain padded until half its length is seven times the limit, where nothing but a count can tell one that is admitted from one that is refused, and the same carried through a document that was edited. One more moves there once a column can answer as well: an aggregate that gathers a long document through a table, which is the shape that stood because the table was where the counting stopped. A debug build reads every marked value back and asserts that it reads as a document, that it is formatted the way it was said to be, and that it is no deeper than was claimed, those readings being the debug build's work and going uncounted. Those detectors are compiled wherever the assertions that read them are, which is not the same set of builds as the debug one: an assertion asked to print rather than to stop survives into a build with `DBUG_OFF` set, and the expression it is given is compiled there. It is the depth that needed it: two claims were short when first written, one for a value that goes in quoted where the wrapping had already spent a level, and one for merging an array with an object. Six of those readings are taken at the READ rather than at any of the writes, and they are the only ones that do not care how the value arrived: a fill site nobody taught to ask its source has to announce itself somewhere, and the first read of the column is the last place it still can. Six rather than three because the two fields an item has are asked separately, each assertion naming the field its own answer came from; three bodies serve them, one per question, since the three are asked one at a time and answering the formatting means writing the whole value out again - which the two cheap ones would then be paying for as well. Nothing else reads any of these marks - they appear in no result and change no output, so a wrong one would go unnoticed until something acted on it, a long way from the store or the walk that made the promise. --- mysql-test/main/func_json_check_store.result | 65 ++ mysql-test/main/func_json_check_store.test | 48 ++ mysql-test/main/func_json_scan_count.result | 141 +++- mysql-test/main/func_json_scan_count.test | 147 +++- mysql-test/main/func_json_search_typed.result | 23 + mysql-test/main/func_json_search_typed.test | 25 + mysql-test/main/func_json_splice_depth.result | 64 ++ mysql-test/main/func_json_splice_depth.test | 76 +- mysql-test/main/func_json_tmp_pushdown.result | 50 ++ mysql-test/main/func_json_tmp_pushdown.test | 43 ++ .../main/func_json_tmp_scan_count.result | 670 ++++++++++++++++++ mysql-test/main/func_json_tmp_scan_count.test | 551 ++++++++++++++ mysql-test/main/func_json_tmp_trust.result | 584 +++++++++++++++ mysql-test/main/func_json_tmp_trust.test | 421 +++++++++++ sql/derived_handler.cc | 12 + sql/field.cc | 266 +++++++ sql/field.h | 99 ++- sql/field_conv.cc | 19 +- sql/group_by_handler.cc | 9 + sql/item.cc | 165 ++++- sql/item.h | 319 ++++++++- sql/item_func.h | 12 + sql/item_jsonfunc.cc | 640 +++++++++++++---- sql/item_jsonfunc.h | 76 +- sql/item_strfunc.cc | 17 +- sql/item_strfunc.h | 6 +- sql/select_handler.cc | 36 +- sql/sql_class.h | 32 +- sql/sql_insert.cc | 8 + sql/sql_select.cc | 72 ++ sql/sql_tvc.cc | 4 +- sql/sql_union.cc | 11 +- sql/table.h | 100 +++ unittest/sql/copy_field-t.cc | 30 +- 34 files changed, 4633 insertions(+), 208 deletions(-) create mode 100644 mysql-test/main/func_json_tmp_pushdown.result create mode 100644 mysql-test/main/func_json_tmp_pushdown.test create mode 100644 mysql-test/main/func_json_tmp_scan_count.result create mode 100644 mysql-test/main/func_json_tmp_scan_count.test create mode 100644 mysql-test/main/func_json_tmp_trust.result create mode 100644 mysql-test/main/func_json_tmp_trust.test diff --git a/mysql-test/main/func_json_check_store.result b/mysql-test/main/func_json_check_store.result index 8e00c710bb175..7bac511755bf7 100644 --- a/mysql-test/main/func_json_check_store.result +++ b/mysql-test/main/func_json_check_store.result @@ -278,3 +278,68 @@ id j ty s e 2 {"b": 2, "piece": 2} OBJECT 2018-01-01 2018-12-12 2 {"portion": 2} OBJECT 2000-01-01 2018-01-01 DROP TABLE t3, t2, t1; +# +# 7. A value that came out SQL NULL, put into a column that cannot +# hold one. +# +# A column of a temporary table is attested to once, when the +# table is built out of something that attests to every value it +# will make. That answer is about the column and stands over +# every row of it, a row whose value came out NULL among them. +# What such a row puts in the destination is not the NULL but +# whatever the destination takes in place of one, and nothing has +# attested that. +# +CREATE TABLE src (d LONGTEXT) CHARSET utf8mb4; +INSERT INTO src VALUES ('{"a":1}'), ('{"b":2}'); +# grouping builds the temporary table, and the path finds nothing +CREATE TABLE t1 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +SET @save_sql_mode= @@sql_mode; +SET SESSION sql_mode= ''; +INSERT INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') FROM src GROUP BY d; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SET SESSION sql_mode= @save_sql_mode; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +# the same insert told to carry on past what it cannot store +INSERT IGNORE INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') +FROM src GROUP BY d; +Warnings: +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +# a union builds one too, and the rows where the path does find a +# value go in while the ones where it does not are refused +CREATE TABLE t2 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT IGNORE INTO t2 (j) +SELECT JSON_EXTRACT(d, '$.nosuch') FROM src UNION ALL +SELECT JSON_EXTRACT(d, '$.a') FROM src; +Warnings: +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t2.j` failed for `test`.`t2` +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t2.j` failed for `test`.`t2` +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t2.j` failed for `test`.`t2` +SELECT j, JSON_TYPE(j) AS ty FROM t2; +j ty +1 INTEGER +# a column that CAN hold a NULL holds the NULL itself, which is +# not a document and is not asked to be one +CREATE TABLE t3 (j LONGTEXT CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t3 (j) SELECT JSON_EXTRACT(d, '$.a') FROM src GROUP BY d; +SELECT j, j IS NULL AS was_null FROM t3; +j was_null +1 0 +NULL 1 +DROP TABLE t3, t2, t1, src; diff --git a/mysql-test/main/func_json_check_store.test b/mysql-test/main/func_json_check_store.test index 6449a1affcf98..7ca4a500faccb 100644 --- a/mysql-test/main/func_json_check_store.test +++ b/mysql-test/main/func_json_check_store.test @@ -247,3 +247,51 @@ UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t3; DROP TABLE t3, t2, t1; + +--echo # +--echo # 7. A value that came out SQL NULL, put into a column that cannot +--echo # hold one. +--echo # +--echo # A column of a temporary table is attested to once, when the +--echo # table is built out of something that attests to every value it +--echo # will make. That answer is about the column and stands over +--echo # every row of it, a row whose value came out NULL among them. +--echo # What such a row puts in the destination is not the NULL but +--echo # whatever the destination takes in place of one, and nothing has +--echo # attested that. +--echo # + +CREATE TABLE src (d LONGTEXT) CHARSET utf8mb4; +INSERT INTO src VALUES ('{"a":1}'), ('{"b":2}'); + +--echo # grouping builds the temporary table, and the path finds nothing +CREATE TABLE t1 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +SET @save_sql_mode= @@sql_mode; +SET SESSION sql_mode= ''; +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') FROM src GROUP BY d; +SET SESSION sql_mode= @save_sql_mode; +SELECT COUNT(*) FROM t1; + +--echo # the same insert told to carry on past what it cannot store +INSERT IGNORE INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') + FROM src GROUP BY d; +SELECT COUNT(*) FROM t1; + +--echo # a union builds one too, and the rows where the path does find a +--echo # value go in while the ones where it does not are refused +CREATE TABLE t2 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT IGNORE INTO t2 (j) + SELECT JSON_EXTRACT(d, '$.nosuch') FROM src UNION ALL + SELECT JSON_EXTRACT(d, '$.a') FROM src; +--sorted_result +SELECT j, JSON_TYPE(j) AS ty FROM t2; + +--echo # a column that CAN hold a NULL holds the NULL itself, which is +--echo # not a document and is not asked to be one +CREATE TABLE t3 (j LONGTEXT CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t3 (j) SELECT JSON_EXTRACT(d, '$.a') FROM src GROUP BY d; +--sorted_result +SELECT j, j IS NULL AS was_null FROM t3; + +DROP TABLE t3, t2, t1, src; diff --git a/mysql-test/main/func_json_scan_count.result b/mysql-test/main/func_json_scan_count.result index 572565d4d07d7..3921057d883f5 100644 --- a/mysql-test/main/func_json_scan_count.result +++ b/mysql-test/main/func_json_scan_count.result @@ -250,16 +250,129 @@ Json_scans 10 # # What is skipped there is a reading that would have measured how # deep the value goes, and how deep it ends up decides whether the -# result can be read at all. How long the value is answers that -# for a short one - it cannot nest deeper than half its length - -# and answers nothing for a long one, which is read as before. +# result can be read at all. Two things answer that without a +# reading, and the smaller of them is taken. How long the value +# is answers it for a short one, which cannot nest deeper than half +# its length, and says nothing about a long one. What the function +# that wrote the value counted as it wrote answers it whatever the +# length - which is what the documents below are, at eighty-odd +# characters and one structure deep. # FLUSH STATUS; SELECT JSON_ARRAY(JSON_EXTRACT(js, '$')) FROM t3; SHOW STATUS LIKE 'Json_scans'; Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_OBJECT('k', JSON_EXTRACT(js, '$')) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# And it carries, which is what the length could never do: every +# step out used to read the whole of the long value again. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3)) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value Json_scans 4 # +# A name that only asks for a formatting hands the value straight +# on, and passes on what was counted about it with it. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_COMPACT(JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A value nobody counted is read, as it always was. The column +# here is a caller's own text: it is cut into by JSON_QUERY, which +# can say the piece is a document but not how deep the document it +# came out of went. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_QUERY(js, '$')) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# A group writes its pairs as they arrive and is as deep as the +# deepest of them, so the object it makes is spliced without being +# read. Its sister gathers its elements through a table, and the +# column of that table keeps the deepest of the rows put into it, +# so they are not read on the way back out either. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_OBJECTAGG(id, JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A name given to a value in the select list and used again later +# in the same statement reaches it through a reference, and a +# reference reads what the item's RESULT side passes. For +# nearly every item that is the value the item made, so what was +# said about the value is said about this - and the splice below +# costs the reading it costs when the expression is written out +# where it is used. +# +FLUSH STATUS; +SELECT JSON_OBJECT('a', js) AS o FROM t1 HAVING JSON_ARRAY(o) IS NOT NULL; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# A grouped query moves the aggregate out of the expression around +# it and puts a reference in its place, so the array below is +# built over a reference rather than over the aggregate. All +# three answers travel together through it, the depth among them: +# the array these gather is long enough that its length says more +# levels than a document is allowed, so a depth that went missing +# here would show up as the whole thing being read again. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3 GROUP BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A search returns one path as a string and several as an array +# of strings, so what it makes nests one level at the most and it +# knows which as it writes. The paths here are long, so the +# length says nothing useful and only that count keeps the answer +# from being read again where it is spliced. +# +CREATE TABLE t4 (id INT, js TEXT); +INSERT INTO t4 VALUES +(1, '{"customer_record":{"order_lines":[{"sku_code":"xa"},{"sku_code":"xb"}]}}'), +(2, '{"customer_record":{"order_lines":[{"sku_code":"xc"},{"sku_code":"xd"}]}}'); +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_SEARCH(js, 'all', 'x%')) FROM t4; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# One path found, which is a string and nests nothing at all +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_SEARCH(js, 'one', 'x%')) FROM t4; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t4; +# # Asking whether a value is a document reads it once. There is # nothing here left to take away. # @@ -376,3 +489,25 @@ CONVERT(JSON_OBJECT('a', _utf8mb4 0xE4BDA0) USING latin1)); SHOW STATUS LIKE 'Json_scans'; Variable_name Value Json_scans 3 +# +# A conversion FROM the binary character set is not a conversion at +# all: there is nothing to convert from, so the bytes go across +# untouched and are read as whatever the other set makes of them. +# That is a different string of characters and, for a document, +# usually not one - so nothing carries and the value that comes out +# is read wherever it is used. +# +SELECT HEX(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary; +from_binary +5B226162225D +SELECT JSON_VALID(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary_valid; +from_binary_valid +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +# +# And the same going the other way, which was always refused +# +SELECT HEX(CONVERT(JSON_ARRAY('ab') USING binary)) AS to_binary; +to_binary +5B226162225D diff --git a/mysql-test/main/func_json_scan_count.test b/mysql-test/main/func_json_scan_count.test index 71163208b4259..d90f845da529d 100644 --- a/mysql-test/main/func_json_scan_count.test +++ b/mysql-test/main/func_json_scan_count.test @@ -305,9 +305,13 @@ SHOW STATUS LIKE 'Json_scans'; --echo # --echo # What is skipped there is a reading that would have measured how --echo # deep the value goes, and how deep it ends up decides whether the ---echo # result can be read at all. How long the value is answers that ---echo # for a short one - it cannot nest deeper than half its length - ---echo # and answers nothing for a long one, which is read as before. +--echo # result can be read at all. Two things answer that without a +--echo # reading, and the smaller of them is taken. How long the value +--echo # is answers it for a short one, which cannot nest deeper than half +--echo # its length, and says nothing about a long one. What the function +--echo # that wrote the value counted as it wrote answers it whatever the +--echo # length - which is what the documents below are, at eighty-odd +--echo # characters and one structure deep. --echo # FLUSH STATUS; --disable_result_log @@ -315,6 +319,127 @@ SELECT JSON_ARRAY(JSON_EXTRACT(js, '$')) FROM t3; --enable_result_log SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_OBJECT('k', JSON_EXTRACT(js, '$')) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And it carries, which is what the length could never do: every +--echo # step out used to read the whole of the long value again. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3)) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A name that only asks for a formatting hands the value straight +--echo # on, and passes on what was counted about it with it. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_COMPACT(JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A value nobody counted is read, as it always was. The column +--echo # here is a caller's own text: it is cut into by JSON_QUERY, which +--echo # can say the piece is a document but not how deep the document it +--echo # came out of went. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_QUERY(js, '$')) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A group writes its pairs as they arrive and is as deep as the +--echo # deepest of them, so the object it makes is spliced without being +--echo # read. Its sister gathers its elements through a table, and the +--echo # column of that table keeps the deepest of the rows put into it, +--echo # so they are not read on the way back out either. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_OBJECTAGG(id, JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A name given to a value in the select list and used again later +--echo # in the same statement reaches it through a reference, and a +--echo # reference reads what the item's RESULT side passes. For +--echo # nearly every item that is the value the item made, so what was +--echo # said about the value is said about this - and the splice below +--echo # costs the reading it costs when the expression is written out +--echo # where it is used. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_OBJECT('a', js) AS o FROM t1 HAVING JSON_ARRAY(o) IS NOT NULL; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A grouped query moves the aggregate out of the expression around +--echo # it and puts a reference in its place, so the array below is +--echo # built over a reference rather than over the aggregate. All +--echo # three answers travel together through it, the depth among them: +--echo # the array these gather is long enough that its length says more +--echo # levels than a document is allowed, so a depth that went missing +--echo # here would show up as the whole thing being read again. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3 GROUP BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A search returns one path as a string and several as an array +--echo # of strings, so what it makes nests one level at the most and it +--echo # knows which as it writes. The paths here are long, so the +--echo # length says nothing useful and only that count keeps the answer +--echo # from being read again where it is spliced. +--echo # +CREATE TABLE t4 (id INT, js TEXT); +INSERT INTO t4 VALUES + (1, '{"customer_record":{"order_lines":[{"sku_code":"xa"},{"sku_code":"xb"}]}}'), + (2, '{"customer_record":{"order_lines":[{"sku_code":"xc"},{"sku_code":"xd"}]}}'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_SEARCH(js, 'all', 'x%')) FROM t4; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # One path found, which is a string and nests nothing at all +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_SEARCH(js, 'one', 'x%')) FROM t4; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t4; + --echo # --echo # Asking whether a value is a document reads it once. There is --echo # nothing here left to take away. @@ -432,3 +557,19 @@ SELECT JSON_SET('{"x":1}', '$.p', CONVERT(JSON_OBJECT('a', _utf8mb4 0xE4BDA0) USING latin1)); --enable_result_log SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A conversion FROM the binary character set is not a conversion at +--echo # all: there is nothing to convert from, so the bytes go across +--echo # untouched and are read as whatever the other set makes of them. +--echo # That is a different string of characters and, for a document, +--echo # usually not one - so nothing carries and the value that comes out +--echo # is read wherever it is used. +--echo # +SELECT HEX(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary; +SELECT JSON_VALID(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary_valid; + +--echo # +--echo # And the same going the other way, which was always refused +--echo # +SELECT HEX(CONVERT(JSON_ARRAY('ab') USING binary)) AS to_binary; diff --git a/mysql-test/main/func_json_search_typed.result b/mysql-test/main/func_json_search_typed.result index 95b1a990974e0..92a6e43ba67eb 100644 --- a/mysql-test/main/func_json_search_typed.result +++ b/mysql-test/main/func_json_search_typed.result @@ -84,4 +84,27 @@ id p ty 1 ["$.a", "$.b"] ARRAY 2 ["$.c", "$.d"] ARRAY DROP TABLE t3; +# +# 6. The reading that is not done +# +# A path spliced into a new document out of a temporary table +# the server built is taken as it stands; the same query with +# the column's answer taken away reads it again. +# +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d +GROUP BY p; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d +GROUP BY p; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; DROP TABLE t1; diff --git a/mysql-test/main/func_json_search_typed.test b/mysql-test/main/func_json_search_typed.test index 9ea0a080c86f2..7ceebc6b927a3 100644 --- a/mysql-test/main/func_json_search_typed.test +++ b/mysql-test/main/func_json_search_typed.test @@ -72,4 +72,29 @@ INSERT INTO t3 SELECT id, JSON_SEARCH(j, 'all', '%') FROM t1; SELECT id, p, JSON_TYPE(p) AS ty FROM t3 ORDER BY id; DROP TABLE t3; +--echo # +--echo # 6. The reading that is not done +--echo # +--echo # A path spliced into a new document out of a temporary table +--echo # the server built is taken as it stands; the same query with +--echo # the column's answer taken away reads it again. +--echo # +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d + GROUP BY p; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d + GROUP BY p; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; + DROP TABLE t1; diff --git a/mysql-test/main/func_json_splice_depth.result b/mysql-test/main/func_json_splice_depth.result index 83a4475ba1644..6810fee6fac61 100644 --- a/mysql-test/main/func_json_splice_depth.result +++ b/mysql-test/main/func_json_splice_depth.result @@ -76,6 +76,70 @@ SELECT JSON_ARRAY(JSON_EXTRACT(@d30a, '$')) AS built_limit; built_limit [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] # +# 7. A chain padded until its length says nothing about it - each +# level here is fourteen characters, so half the length is +# seven times the limit and only a count can tell one of these +# from the other +# +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), +'1', REPEAT('}', 31)); +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), +'1', REPEAT('}', 30)); +SELECT JSON_ARRAY(JSON_EXTRACT(@k31, '$')) AS padded_deep; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k31, '$'))) AS padded_deep_depth; +padded_deep_depth +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 452 +SELECT JSON_ARRAY(JSON_EXTRACT(@k30, '$')) AS padded_limit; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_limit_depth; +padded_limit_depth +32 +# +# 8. The same carried one step further out, where the count is all +# there is: the array of case 7 is longer again and is what the +# outer one is asked to take +# +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_two; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$')))) +AS padded_two_depth; +padded_two_depth +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 483 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 438 +# +# 9. A document that was EDITED is as deep as it was or as deep as +# what went into it, whichever is more, and says so without +# being read again +# +SET @k29= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 29), +'1', REPEAT('}', 29)); +SET @k28= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 28), +'1', REPEAT('}', 28)); +SET @kp29= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 29)); +SET @kp28= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 28)); +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, +JSON_EXTRACT('{"b":{"c":1}}', '$'))) +AS edited_limit; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, +JSON_EXTRACT('{"b":{"c":1}}', '$')))) +AS edited_limit_depth; +edited_limit_depth +32 +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, +JSON_EXTRACT('{"b":{"c":1}}', '$'))) +AS edited_deep; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, +JSON_EXTRACT('{"b":{"c":1}}', '$')))) +AS edited_deep_depth; +edited_deep_depth +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 479 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 443 +# # 10. Wrapping, where the new array is a structure the document # being read never had. What was found is put inside it and # the value goes in beside them both, so the value sits one diff --git a/mysql-test/main/func_json_splice_depth.test b/mysql-test/main/func_json_splice_depth.test index ad4241e2d98ff..1c236ec19fffe 100644 --- a/mysql-test/main/func_json_splice_depth.test +++ b/mysql-test/main/func_json_splice_depth.test @@ -8,12 +8,18 @@ # # The value's own depth is found by reading it. A value that something # else has already read and attested to is not read again, so the -# reading that would have measured it is not done - except that it does -# not have to be, a value only nesting as deeply as it is long. The -# cases below are the ones that tell those two apart: a value of -# nothing but brackets is as deep as half its length and is where the -# reckoning has to be exact, and a scalar is no deep at all however -# long the document holding it. +# reading that would have measured it is not done. Two things stand in +# for it, and the smaller is taken. A value only nests as deeply as it +# is long, which is exact for a value of nothing but brackets and +# useless for a long shallow one. And the function that wrote the value +# may have counted the depth as it wrote, which is exact whatever the +# length. +# +# The cases below are the ones that tell those apart, and each is set +# where being wrong by one would show: a value of nothing but brackets +# is as deep as half its length; a scalar is no deep at all however long +# the document holding it; and a long padded chain is beyond anything +# its length can say, so only a count admits it. # # The answers are the ones the server has always given. # @@ -72,6 +78,64 @@ SET @d30a= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); SELECT JSON_ARRAY(JSON_EXTRACT(@d31, '$')) AS built_deep; SELECT JSON_ARRAY(JSON_EXTRACT(@d30a, '$')) AS built_limit; +--echo # +--echo # 7. A chain padded until its length says nothing about it - each +--echo # level here is fourteen characters, so half the length is +--echo # seven times the limit and only a count can tell one of these +--echo # from the other +--echo # +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), + '1', REPEAT('}', 31)); +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), + '1', REPEAT('}', 30)); +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(@k31, '$')) AS padded_deep; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k31, '$'))) AS padded_deep_depth; +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(@k30, '$')) AS padded_limit; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_limit_depth; + +--echo # +--echo # 8. The same carried one step further out, where the count is all +--echo # there is: the array of case 7 is longer again and is what the +--echo # outer one is asked to take +--echo # +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_two; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$')))) + AS padded_two_depth; + +--echo # +--echo # 9. A document that was EDITED is as deep as it was or as deep as +--echo # what went into it, whichever is more, and says so without +--echo # being read again +--echo # +SET @k29= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 29), + '1', REPEAT('}', 29)); +SET @k28= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 28), + '1', REPEAT('}', 28)); +SET @kp29= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 29)); +SET @kp28= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 28)); +--disable_result_log +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, + JSON_EXTRACT('{"b":{"c":1}}', '$'))) + AS edited_limit; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, + JSON_EXTRACT('{"b":{"c":1}}', '$')))) + AS edited_limit_depth; +--disable_result_log +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, + JSON_EXTRACT('{"b":{"c":1}}', '$'))) + AS edited_deep; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, + JSON_EXTRACT('{"b":{"c":1}}', '$')))) + AS edited_deep_depth; + --echo # --echo # 10. Wrapping, where the new array is a structure the document --echo # being read never had. What was found is put inside it and diff --git a/mysql-test/main/func_json_tmp_pushdown.result b/mysql-test/main/func_json_tmp_pushdown.result new file mode 100644 index 0000000000000..c2c567b5be2b5 --- /dev/null +++ b/mysql-test/main/func_json_tmp_pushdown.result @@ -0,0 +1,50 @@ +# +# A temporary table an engine fills carries no attestations about +# its values. +# +# A query pushed down whole borrows the server's own temporary table +# for its columns and then has the engine write the rows into it, so +# nothing a column was made from ever sees what arrives in it. The +# table drops its attestations where it is filled, and a debug +# build asserts at the other end - where the engine's rows are read +# - that something on the way in did drop them. +# +# Nothing here is a document. The engine below gathers a sum, and a +# sum is not one; what is reached is that dropping itself, which is +# what is being asked about. The results are ordinary results and +# are here to stay ordinary. +# +# Each case is explained before it is run. An engine that stopped +# taking one of these queries would leave the case running down the +# ordinary path and passing without going near what it is here for, +# and the EXPLAIN is what says it did not. +# +# +# 1. A group gathered by the engine and sent straight out. +# +EXPLAIN SELECT SUM(seq) FROM seq_1_to_10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +SELECT SUM(seq) FROM seq_1_to_10; +SUM(seq) +55 +# +# 2. The same held in a temporary table on the way, which is the +# other half of that execution and the half that writes rows. +# +EXPLAIN SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; +SUM(seq) COUNT(*) +5050 100 +# +# 3. Two gatherings ordered, so the rows are held and then read +# back out of the table they were held in. +# +EXPLAIN SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; +SUM(seq) COUNT(seq) +15 5 diff --git a/mysql-test/main/func_json_tmp_pushdown.test b/mysql-test/main/func_json_tmp_pushdown.test new file mode 100644 index 0000000000000..cbd203215b0bb --- /dev/null +++ b/mysql-test/main/func_json_tmp_pushdown.test @@ -0,0 +1,43 @@ +--source include/have_sequence.inc + +--echo # +--echo # A temporary table an engine fills carries no attestations about +--echo # its values. +--echo # +--echo # A query pushed down whole borrows the server's own temporary table +--echo # for its columns and then has the engine write the rows into it, so +--echo # nothing a column was made from ever sees what arrives in it. The +--echo # table drops its attestations where it is filled, and a debug +--echo # build asserts at the other end - where the engine's rows are read +--echo # - that something on the way in did drop them. +--echo # +--echo # Nothing here is a document. The engine below gathers a sum, and a +--echo # sum is not one; what is reached is that dropping itself, which is +--echo # what is being asked about. The results are ordinary results and +--echo # are here to stay ordinary. +--echo # +--echo # Each case is explained before it is run. An engine that stopped +--echo # taking one of these queries would leave the case running down the +--echo # ordinary path and passing without going near what it is here for, +--echo # and the EXPLAIN is what says it did not. +--echo # + +--echo # +--echo # 1. A group gathered by the engine and sent straight out. +--echo # +EXPLAIN SELECT SUM(seq) FROM seq_1_to_10; +SELECT SUM(seq) FROM seq_1_to_10; + +--echo # +--echo # 2. The same held in a temporary table on the way, which is the +--echo # other half of that execution and the half that writes rows. +--echo # +EXPLAIN SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; +SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; + +--echo # +--echo # 3. Two gatherings ordered, so the rows are held and then read +--echo # back out of the table they were held in. +--echo # +EXPLAIN SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; +SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; diff --git a/mysql-test/main/func_json_tmp_scan_count.result b/mysql-test/main/func_json_tmp_scan_count.result new file mode 100644 index 0000000000000..5a847106e5ada --- /dev/null +++ b/mysql-test/main/func_json_tmp_scan_count.result @@ -0,0 +1,670 @@ +# +# How many times a value out of the server's own temporary table is +# read to find out whether it is a document. +# +# No query can see the difference - the reading could only have found +# out what was already known - so nothing but Json_scans says whether +# it happened. The counts below are the record of it. A count that +# goes up is a reading that came back. +# +# A debug build reads values back to check what was claimed about +# them, and none of those readings are counted: they are the debug +# build's work and not the server's. A count of 0 below therefore +# means nothing read, not that a reading went uncounted. +# +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +# +# 1. TWO rows out of a base column, spliced into a new document. +# A column says nothing about itself, so each row is read once and +# that reading is not work that can be taken away. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +r +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 2. The same two rows, put through a function that attests to what +# it returns, and spliced without a temporary table in between. +# JSON_EXTRACT reads its input once; the splice reads nothing. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 3. The same, with a temporary table in between. What the function +# answered is what the column was built to hold, so the count is the +# function's own readings and nothing else. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 4. The same shape with a producer that answers per evaluation +# rather than for every one of them. Nothing can be said about the +# column at the time it is made, so each row is read on the way out. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 5. Through a GROUP BY, which writes a temporary table of its own. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1 GROUP BY id, j) d ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 6. A UNION, whose column has one producer per branch. The +# branches are where those producers are all in reach, so what the +# column answers is what every one of them answers, and the count +# is their own readings and nothing else. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT JSON_EXTRACT(j, '$') FROM t1) d; +r +[{"a": 1}] +[{"b": 2}] +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 7. A table the user asked for, which is not the server's own. +# +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_EXTRACT(j, '$') FROM t1; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM tt; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TEMPORARY TABLE tt; +# +# 8. A store that came up short takes back what was said about the +# column, so its rows are read on the way out like anything else. +# +# No function asks for less room than it goes on to use, so nothing +# arrives short of its own accord and the short store is arranged +# for here. The query is the one from 3, so the count below is what +# 3 would have come to with nothing said about its column. +# +SET @old_debug= @@SESSION.debug_dbug; +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +# +# 9. Two temporary tables one after the other. The second is built +# before a row of the first exists, so what the first says then is +# the most it will ever say. Every row put across asks it again, +# and what is left to count is the first table's own readings. +# +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 10. Asking what a materialized value IS, rather than splicing it. +# JSON_TYPE reads its argument whatever anyone says about it: what is +# promised is that the value is a document, not which one. +# +FLUSH STATUS; +SELECT JSON_TYPE(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +OBJECT +OBJECT +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 11. An aggregate gathers its elements in a table of its own and +# reads them back out of a record rather than off the item that put +# them there. The column is what attests to them. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS r FROM t1; +r +[{"a": 1},{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 12. The same, with a producer that attests per evaluation rather +# than to every value it makes. Its column is read on the way out, +# so the total matches 11 from the other side: 11's two readings are +# the producer's own, these two are the readings back 11 was spared. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS r FROM t1; +r +[{"a": 1},{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 13. An aggregate whose arguments are buffered before the group can +# be built - a join with a grouping the rows do not arrive in. The +# argument is put into a table of its own on the way past, and read +# back off that column. +# +# Each group gathers the same row more than once, so what is +# gathered reads the same whichever order the rows arrive in - an +# order a grouping does not fix and this file must not depend on. +# +CREATE TABLE t13 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) +CHARSET utf8mb4; +INSERT INTO t13 VALUES (1, 1, '{"a":1}'), (2, 2, '{"b":2}'); +CREATE TABLE t13m (g INT); +INSERT INTO t13m VALUES (1), (1), (2), (2); +FLUSH STATUS; +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b +WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +r +{"2":{"b": 2}, "2":{"b": 2}} +{"1":{"a": 1}, "1":{"a": 1}} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 14. The same shape gathered into an array instead. The table the +# aggregate gathers in is built out of the buffer's column rather +# than out of the function that filled it, and asks that column +# again for every element it is handed. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b +WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +r +[{"b": 2},{"b": 2}] +[{"a": 1},{"a": 1}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 15. The chain of 9, with a store into the FIRST table that comes +# up short. That column gives up its answer, and the second has to +# hear about it although it was built long before. The query is +# the one from 9, so the count is what 9 would have come to with +# neither column answering. +# +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +# +# 16. The same chain, with the row that crosses BETWEEN the two +# tables arriving short instead. The first column still answers +# for what it holds; it is the second that stops. +# +SET SESSION debug_dbug='+d,json_tmp_chain_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +# +# 17. The buffered aggregate of 14, with the store into the buffer +# coming up short. The aggregate's own table is filled from that +# buffer field to field rather than off an item, so this is the +# case that says the field-to-field route asks as well. +# +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b +WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +r +[{"b": 2},{"b": 2}] +[{"a": 1},{"a": 1}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +SET SESSION debug_dbug= @old_debug; +# +# 18. A grouping that needs no temporary table of its own still has +# to hold a value across the group it belongs to, and it holds it in +# a buffer rather than in a column. A HAVING or an ORDER BY reads +# what it reads out of that buffer. +# +# A buffer holds one value at a time and is filled right beside the +# item that made it, so what that item says about the value is put +# away together with the value and is about it still when it is read +# back. +# +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 +HAVING JSON_ARRAY(x) <> '[]'; +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 +ORDER BY JSON_ARRAY(x); +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# +# 19. The same with a producer that attests to the value it has +# just made rather than for every value it will ever make. A column +# cannot be told about one of those, being made before any of them +# exists; a buffer is filled after, so what it holds is attested to +# like anything else and nothing is read at all. +# +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t1 +HAVING JSON_ARRAY(x) <> '[]'; +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 20. A buffer filled from a COLUMN rather than from an item, which +# is the route a value takes when it is wide enough to be kept apart +# from the row it belongs to. What the column says is what the +# buffer puts away. +# +CREATE TABLE t20 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t20 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +FLUSH STATUS; +SELECT MAX(d.id) AS m, d.x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t20) d +HAVING JSON_ARRAY(y) <> '[]'; +m y +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t20; +# +# 21. A buffer that could not take what it was handed keeps what it +# held before, which nothing here can attest to. Nothing arrives +# that way of its own accord - it takes a failure to find the room - +# so it is arranged for. The query is the one from 18, so the count +# is what 18 would have come to with the buffer saying nothing. +# +SET SESSION debug_dbug='+d,json_copy_not_kept'; +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 +HAVING JSON_ARRAY(x) <> '[]'; +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +SET SESSION debug_dbug= @old_debug; +# +# 22. The same UNION with one branch that attests per evaluation +# rather than to every value it makes. One branch not attesting is +# the whole column not attesting, there being no telling which +# branch a row came out of once it is in there. So every row is +# read on the way out - four - on top of the two the JSON_EXTRACT +# branch read itself. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT JSON_OBJECT('a', id) FROM t1) d; +r +[{"a": 1}] +[{"b": 2}] +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 23. The same again with a branch that is a column, which says +# nothing about itself and so says nothing for the column it is +# written into either. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT j FROM t1) d; +r +[{"a": 1}] +[{"b": 2}] +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 24. Rows written one list at a time rather than one branch at a +# time, which is the same column with the same one producer per +# row and is asked the same way. +# +FLUSH STATUS; +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('{"b":2}', '$'))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +r +[{"a": 1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# And one list that answers per evaluation is the same column not +# answering, exactly as one branch is. +# +FLUSH STATUS; +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 5 +# +# 25. A recursive CTE, whose column is built once the parts that do +# not go round have agreed on a type - the parts that DO go round +# not being reached at all, so nothing there can be asked and the +# ones that were reached do not answer in their place. +# +FLUSH STATUS; +WITH RECURSIVE r AS ( +SELECT 1 AS n, JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT n + 1, JSON_OBJECT('n', n) FROM r WHERE n < 2) +SELECT JSON_ARRAY(x) AS y FROM r ORDER BY y; +y +[{"a": 1}] +[{"b": 2}] +[{"n": 1}] +[{"n": 1}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +# +# 26. A function that EDITS a document rather than building one. +# It composes its answer and then reads the whole of it back, that +# reading being where the spacing gets settled - unless what it was +# given was already spaced that way, which is a second thing such a +# column can say and says here. +# +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 27. The same over a column whose producer returns a document +# formatted some other way - JSON_QUERY copying a sub-document out as +# it stands rather than writing one. Attested, and not written +# for, so the reading back stays. +# +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 28. The formatting carried from one such table into the next, which +# the second column asks the first about at every row it is given. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# And taken back where what the first holds is formatted otherwise. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 29. The same across the copier that fills a grouping's table, +# which is a different way of putting a row across and is asked the +# same thing. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 +ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 30. And that copier taking it back, for the same reason and by +# asking the same source. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 +ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# Everything above is short enough that half its length is inside +# the limit, so nothing there was ever read to find out how deep it +# goes. These are not: at over a hundred characters the length says +# nothing useful, and what the column can say about the depth is the +# only thing left. +# +CREATE TABLE t31 (id INT, j VARCHAR(400) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t31 VALUES +(1, CONCAT('{"a":[1,2],"pad":"', REPEAT('x', 100), '"}')), +(2, CONCAT('{"b":[3,4],"pad":"', REPEAT('y', 100), '"}')); +# +# 31. Two long rows out of a base column. A column of a table the +# user asked for says nothing about how deep its values go any more +# than about anything else, so each row is read. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t31 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 32. The same through a temporary table. The producer counted the +# depth while it wrote and the column kept the figure, so the splice +# has what it needs without the length being asked. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 33. The same carried from one such table into the next, which the +# second column takes from the first at every row it is given. +# +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d1) d2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 34. An aggregate gathering its elements out of such a column, +# which is the only thing left there to ask. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 35. A producer that returns a piece of a document it was given +# rather than one it wrote counted nothing, so the column has no +# figure to keep and the reading stays. The count is what 32 would +# have come to with nothing said. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t31) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 36. A column written by more than one producer keeps the deepest +# of all its rows, so where every one of them counted, every row +# gets the figure. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 +UNION ALL +SELECT id, JSON_EXTRACT(j, '$.a') FROM t31) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# And one row that nobody counted is the whole column saying +# nothing, the deepest of the rows being unknown once one of them +# is. The same four rows, one pair of them uncounted. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 +UNION ALL +SELECT id, JSON_QUERY(j, '$') FROM t31) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +DROP TABLE t31; +# +# 37. Branches written in different character sets. A column takes +# one set for all of them, and a branch in another set is stored +# into it converting - except into the binary set, which converts +# nothing at all and keeps every byte under the other set's name. +# Rows that arrived that way are not the documents that were +# written, so the column takes its answer back and every row is +# read on the way out. +# +CREATE TABLE t37 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t37 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t37b (b BLOB); +INSERT INTO t37b VALUES ('{"b":2}'), ('[3,4]'); +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t37 +UNION ALL +SELECT JSON_QUERY(b, '$') FROM t37b) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# 38. The same two branches with the wide one alone, so that the +# column takes that same set and every character arrives as the one +# written. Nothing is taken back, and the count is the branches' +# own readings and nothing else - which is what 37 would have come +# to had the characters come across. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t37 +UNION ALL +SELECT JSON_QUERY(j, '$') FROM t37) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +DROP TABLE t37, t37b; +DROP TABLE t1; +DROP TABLE t13, t13m; diff --git a/mysql-test/main/func_json_tmp_scan_count.test b/mysql-test/main/func_json_tmp_scan_count.test new file mode 100644 index 0000000000000..f82b778c415e7 --- /dev/null +++ b/mysql-test/main/func_json_tmp_scan_count.test @@ -0,0 +1,551 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a value out of the server's own temporary table is +--echo # read to find out whether it is a document. +--echo # +--echo # No query can see the difference - the reading could only have found +--echo # out what was already known - so nothing but Json_scans says whether +--echo # it happened. The counts below are the record of it. A count that +--echo # goes up is a reading that came back. +--echo # +--echo # A debug build reads values back to check what was claimed about +--echo # them, and none of those readings are counted: they are the debug +--echo # build's work and not the server's. A count of 0 below therefore +--echo # means nothing read, not that a reading went uncounted. +--echo # + +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +--echo # +--echo # 1. TWO rows out of a base column, spliced into a new document. +--echo # A column says nothing about itself, so each row is read once and +--echo # that reading is not work that can be taken away. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 2. The same two rows, put through a function that attests to what +--echo # it returns, and spliced without a temporary table in between. +--echo # JSON_EXTRACT reads its input once; the splice reads nothing. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 3. The same, with a temporary table in between. What the function +--echo # answered is what the column was built to hold, so the count is the +--echo # function's own readings and nothing else. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 4. The same shape with a producer that answers per evaluation +--echo # rather than for every one of them. Nothing can be said about the +--echo # column at the time it is made, so each row is read on the way out. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 5. Through a GROUP BY, which writes a temporary table of its own. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1 GROUP BY id, j) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 6. A UNION, whose column has one producer per branch. The +--echo # branches are where those producers are all in reach, so what the +--echo # column answers is what every one of them answers, and the count +--echo # is their own readings and nothing else. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT JSON_EXTRACT(j, '$') FROM t1) d; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 7. A table the user asked for, which is not the server's own. +--echo # +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_EXTRACT(j, '$') FROM t1; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM tt; +SHOW STATUS LIKE 'Json_scans'; +DROP TEMPORARY TABLE tt; + +--echo # +--echo # 8. A store that came up short takes back what was said about the +--echo # column, so its rows are read on the way out like anything else. +--echo # +--echo # No function asks for less room than it goes on to use, so nothing +--echo # arrives short of its own accord and the short store is arranged +--echo # for here. The query is the one from 3, so the count below is what +--echo # 3 would have come to with nothing said about its column. +--echo # +SET @old_debug= @@SESSION.debug_dbug; +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 9. Two temporary tables one after the other. The second is built +--echo # before a row of the first exists, so what the first says then is +--echo # the most it will ever say. Every row put across asks it again, +--echo # and what is left to count is the first table's own readings. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 10. Asking what a materialized value IS, rather than splicing it. +--echo # JSON_TYPE reads its argument whatever anyone says about it: what is +--echo # promised is that the value is a document, not which one. +--echo # +FLUSH STATUS; +SELECT JSON_TYPE(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 11. An aggregate gathers its elements in a table of its own and +--echo # reads them back out of a record rather than off the item that put +--echo # them there. The column is what attests to them. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS r FROM t1; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 12. The same, with a producer that attests per evaluation rather +--echo # than to every value it makes. Its column is read on the way out, +--echo # so the total matches 11 from the other side: 11's two readings are +--echo # the producer's own, these two are the readings back 11 was spared. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS r FROM t1; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 13. An aggregate whose arguments are buffered before the group can +--echo # be built - a join with a grouping the rows do not arrive in. The +--echo # argument is put into a table of its own on the way past, and read +--echo # back off that column. +--echo # +--echo # Each group gathers the same row more than once, so what is +--echo # gathered reads the same whichever order the rows arrive in - an +--echo # order a grouping does not fix and this file must not depend on. +--echo # +CREATE TABLE t13 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) + CHARSET utf8mb4; +INSERT INTO t13 VALUES (1, 1, '{"a":1}'), (2, 2, '{"b":2}'); +CREATE TABLE t13m (g INT); +INSERT INTO t13m VALUES (1), (1), (2), (2); +FLUSH STATUS; +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b + WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 14. The same shape gathered into an array instead. The table the +--echo # aggregate gathers in is built out of the buffer's column rather +--echo # than out of the function that filled it, and asks that column +--echo # again for every element it is handed. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b + WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 15. The chain of 9, with a store into the FIRST table that comes +--echo # up short. That column gives up its answer, and the second has to +--echo # hear about it although it was built long before. The query is +--echo # the one from 9, so the count is what 9 would have come to with +--echo # neither column answering. +--echo # +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 16. The same chain, with the row that crosses BETWEEN the two +--echo # tables arriving short instead. The first column still answers +--echo # for what it holds; it is the second that stops. +--echo # +SET SESSION debug_dbug='+d,json_tmp_chain_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 17. The buffered aggregate of 14, with the store into the buffer +--echo # coming up short. The aggregate's own table is filled from that +--echo # buffer field to field rather than off an item, so this is the +--echo # case that says the field-to-field route asks as well. +--echo # +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b + WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 18. A grouping that needs no temporary table of its own still has +--echo # to hold a value across the group it belongs to, and it holds it in +--echo # a buffer rather than in a column. A HAVING or an ORDER BY reads +--echo # what it reads out of that buffer. +--echo # +--echo # A buffer holds one value at a time and is filled right beside the +--echo # item that made it, so what that item says about the value is put +--echo # away together with the value and is about it still when it is read +--echo # back. +--echo # +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 + HAVING JSON_ARRAY(x) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 + ORDER BY JSON_ARRAY(x); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 19. The same with a producer that attests to the value it has +--echo # just made rather than for every value it will ever make. A column +--echo # cannot be told about one of those, being made before any of them +--echo # exists; a buffer is filled after, so what it holds is attested to +--echo # like anything else and nothing is read at all. +--echo # +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t1 + HAVING JSON_ARRAY(x) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 20. A buffer filled from a COLUMN rather than from an item, which +--echo # is the route a value takes when it is wide enough to be kept apart +--echo # from the row it belongs to. What the column says is what the +--echo # buffer puts away. +--echo # +CREATE TABLE t20 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t20 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +FLUSH STATUS; +SELECT MAX(d.id) AS m, d.x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t20) d + HAVING JSON_ARRAY(y) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t20; + +--echo # +--echo # 21. A buffer that could not take what it was handed keeps what it +--echo # held before, which nothing here can attest to. Nothing arrives +--echo # that way of its own accord - it takes a failure to find the room - +--echo # so it is arranged for. The query is the one from 18, so the count +--echo # is what 18 would have come to with the buffer saying nothing. +--echo # +SET SESSION debug_dbug='+d,json_copy_not_kept'; +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 + HAVING JSON_ARRAY(x) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 22. The same UNION with one branch that attests per evaluation +--echo # rather than to every value it makes. One branch not attesting is +--echo # the whole column not attesting, there being no telling which +--echo # branch a row came out of once it is in there. So every row is +--echo # read on the way out - four - on top of the two the JSON_EXTRACT +--echo # branch read itself. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT JSON_OBJECT('a', id) FROM t1) d; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 23. The same again with a branch that is a column, which says +--echo # nothing about itself and so says nothing for the column it is +--echo # written into either. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT j FROM t1) d; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 24. Rows written one list at a time rather than one branch at a +--echo # time, which is the same column with the same one producer per +--echo # row and is asked the same way. +--echo # +FLUSH STATUS; +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('{"b":2}', '$'))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +SHOW STATUS LIKE 'Json_scans'; +--echo # +--echo # And one list that answers per evaluation is the same column not +--echo # answering, exactly as one branch is. +--echo # +FLUSH STATUS; +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 25. A recursive CTE, whose column is built once the parts that do +--echo # not go round have agreed on a type - the parts that DO go round +--echo # not being reached at all, so nothing there can be asked and the +--echo # ones that were reached do not answer in their place. +--echo # +FLUSH STATUS; +WITH RECURSIVE r AS ( + SELECT 1 AS n, JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT n + 1, JSON_OBJECT('n', n) FROM r WHERE n < 2) +SELECT JSON_ARRAY(x) AS y FROM r ORDER BY y; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 26. A function that EDITS a document rather than building one. +--echo # It composes its answer and then reads the whole of it back, that +--echo # reading being where the spacing gets settled - unless what it was +--echo # given was already spaced that way, which is a second thing such a +--echo # column can say and says here. +--echo # +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 27. The same over a column whose producer returns a document +--echo # formatted some other way - JSON_QUERY copying a sub-document out as +--echo # it stands rather than writing one. Attested, and not written +--echo # for, so the reading back stays. +--echo # +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 28. The formatting carried from one such table into the next, which +--echo # the second column asks the first about at every row it is given. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +--echo # +--echo # And taken back where what the first holds is formatted otherwise. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 29. The same across the copier that fills a grouping's table, +--echo # which is a different way of putting a row across and is asked the +--echo # same thing. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 + ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 30. And that copier taking it back, for the same reason and by +--echo # asking the same source. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 + ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Everything above is short enough that half its length is inside +--echo # the limit, so nothing there was ever read to find out how deep it +--echo # goes. These are not: at over a hundred characters the length says +--echo # nothing useful, and what the column can say about the depth is the +--echo # only thing left. +--echo # +CREATE TABLE t31 (id INT, j VARCHAR(400) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t31 VALUES + (1, CONCAT('{"a":[1,2],"pad":"', REPEAT('x', 100), '"}')), + (2, CONCAT('{"b":[3,4],"pad":"', REPEAT('y', 100), '"}')); + +--echo # +--echo # 31. Two long rows out of a base column. A column of a table the +--echo # user asked for says nothing about how deep its values go any more +--echo # than about anything else, so each row is read. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(j) AS r FROM t31 ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 32. The same through a temporary table. The producer counted the +--echo # depth while it wrote and the column kept the figure, so the splice +--echo # has what it needs without the length being asked. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 33. The same carried from one such table into the next, which the +--echo # second column takes from the first at every row it is given. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(y) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d1) d2 ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 34. An aggregate gathering its elements out of such a column, +--echo # which is the only thing left there to ask. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 35. A producer that returns a piece of a document it was given +--echo # rather than one it wrote counted nothing, so the column has no +--echo # figure to keep and the reading stays. The count is what 32 would +--echo # have come to with nothing said. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t31) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 36. A column written by more than one producer keeps the deepest +--echo # of all its rows, so where every one of them counted, every row +--echo # gets the figure. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 + UNION ALL + SELECT id, JSON_EXTRACT(j, '$.a') FROM t31) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And one row that nobody counted is the whole column saying +--echo # nothing, the deepest of the rows being unknown once one of them +--echo # is. The same four rows, one pair of them uncounted. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 + UNION ALL + SELECT id, JSON_QUERY(j, '$') FROM t31) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t31; + +--echo # +--echo # 37. Branches written in different character sets. A column takes +--echo # one set for all of them, and a branch in another set is stored +--echo # into it converting - except into the binary set, which converts +--echo # nothing at all and keeps every byte under the other set's name. +--echo # Rows that arrived that way are not the documents that were +--echo # written, so the column takes its answer back and every row is +--echo # read on the way out. +--echo # +CREATE TABLE t37 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t37 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t37b (b BLOB); +INSERT INTO t37b VALUES ('{"b":2}'), ('[3,4]'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t37 + UNION ALL + SELECT JSON_QUERY(b, '$') FROM t37b) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 38. The same two branches with the wide one alone, so that the +--echo # column takes that same set and every character arrives as the one +--echo # written. Nothing is taken back, and the count is the branches' +--echo # own readings and nothing else - which is what 37 would have come +--echo # to had the characters come across. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t37 + UNION ALL + SELECT JSON_QUERY(j, '$') FROM t37) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t37, t37b; + +DROP TABLE t1; +DROP TABLE t13, t13m; diff --git a/mysql-test/main/func_json_tmp_trust.result b/mysql-test/main/func_json_tmp_trust.result new file mode 100644 index 0000000000000..d2242effe9525 --- /dev/null +++ b/mysql-test/main/func_json_tmp_trust.result @@ -0,0 +1,584 @@ +# +# What a JSON function reads back out of a temporary table the server +# built for itself. +# +# Such a table is written, read and thrown away inside one query, and +# no statement can name it, so what goes into a column of one is +# decided once, when the column is made. A column whose one producer +# always gives back a document therefore holds documents, and a +# reader of it can be spared finding that out again. +# +# A column of a table anybody can name says nothing of the sort: a row +# can hold bytes no check ever saw, and nothing in one records where +# they came from. The answers below are the same either way - the +# reading that is left out could only have found out what was already +# known - so this file is here to say that they stay the same. +# +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); +# +# 1. A document out of a temporary table, spliced into a new one. +# +SELECT id, JSON_ARRAY(x) AS spliced FROM +(SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +id spliced +1 [{"a": 1, "c": 1}] +2 [[1, 2]] +SELECT id, JSON_OBJECT('k', x) AS spliced FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +id spliced +1 {"k": {"a": 1}} +2 {"k": [1, 2]} +# +# 2. The same, edited rather than spliced. +# +SELECT id, JSON_SET(x, '$.d', 4) AS edited FROM +(SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +id edited +1 {"a": 1, "c": 1, "d": 4} +2 [1, 2] +SELECT id, JSON_MERGE_PATCH(x, '{"e":5}') AS merged FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +id merged +1 {"a": 1, "e": 5} +2 {"e": 5} +# +# 3. Asked about, rather than read. +# +SELECT id, JSON_VALID(x) AS valid, JSON_TYPE(x) AS what, JSON_DEPTH(x) AS deep +FROM (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +id valid what deep +1 1 OBJECT 2 +2 1 ARRAY 2 +# +# 4. Through a GROUP BY, a DISTINCT and an ORDER BY, each of which +# writes a temporary table of its own. +# +SELECT JSON_ARRAY(x) AS grouped FROM +(SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1 GROUP BY id, j) d +ORDER BY x; +grouped +[[1, 2]] +[{"a": 1, "c": 1}] +SELECT DISTINCT JSON_ARRAY(x) AS distincted FROM +(SELECT JSON_SET(j, '$.c', 1) AS x FROM t1) d ORDER BY 1; +distincted +[[1, 2]] +[{"a": 1, "c": 1}] +# +# 5. Through a UNION, whose column has one producer per branch and +# answers what all of them answer - the branches being where they +# are all in reach. One that does not answer is the whole column +# not answering, there being no telling afterwards which branch a +# row came out of. +# +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 +UNION ALL +SELECT j FROM t1) d ORDER BY 1; +unioned +[[1, 2]] +[[1,2]] +[{"a": 1, "c": 1}] +[{"a":1}] +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 +UNION ALL +SELECT JSON_EXTRACT(j, '$') FROM t1) d ORDER BY 1; +unioned +[[1, 2]] +[[1, 2]] +[{"a": 1, "c": 1}] +[{"a": 1}] +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION +SELECT JSON_QUERY(j, '$') FROM t1) d ORDER BY 1; +unioned +[[1, 2]] +[[1,2]] +[{"a": 1}] +[{"a":1}] +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +EXCEPT +SELECT JSON_EXTRACT(j, '$') FROM t1 WHERE id = 9) d ORDER BY 1; +unioned +[[1, 2]] +[{"a": 1}] +# +# The same column written one list of values at a time. +# +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('[1,2]', '$'))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +unioned +[[1,2]] +[{"a": 1}] +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +unioned +[{"a": 1}] +[{"b": 2}] +# +# A column of a recursive CTE is built once the parts that do not go +# round have agreed on a type, the parts that DO go round not being +# reached at all. So a value can arrive there from a producer +# nothing asked, and here one arrives that is no document: a column +# typed JSON holds whatever was put in it while its check was off. +# +CREATE TABLE t5 (good JSON, bad JSON); +SET @old_check= @@SESSION.check_constraint_checks; +SET SESSION check_constraint_checks= OFF; +INSERT INTO t5 VALUES ('{"a":1}', 'not a document'); +SET SESSION check_constraint_checks= @old_check; +WITH RECURSIVE r AS ( +SELECT 1 AS n, JSON_EXTRACT(good, '$') AS x FROM t5 +UNION ALL +SELECT n + 1, t5.bad FROM r, t5 WHERE n < 2) +SELECT n, JSON_ARRAY(x) AS unioned FROM r ORDER BY n; +n unioned +1 [{"a": 1}] +2 [not a document] +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +DROP TABLE t5; +# +# Branches written in different character sets. The column takes +# one set for all of them, and a branch whose own set is not that +# one is stored into it converting. A store into the binary set +# converts nothing: it keeps every byte and calls them by the other +# set's name, so a document written in a wide set arrives as bytes +# that nothing reads as a document, and it arrives at exactly the +# length it was written at. A byte count is therefore the one +# measure that cannot tell this apart, and each branch is asked +# about its characters as well as its length. +# +CREATE TABLE t6 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t6 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t7 (b BLOB); +INSERT INTO t7 VALUES ('{"b":2}'), ('[3,4]'); +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, +HEX(x) AS bytes FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t6 +UNION ALL +SELECT JSON_QUERY(b, '$') FROM t7) d ORDER BY bytes; +packed still_a_document bytes +5B005B0031002C0032005D5D 0 005B0031002C0032005D +5B007B002200610022003A0031007D5D 0 007B002200610022003A0031007D +5B5B332C345D5D 1 5B332C345D +5B7B2262223A327D5D 1 7B2262223A327D +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +# +# The same two branches with the wide one on the right, so that it +# is the second producer rather than the first that has to move. +# +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, +HEX(x) AS bytes FROM +(SELECT JSON_QUERY(b, '$') AS x FROM t7 +UNION ALL +SELECT JSON_QUERY(j, '$') FROM t6) d ORDER BY bytes; +packed still_a_document bytes +5B005B0031002C0032005D5D 0 005B0031002C0032005D +5B007B002200610022003A0031007D5D 0 007B002200610022003A0031007D +5B5B332C345D5D 1 5B332C345D +5B7B2262223A327D5D 1 7B2262223A327D +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +# +# And both branches in the wide set, where the column takes that +# same set and every character comes across as the one written. +# +SELECT JSON_ARRAY(x) AS unioned, JSON_VALID(x) AS still_a_document FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t6 +UNION ALL +SELECT JSON_QUERY(j, '$') FROM t6 WHERE j <> '[1,2]') d ORDER BY 1; +unioned still_a_document +[[1,2]] 1 +[{"a":1}] 1 +[{"a":1}] 1 +DROP TABLE t6, t7; +# +# 6. A producer that is a document only some of the time keeps its +# column untrusted. JSON_OBJECT answers per evaluation, not for the +# class, so a column fed by one is read like any other. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY 1; +spliced +[{"a": 1}] +[{"a": 2}] +# +# 7. A function that returns more than it was given. JSON_REMOVE +# writes a space after every separator it copies, so taking one +# element out of a dense array returns more than there was to begin +# with, and the column it goes into has to have been asked for wide +# enough to hold it. +# +CREATE TABLE t7 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t7 VALUES +('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), +('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); +SELECT LENGTH(j) AS given, LENGTH(JSON_REMOVE(j, '$[0]')) AS produced +FROM t7 ORDER BY 1, 2; +given produced +61 87 +61 87 +SELECT LENGTH(x) AS kept, JSON_VALID(x) AS still_a_document FROM +(SELECT JSON_REMOVE(j, '$[0]') AS x FROM t7) d ORDER BY 1; +kept still_a_document +87 1 +87 1 +DROP TABLE t7; +# +# The same function on a document that leaves room to spare. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_REMOVE(j, '$.a') AS x FROM t1 WHERE id = 1) d; +spliced +[{}] +# +# 8. A table the user asked for is not the server's own, whatever is +# written into it and whatever check it carries. +# +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_SET(j, '$.c', id) FROM t1; +SELECT JSON_ARRAY(x) AS spliced FROM tt ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +DROP TEMPORARY TABLE tt; +# +# 9. A materialized subquery. +# +SELECT id FROM t1 WHERE JSON_ARRAY(j) IN +(SELECT JSON_ARRAY(x) FROM (SELECT j AS x FROM t1) d) ORDER BY id; +id +1 +2 +# +# 10. Two temporary tables one after the other. The second was +# built before a row of the first existed, so what it was given +# then is only as much as the first can still say later, and it +# asks again for every row put across. +# +SELECT JSON_ARRAY(y) AS spliced FROM +(SELECT x AS y FROM +(SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2 ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +# +# The same three deep, and with a grouping in the middle so the row +# crosses by more than one route. +# +SELECT JSON_ARRAY(z) AS spliced FROM +(SELECT y AS z FROM +(SELECT x AS y FROM +(SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2) d3 ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +SELECT JSON_ARRAY(y) AS spliced FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY x) d2 ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1}] +# +# 11. A NULL, which is not a document and is not read as one. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_SET(NULL, '$.c', 1) AS x FROM t1) d ORDER BY 1; +spliced +[null] +[null] +# +# 12. A document written in one character set and read in another. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT CONVERT(JSON_SET(j, '$.c', 1) USING latin1) AS x FROM t1) d +ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_SET(CONVERT(j USING latin1), '$.c', 1) AS x FROM t1) d +ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +DROP TABLE t1; +# +# 13. A window function's temporary table. +# +CREATE TABLE t2 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) +CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, 1, '{"a":1}'), (2, 1, '{"b":2}'), (3, 2, '[3]'); +SELECT id, JSON_ARRAY(x) AS spliced FROM +(SELECT id, JSON_SET(j, '$.n', +ROW_NUMBER() OVER (PARTITION BY g ORDER BY id)) AS x +FROM t2) d ORDER BY id; +id spliced +1 [{"a": 1, "n": 1}] +2 [{"b": 2, "n": 2}] +3 [[3]] +# +# 14. An aggregate, which gathers its elements in a table of its own +# and reads them back out of a record. +# +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS gathered FROM t2; +gathered +[{"a": 1},{"b": 2},[3]] +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS gathered FROM t2; +gathered +[{"a": 1},{"a": 2},{"a": 3}] +SELECT g, JSON_OBJECTAGG(id, JSON_EXTRACT(j, '$')) AS gathered FROM t2 +GROUP BY g ORDER BY g; +g gathered +1 {"1":{"a": 1}, "2":{"b": 2}} +2 {"3":[3]} +# +# 15. The same, with the arguments buffered before the group can be +# built. Each group gathers the same row more than once, so what is +# gathered reads the same whichever order the rows arrive in - an +# order a grouping does not fix and this file must not depend on. +# +CREATE TABLE t15m (g INT); +INSERT INTO t15m VALUES (1), (1), (2), (2); +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS gathered +FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) +GROUP BY a.g ORDER BY a.g DESC; +gathered +{"3":[3], "3":[3]} +{"1":{"a": 1}, "1":{"a": 1}} +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS gathered +FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) +GROUP BY a.g ORDER BY a.g DESC; +gathered +[[3],[3]] +[{"a": 1},{"a": 1}] +DROP TABLE t15m; +# +# 16. A value the aggregate cannot carry as a document. +# +SELECT JSON_ARRAYAGG(j) AS gathered FROM t2; +gathered +[{"a":1},{"b":2},[3]] +# +# 17. A grouping that writes no temporary table holds its values in +# buffers instead, and a HAVING or an ORDER BY reads them back out +# of those rather than out of a record. +# +SELECT MAX(id) AS m, JSON_SET(j, '$.c', 1) AS x FROM t2 +HAVING JSON_ARRAY(x) <> '[]'; +m x +3 {"a": 1, "c": 1} +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 +ORDER BY JSON_ARRAY(x); +m x +3 {"a": 1} +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t2 +HAVING JSON_ARRAY(x) <> '[]'; +m x +3 {"a": 1} +# +# The same where the buffer is filled from a column rather than +# from an item, which is the route a value wide enough to be kept +# apart from its row takes. +# +CREATE TABLE t17 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t17 VALUES (1, '{"a":1}'), (2, '[1,2]'); +SELECT MAX(d.id) AS m, d.x AS y FROM +(SELECT id, JSON_SET(j, '$.c', 1) AS x FROM t17) d +HAVING JSON_ARRAY(y) <> '[]'; +m y +2 {"a": 1, "c": 1} +SELECT MAX(d.id) AS m, d.j AS y FROM t17 d +HAVING JSON_ARRAY(y) <> '[]'; +m y +2 {"a":1} +DROP TABLE t17; +# +# A NULL, and a character set conversion around the producer - +# which carries the value through while the type stops at it, so the +# buffer attests to nothing and the value is quoted, not spliced. +# +SELECT MAX(id) AS m, JSON_SET(NULL, '$.c', 1) AS x FROM t2 +HAVING JSON_ARRAY(x) IS NULL; +m x +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 +ORDER BY JSON_ARRAY(x); +m x +3 {"a": 1} +# +# Quoted rather than spliced is the whole of what the buffer was +# asked, and a buffer is read from a HAVING or an ORDER BY and never +# from the select list - so what went into the document is only to +# be had through a comparison. Asking what its first element IS +# tells the two apart, and the row comes back for whichever +# happened. +# +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +m x +3 {"a": 1} +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +m x +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'STRING'; +m x +3 {"a": 1} +# +# And the same conversion with no buffer in the way IS looked +# through, which is what makes the two above a fact about the buffer +# rather than about the conversion. +# +SELECT id, JSON_ARRAY(CONVERT(JSON_EXTRACT(j, '$') USING latin1)) AS r +FROM t2 ORDER BY id; +id r +1 [{"a": 1}] +2 [{"b": 2}] +3 [[3]] +# +# 18. How a value is FORMATTED, which is a second thing such a column +# can attest to and is come by differently. Being a document is +# promised by the producer before a row exists; the formatting is +# settled one value at a time, so it is granted along with the first +# answer and spent by whatever gets written afterwards. +# +CREATE TABLE t18 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t18 VALUES (1, '{"a":1}'), (2, '[1,2]'); +# +# Edited out of a column whose producer writes the loose way, out of +# one that copies a sub-document in whatever way it found it, and out +# of one that writes a form of its own. +# +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM +(SELECT id, JSON_NORMALIZE(j) AS x FROM t18) d ORDER BY id; +id r +1 {"a": 1.0E0, "c": 1} +2 [1.0E0, 2.0E0] +# +# Carried from one such table into the next, and taken back there +# where what the first holds was written some other way. +# +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +# +# The same across the copier that fills a grouping's table. +# +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 +ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 +ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +# +# An aggregate reads its elements out of a record, the column being +# the only thing left there to ask - so what it says about their +# formatting is what the gathered document can say about its own. +# +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d; +r +[[{"a": 1},[1, 2]]] +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d; +r +[[{"a":1},[1,2]]] +DROP TABLE t18; +# +# 19. How deep the values went, which is a figure rather than a yes. +# +# What a column keeps is the deepest of the rows written into it so +# far, not the depth of the row being read - more than that row +# needs and never less. More is the direction this one is allowed +# to be wrong in: a figure too large costs a reading that could have +# been skipped, while one too small would admit a document that +# nothing can read back. Nothing acts on the figure except to skip +# a reading, so the answers below are the ones the server has always +# given. +# +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), +'1', REPEAT('}', 30)); +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), +'1', REPEAT('}', 31)); +CREATE TABLE t19 (id INT, j LONGTEXT) CHARSET utf8mb4; +INSERT INTO t19 VALUES (1, @k30), (2, '{"a":1}'); +# +# One row at the limit and a shallow one beside it, put one level +# in. The column answers 30 for both, which is exact for the first +# and too large for the second, and each still ends up where it +# belongs. +# +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +id d +1 32 +2 3 +# +# A byte past it, where the same shape is refused - and the shallow +# row in the same column is not, the figure being about the column +# and the limit about each document that comes out of it. +# +UPDATE t19 SET j= @k31 WHERE id= 1; +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +id d +1 NULL +2 3 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 452 +# +# Each level of these is fourteen characters, so half the length is +# seven times the limit and the length can tell neither of them from +# the other. Only the figure the column kept can. +# +SELECT JSON_DEPTH(JSON_ARRAY(y)) AS d FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d1 WHERE id= 2) d2; +d +3 +DROP TABLE t19; +DROP TABLE t2; diff --git a/mysql-test/main/func_json_tmp_trust.test b/mysql-test/main/func_json_tmp_trust.test new file mode 100644 index 0000000000000..2fb43e3e0f03d --- /dev/null +++ b/mysql-test/main/func_json_tmp_trust.test @@ -0,0 +1,421 @@ +--echo # +--echo # What a JSON function reads back out of a temporary table the server +--echo # built for itself. +--echo # +--echo # Such a table is written, read and thrown away inside one query, and +--echo # no statement can name it, so what goes into a column of one is +--echo # decided once, when the column is made. A column whose one producer +--echo # always gives back a document therefore holds documents, and a +--echo # reader of it can be spared finding that out again. +--echo # +--echo # A column of a table anybody can name says nothing of the sort: a row +--echo # can hold bytes no check ever saw, and nothing in one records where +--echo # they came from. The answers below are the same either way - the +--echo # reading that is left out could only have found out what was already +--echo # known - so this file is here to say that they stay the same. +--echo # + +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); + +--echo # +--echo # 1. A document out of a temporary table, spliced into a new one. +--echo # +SELECT id, JSON_ARRAY(x) AS spliced FROM + (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +SELECT id, JSON_OBJECT('k', x) AS spliced FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; + +--echo # +--echo # 2. The same, edited rather than spliced. +--echo # +SELECT id, JSON_SET(x, '$.d', 4) AS edited FROM + (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +SELECT id, JSON_MERGE_PATCH(x, '{"e":5}') AS merged FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; + +--echo # +--echo # 3. Asked about, rather than read. +--echo # +SELECT id, JSON_VALID(x) AS valid, JSON_TYPE(x) AS what, JSON_DEPTH(x) AS deep + FROM (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; + +--echo # +--echo # 4. Through a GROUP BY, a DISTINCT and an ORDER BY, each of which +--echo # writes a temporary table of its own. +--echo # +SELECT JSON_ARRAY(x) AS grouped FROM + (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1 GROUP BY id, j) d + ORDER BY x; +SELECT DISTINCT JSON_ARRAY(x) AS distincted FROM + (SELECT JSON_SET(j, '$.c', 1) AS x FROM t1) d ORDER BY 1; + +--echo # +--echo # 5. Through a UNION, whose column has one producer per branch and +--echo # answers what all of them answer - the branches being where they +--echo # are all in reach. One that does not answer is the whole column +--echo # not answering, there being no telling afterwards which branch a +--echo # row came out of. +--echo # +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 + UNION ALL + SELECT j FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 + UNION ALL + SELECT JSON_EXTRACT(j, '$') FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION + SELECT JSON_QUERY(j, '$') FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + EXCEPT + SELECT JSON_EXTRACT(j, '$') FROM t1 WHERE id = 9) d ORDER BY 1; +--echo # +--echo # The same column written one list of values at a time. +--echo # +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('[1,2]', '$'))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +--echo # +--echo # A column of a recursive CTE is built once the parts that do not go +--echo # round have agreed on a type, the parts that DO go round not being +--echo # reached at all. So a value can arrive there from a producer +--echo # nothing asked, and here one arrives that is no document: a column +--echo # typed JSON holds whatever was put in it while its check was off. +--echo # +CREATE TABLE t5 (good JSON, bad JSON); +SET @old_check= @@SESSION.check_constraint_checks; +SET SESSION check_constraint_checks= OFF; +INSERT INTO t5 VALUES ('{"a":1}', 'not a document'); +SET SESSION check_constraint_checks= @old_check; +WITH RECURSIVE r AS ( + SELECT 1 AS n, JSON_EXTRACT(good, '$') AS x FROM t5 + UNION ALL + SELECT n + 1, t5.bad FROM r, t5 WHERE n < 2) +SELECT n, JSON_ARRAY(x) AS unioned FROM r ORDER BY n; +DROP TABLE t5; +--echo # +--echo # Branches written in different character sets. The column takes +--echo # one set for all of them, and a branch whose own set is not that +--echo # one is stored into it converting. A store into the binary set +--echo # converts nothing: it keeps every byte and calls them by the other +--echo # set's name, so a document written in a wide set arrives as bytes +--echo # that nothing reads as a document, and it arrives at exactly the +--echo # length it was written at. A byte count is therefore the one +--echo # measure that cannot tell this apart, and each branch is asked +--echo # about its characters as well as its length. +--echo # +CREATE TABLE t6 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t6 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t7 (b BLOB); +INSERT INTO t7 VALUES ('{"b":2}'), ('[3,4]'); +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, + HEX(x) AS bytes FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t6 + UNION ALL + SELECT JSON_QUERY(b, '$') FROM t7) d ORDER BY bytes; +--echo # +--echo # The same two branches with the wide one on the right, so that it +--echo # is the second producer rather than the first that has to move. +--echo # +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, + HEX(x) AS bytes FROM + (SELECT JSON_QUERY(b, '$') AS x FROM t7 + UNION ALL + SELECT JSON_QUERY(j, '$') FROM t6) d ORDER BY bytes; +--echo # +--echo # And both branches in the wide set, where the column takes that +--echo # same set and every character comes across as the one written. +--echo # +SELECT JSON_ARRAY(x) AS unioned, JSON_VALID(x) AS still_a_document FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t6 + UNION ALL + SELECT JSON_QUERY(j, '$') FROM t6 WHERE j <> '[1,2]') d ORDER BY 1; +DROP TABLE t6, t7; + +--echo # +--echo # 6. A producer that is a document only some of the time keeps its +--echo # column untrusted. JSON_OBJECT answers per evaluation, not for the +--echo # class, so a column fed by one is read like any other. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY 1; + +--echo # +--echo # 7. A function that returns more than it was given. JSON_REMOVE +--echo # writes a space after every separator it copies, so taking one +--echo # element out of a dense array returns more than there was to begin +--echo # with, and the column it goes into has to have been asked for wide +--echo # enough to hold it. +--echo # +CREATE TABLE t7 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t7 VALUES + ('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), + ('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); +SELECT LENGTH(j) AS given, LENGTH(JSON_REMOVE(j, '$[0]')) AS produced + FROM t7 ORDER BY 1, 2; +SELECT LENGTH(x) AS kept, JSON_VALID(x) AS still_a_document FROM + (SELECT JSON_REMOVE(j, '$[0]') AS x FROM t7) d ORDER BY 1; +DROP TABLE t7; +--echo # +--echo # The same function on a document that leaves room to spare. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_REMOVE(j, '$.a') AS x FROM t1 WHERE id = 1) d; + +--echo # +--echo # 8. A table the user asked for is not the server's own, whatever is +--echo # written into it and whatever check it carries. +--echo # +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_SET(j, '$.c', id) FROM t1; +SELECT JSON_ARRAY(x) AS spliced FROM tt ORDER BY 1; +DROP TEMPORARY TABLE tt; + +--echo # +--echo # 9. A materialized subquery. +--echo # +SELECT id FROM t1 WHERE JSON_ARRAY(j) IN + (SELECT JSON_ARRAY(x) FROM (SELECT j AS x FROM t1) d) ORDER BY id; + +--echo # +--echo # 10. Two temporary tables one after the other. The second was +--echo # built before a row of the first existed, so what it was given +--echo # then is only as much as the first can still say later, and it +--echo # asks again for every row put across. +--echo # +SELECT JSON_ARRAY(y) AS spliced FROM + (SELECT x AS y FROM + (SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2 ORDER BY 1; +--echo # +--echo # The same three deep, and with a grouping in the middle so the row +--echo # crosses by more than one route. +--echo # +SELECT JSON_ARRAY(z) AS spliced FROM + (SELECT y AS z FROM + (SELECT x AS y FROM + (SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2) d3 ORDER BY 1; +SELECT JSON_ARRAY(y) AS spliced FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY x) d2 ORDER BY 1; + +--echo # +--echo # 11. A NULL, which is not a document and is not read as one. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_SET(NULL, '$.c', 1) AS x FROM t1) d ORDER BY 1; + +--echo # +--echo # 12. A document written in one character set and read in another. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT CONVERT(JSON_SET(j, '$.c', 1) USING latin1) AS x FROM t1) d + ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_SET(CONVERT(j USING latin1), '$.c', 1) AS x FROM t1) d + ORDER BY 1; + +DROP TABLE t1; + +--echo # +--echo # 13. A window function's temporary table. +--echo # +CREATE TABLE t2 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) + CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, 1, '{"a":1}'), (2, 1, '{"b":2}'), (3, 2, '[3]'); +SELECT id, JSON_ARRAY(x) AS spliced FROM + (SELECT id, JSON_SET(j, '$.n', + ROW_NUMBER() OVER (PARTITION BY g ORDER BY id)) AS x + FROM t2) d ORDER BY id; + +--echo # +--echo # 14. An aggregate, which gathers its elements in a table of its own +--echo # and reads them back out of a record. +--echo # +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS gathered FROM t2; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS gathered FROM t2; +SELECT g, JSON_OBJECTAGG(id, JSON_EXTRACT(j, '$')) AS gathered FROM t2 + GROUP BY g ORDER BY g; + +--echo # +--echo # 15. The same, with the arguments buffered before the group can be +--echo # built. Each group gathers the same row more than once, so what is +--echo # gathered reads the same whichever order the rows arrive in - an +--echo # order a grouping does not fix and this file must not depend on. +--echo # +CREATE TABLE t15m (g INT); +INSERT INTO t15m VALUES (1), (1), (2), (2); +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS gathered + FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) + GROUP BY a.g ORDER BY a.g DESC; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS gathered + FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) + GROUP BY a.g ORDER BY a.g DESC; +DROP TABLE t15m; + +--echo # +--echo # 16. A value the aggregate cannot carry as a document. +--echo # +SELECT JSON_ARRAYAGG(j) AS gathered FROM t2; + +--echo # +--echo # 17. A grouping that writes no temporary table holds its values in +--echo # buffers instead, and a HAVING or an ORDER BY reads them back out +--echo # of those rather than out of a record. +--echo # +SELECT MAX(id) AS m, JSON_SET(j, '$.c', 1) AS x FROM t2 + HAVING JSON_ARRAY(x) <> '[]'; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 + ORDER BY JSON_ARRAY(x); +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t2 + HAVING JSON_ARRAY(x) <> '[]'; +--echo # +--echo # The same where the buffer is filled from a column rather than +--echo # from an item, which is the route a value wide enough to be kept +--echo # apart from its row takes. +--echo # +CREATE TABLE t17 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t17 VALUES (1, '{"a":1}'), (2, '[1,2]'); +SELECT MAX(d.id) AS m, d.x AS y FROM + (SELECT id, JSON_SET(j, '$.c', 1) AS x FROM t17) d + HAVING JSON_ARRAY(y) <> '[]'; +SELECT MAX(d.id) AS m, d.j AS y FROM t17 d + HAVING JSON_ARRAY(y) <> '[]'; +DROP TABLE t17; +--echo # +--echo # A NULL, and a character set conversion around the producer - +--echo # which carries the value through while the type stops at it, so the +--echo # buffer attests to nothing and the value is quoted, not spliced. +--echo # +SELECT MAX(id) AS m, JSON_SET(NULL, '$.c', 1) AS x FROM t2 + HAVING JSON_ARRAY(x) IS NULL; +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 + ORDER BY JSON_ARRAY(x); +--echo # +--echo # Quoted rather than spliced is the whole of what the buffer was +--echo # asked, and a buffer is read from a HAVING or an ORDER BY and never +--echo # from the select list - so what went into the document is only to +--echo # be had through a comparison. Asking what its first element IS +--echo # tells the two apart, and the row comes back for whichever +--echo # happened. +--echo # +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'STRING'; +--echo # +--echo # And the same conversion with no buffer in the way IS looked +--echo # through, which is what makes the two above a fact about the buffer +--echo # rather than about the conversion. +--echo # +SELECT id, JSON_ARRAY(CONVERT(JSON_EXTRACT(j, '$') USING latin1)) AS r + FROM t2 ORDER BY id; + +--echo # +--echo # 18. How a value is FORMATTED, which is a second thing such a column +--echo # can attest to and is come by differently. Being a document is +--echo # promised by the producer before a row exists; the formatting is +--echo # settled one value at a time, so it is granted along with the first +--echo # answer and spent by whatever gets written afterwards. +--echo # +CREATE TABLE t18 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t18 VALUES (1, '{"a":1}'), (2, '[1,2]'); +--echo # +--echo # Edited out of a column whose producer writes the loose way, out of +--echo # one that copies a sub-document in whatever way it found it, and out +--echo # of one that writes a form of its own. +--echo # +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d ORDER BY id; +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d ORDER BY id; +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM + (SELECT id, JSON_NORMALIZE(j) AS x FROM t18) d ORDER BY id; +--echo # +--echo # Carried from one such table into the next, and taken back there +--echo # where what the first holds was written some other way. +--echo # +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +--echo # +--echo # The same across the copier that fills a grouping's table. +--echo # +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 + ORDER BY id; +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 + ORDER BY id; +--echo # +--echo # An aggregate reads its elements out of a record, the column being +--echo # the only thing left there to ask - so what it says about their +--echo # formatting is what the gathered document can say about its own. +--echo # +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d; +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d; +DROP TABLE t18; + +--echo # +--echo # 19. How deep the values went, which is a figure rather than a yes. +--echo # +--echo # What a column keeps is the deepest of the rows written into it so +--echo # far, not the depth of the row being read - more than that row +--echo # needs and never less. More is the direction this one is allowed +--echo # to be wrong in: a figure too large costs a reading that could have +--echo # been skipped, while one too small would admit a document that +--echo # nothing can read back. Nothing acts on the figure except to skip +--echo # a reading, so the answers below are the ones the server has always +--echo # given. +--echo # +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), + '1', REPEAT('}', 30)); +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), + '1', REPEAT('}', 31)); +CREATE TABLE t19 (id INT, j LONGTEXT) CHARSET utf8mb4; +INSERT INTO t19 VALUES (1, @k30), (2, '{"a":1}'); +--echo # +--echo # One row at the limit and a shallow one beside it, put one level +--echo # in. The column answers 30 for both, which is exact for the first +--echo # and too large for the second, and each still ends up where it +--echo # belongs. +--echo # +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +--echo # +--echo # A byte past it, where the same shape is refused - and the shallow +--echo # row in the same column is not, the figure being about the column +--echo # and the limit about each document that comes out of it. +--echo # +UPDATE t19 SET j= @k31 WHERE id= 1; +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +--echo # +--echo # Each level of these is fourteen characters, so half the length is +--echo # seven times the limit and the length can tell neither of them from +--echo # the other. Only the figure the column kept can. +--echo # +SELECT JSON_DEPTH(JSON_ARRAY(y)) AS d FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d1 WHERE id= 2) d2; +DROP TABLE t19; + +DROP TABLE t2; diff --git a/sql/derived_handler.cc b/sql/derived_handler.cc index cddd1200f5d0f..131aacd150a84 100644 --- a/sql/derived_handler.cc +++ b/sql/derived_handler.cc @@ -53,6 +53,12 @@ int Pushdown_derived::execute() DBUG_ENTER("Pushdown_derived::execute"); + /* + Dropped where the table is given to the engine - see + Pushdown_query::execute(). + */ + DBUG_ASSERT(!table->is_valid_json_static_set); + if ((err= handler->init_scan())) goto error; @@ -106,6 +112,12 @@ void derived_handler::set_derived(TABLE_LIST *tbl) { derived= tbl; table= tbl->table; + /* + The materialisation table was built by the server and is written by + this engine, row by row into record[0] - see + TABLE::set_filled_by_engine(). + */ + table->set_filled_by_engine(); unit= tbl->derived; select= unit->first_select(); tmp_table_param= ((select_unit *)(unit->result))->get_tmp_table_param(); diff --git a/sql/field.cc b/sql/field.cc index f16f0d952586d..1878d22c1acfd 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -36,6 +36,11 @@ #include "tztime.h" // struct Time_zone #include "filesort.h" // change_double_for_sort #include "log_event.h" // class Table_map_log_event +#include "sql_type_json.h" // Type_handler_json_common +#ifndef DBUG_OFF +#include "item_jsonfunc.h" // json_value_reads_as_document, + // json_value_is_nice +#endif #include // Maximum allowed exponent value for converting string to decimal @@ -2792,6 +2797,267 @@ void Field::set_is_valid_json(const Item *item, int store_rc) } +/* + The three of these are here rather than beside their siblings in + field.h for one reason: JSON_DEPTH_UNKNOWN is declared in item.h, + which a field header does not see and should not have to. The call + costs nothing beside the reading it exists to save. + + Asked through the standing answer, so a column that has given that up + has given the depth up with it and cannot be asked to return a + figure about values it no longer attests to. +*/ + +uint Field::json_static_depth() const +{ + return is_valid_json_static() ? table->json_static_depth[field_index] + : JSON_DEPTH_UNKNOWN; +} + + +/* + What a value that has just arrived makes of the running figure. The + entry only ever rises, so this is asked in the same call as the store + and before anything reads the row - which is what makes a figure read + at row N no smaller than the depth of row N. + + A value nobody counted arrives as JSON_DEPTH_UNKNOWN, which is the + largest there is, so it takes the column there and nothing brings it + back. +*/ + +void Field::raise_json_static_depth(uint depth) +{ + if (table->json_static_depth && depth > table->json_static_depth[field_index]) + table->json_static_depth[field_index]= depth; +} + + +void Field::forget_json_static_depth() +{ + if (table->json_static_depth) + table->json_static_depth[field_index]= JSON_DEPTH_UNKNOWN; +} + + +/* + Said once, of a field of a temporary table the server is building for + itself, while the one item that will ever write it is in hand. + + Three things are asked, and all three are properties of the pair rather + than of any value: + + 1. the item returns a document EVERY time it is evaluated, not just + the time somebody happens to look - which is the question this field + needs, there being no evaluation to have looked at yet; + 2. the field puts down the characters it is given, by the same list + is_character_preserving() uses for the check skip; + 3. it writes them in the character set they arrive in, so no character + is encoded again on the way down. A document rewritten into another + character set is still a document, but only where every character of + it can be written there, and that is not a question about the pair. + + The field being JSON-typed is asked too, and not because it makes the + value any more of a document: it is what decides whether anybody will + ever ask; an attestation about a column nothing reads as JSON is + never looked at. + + The formatting is granted here too, as a yes, without anything being + asked. Nothing could be asked: the item is in hand, but the question + is about values it has not made yet, and unlike being a document the + formatting is settled one value at a time. So it is granted here and + spent afterwards - each value that arrives written another way takes + it back, and a field that is never written keeps it and is never read. + + How deep the values go is settled the same way and for the same + reason, but it runs the other direction: it starts at nothing, being + the deepest of no rows, and each value that arrives raises it. Where + no answer is given at all it starts at the largest figure there is, + so that a column nobody attested to cannot be read as a shallow one. +*/ + +void Field::set_is_valid_json_static(const Item *item) +{ + if (!table->is_valid_json_static_set) + return; + DBUG_ASSERT(table->is_nice_json_static_set && table->json_static_depth); + if (item->is_valid_json_static() && is_character_preserving() && + Type_handler_json_common::is_json_type_handler(type_handler()) && + my_charset_same(charset(), item->collation.collation)) + { + bitmap_set_bit(table->is_valid_json_static_set, field_index); + bitmap_set_bit(table->is_nice_json_static_set, field_index); + table->json_static_depth[field_index]= 0; + } + else + { + bitmap_clear_bit(table->is_valid_json_static_set, field_index); + bitmap_clear_bit(table->is_nice_json_static_set, field_index); + table->json_static_depth[field_index]= JSON_DEPTH_UNKNOWN; + } +} + + +/* + Asked after every store into such a field, because the promise above is + about characters and a store is where characters go missing. + + It is asked as "how much did you keep", not "did anything go wrong", + because going wrong is not reported here: Field_longstr:: + report_if_important_data() answers 0 for a truncation unless + count_cuted_fields is raised above CHECK_FIELD_EXPRESSION, and writing + a temporary table does not raise it. A store that dropped the tail of + a document would return success and warn nobody. + + Losing the mark is for good. The value that arrived short is not a + document, and one field of one row being wrong is enough - a reader + believes the field, not the row. + + No function asks for less room than it goes on to use, so nothing gets + here having lost anything; what this catches is one of them ever + doing so. A store that comes up short is arranged for rather than + waited for, so that the losing of the mark is exercised and not merely + reasoned about. + + The formatting is asked of the item instead of the store, that being + where the answer is: a store puts characters down and knows nothing + about the order they are in, while the item has just written them and + says how. The two are asked together because they are spent together + - a store that lost characters has lost the formatting with them, which + is why the length is settled first and the formatting only after. + + The depth comes off the item for the same reason and is taken the + same way round: a value the item did not count arrives as the largest + figure there is and takes the column with it, so nothing has to + decide here what an absent answer means. +*/ + +void Field::confirm_is_valid_json_static(uint32 handed_length, bool is_nice, + uint depth, CHARSET_INFO *cs) +{ + DBUG_EXECUTE_IF("json_tmp_store_kept_short", handed_length++;); + /* + The length alone is not the question, and both the other two ask the + rest of it. A store between a wide set and the binary one keeps + every byte and calls them by the other set's name, so the length + comes back unchanged over characters that are no longer the ones + that were written: a ucs2 document arrives as bytes beginning 00 7B, + which nothing reads as a document. + + Asked here rather than left to the grant, because on the union route + the grant cannot ask it. The field is made from a type holder, whose + own set is the aggregated one and so is always the field's - the + question answers itself there and decides nothing. The branches that + do the storing are the ones with something to say, and this is where + each of them says it. + */ + if (value_length() != handed_length || + !String_copier::conversion_keeps_characters(charset(), cs)) + { + clear_is_valid_json_static(); + return; + } + confirm_json_static_value(is_nice, depth); +} + + +/* + What is left of a confirm once the answer itself has survived it: the + formatting and the depth, which are said about the values rather than + about the column and so are taken one value at a time. + + Both confirms end here, and the debug reading is written once for the + same reason the rule above it is: two copies of a check are two chances + for one of them to stop matching what it is checking. +*/ + +void Field::confirm_json_static_value(bool is_nice, uint depth) +{ + if (!is_nice) + clear_is_nice_json_static(); + raise_json_static_depth(depth); +#ifndef DBUG_OFF + { + StringBuffer kept; + const String *v= val_str(&kept); + /* + Nothing else will ever read this back. The mark appears in no + result and changes no output, so a field wrongly carrying one goes + unnoticed until something acts on it - and by then the store that + made the promise is a long way from the code that believed it. + */ + DBUG_ASSERT(json_value_reads_as_document(v)); + DBUG_ASSERT(!is_nice_json_static() || json_value_is_nice(v)); + DBUG_ASSERT(json_static_depth() >= json_value_depth(v)); + } +#endif +} + + +/* + Asked after such a field is filled from another FIELD rather than from + an item, which is how a temporary table built out of an earlier one is + filled. + + The answer that field was given was copied at build time, before a row + of either table existed, so it is not the answer the source holds now. + It is an upper bound on it: the bit is set in one place, reached only + while such a table is being built, and everything that touches it + afterwards only ever clears, so a source that has given its answer up + cannot get it back and a copy of it can only be too generous. Asking + the source again at every fill is what turns the bound into the + answer, and it is asked where both tables are alive by construction + and the source's own store of this row is already done - so nothing + here rests on the order two tables are written in. + + The source is asked the same two things the store is asked: whether it + attests to its values at all, and whether this field kept every + character of what it holds. The character sets are asked about too, + the length being counted in bytes and two fields of different + character sets being able to agree on a length while disagreeing about + every character in it. A grant is only ever made where they match, so + this decides nothing today; it is here so that the byte count does not + quietly become the wrong question if a fill site ever brings a source + the grant did not look at. + + The formatting comes from the source as well, and here it has to: no + item was evaluated to fill this field, so the only thing that knows + how the characters are arranged is the field they were copied from. + It is a bound in the same way the answer above is, and turned into the + answer the same way, by asking again at every fill. + + The depth comes from the source too, and it is the source's figure + over all its rows rather than the depth of the one value being + copied - the source having kept a running deepest and not a per-row + one. That is too large rather than wrong, and too large is the + direction this figure is allowed to be wrong in. +*/ + +void Field::confirm_is_valid_json_static_from(Field *from) +{ + uint32 handed_length; + + /* + A NULL is not read as a document and nothing attests to it as one, + so a row that puts one here leaves the column's answer where it was. + */ + if (is_null()) + return; + + handed_length= from->value_length(); + DBUG_EXECUTE_IF("json_tmp_chain_kept_short", handed_length++;); + if (from->is_null() || !from->is_valid_json_static() || + !my_charset_same(charset(), from->charset()) || + value_length() != handed_length) + { + clear_is_valid_json_static(); + return; + } + confirm_json_static_value(from->is_nice_json_static(), + from->json_static_depth()); +} + + int Field::set_default() { if (default_value) diff --git a/sql/field.h b/sql/field.h index 1ac6d4b6a7cba..862334c7003b1 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1253,6 +1253,91 @@ class Field: public Value_source bitmap_clear_bit(&table->is_valid_json_set, field_index); } + /* + Attest to every value this field will ever hold, given once while + the table was being built. Only a temporary table the server built + for itself has anywhere to keep such an answer, so everything else + says no by having no bitmap at all. + See TABLE::is_valid_json_static_set. + */ + bool is_valid_json_static() const + { + return table->is_valid_json_static_set && + bitmap_is_set(table->is_valid_json_static_set, field_index); + } + /* + Whether every value that has reached this field was also written the + loose way - see TABLE::is_nice_json_static_set. Asked through the + answer above, a formatting being nothing to say about a column whose + values are not known to be documents. + */ + bool is_nice_json_static() const + { + return is_valid_json_static() && + bitmap_is_set(table->is_nice_json_static_set, field_index); + } + /* + How deep the deepest value that has reached this field goes, or + JSON_DEPTH_UNKNOWN where no such thing is being kept - see + TABLE::json_static_depth. Asked through the answer above like the + formatting is, and out of line because that constant is declared where + a field cannot see it. + */ + uint json_static_depth() const; + void raise_json_static_depth(uint depth); + void forget_json_static_depth(); + void set_is_valid_json_static(const Item *item); + void clear_is_nice_json_static() + { + if (table->is_nice_json_static_set) + bitmap_clear_bit(table->is_nice_json_static_set, field_index); + } + void clear_is_valid_json_static() + { + if (table->is_valid_json_static_set) + bitmap_clear_bit(table->is_valid_json_static_set, field_index); + /* + Kept in step rather than left to the reader above to hide, so that + the two bitmaps say the same thing as each other and not merely the + same thing when read in the one order. + */ + clear_is_nice_json_static(); + forget_json_static_depth(); + } + void confirm_is_valid_json_static(uint32 handed_length, bool is_nice, + uint depth, CHARSET_INFO *cs); + void confirm_is_valid_json_static_from(Field *from); + void confirm_json_static_value(bool is_nice, uint depth); + + + /* + What this field can say about the bytes it is holding. The long + form of why it can be believed is at Item_field::is_valid_json(). + + Asked by everything that reads a value through a field rather than + through the item that made it - Item_field for the field it names, + and the Item_ref family for the result field it reads a copy out + of. + + A row holding no value at all is not attested, and that is asked + here rather than left to the reader. The grant is about the column: + it is given once, where the table is built, and what it says is that + every value the column comes to hold arrived from something that + attests to the values it makes. That is as true of a row that came + out SQL NULL as of any other, there being no value in it to be wrong + about - so the grant goes on standing over such a row while there + are no characters under it to have been attested. A caller that + reads it there puts the answer down somewhere the NULL itself cannot + go, the column taking whatever it takes in place of one, and the + answer is then about bytes nothing ever saw. + */ + bool attests_is_valid_json() const + { return !is_null() && is_valid_json_static(); } + bool attests_is_nice_json() const + { return !is_null() && is_nice_json_static(); } + uint attested_json_depth() const + { return json_static_depth(); } + virtual my_time_t get_timestamp(const uchar *pos, ulong *sec_part) const { DBUG_ASSERT(0); return 0; } my_time_t get_timestamp(ulong *sec_part) const @@ -5950,7 +6035,19 @@ class Copy_field :public Sql_alloc { of entry go in the same array, so start them absent rather than leaving them to read as whatever the memory held. */ - Copy_field() : from_field(NULL), to_field(NULL) {} + /* + Whether the destination attests to every value it holds, worked out + where the entry is made rather than once a row. The bit behind it is + SET only while a temporary table is being built, which is before any + entry here is made; afterwards it is only ever cleared, and the + confirm the copy loop makes reads it again. So a yes recorded here + is an upper bound and costs at most a question that answers no, + while asking per row costs two loads and a test for every copied + column of every grouped query in the instance. + */ + bool to_needs_confirm; + Copy_field() + : from_field(NULL), to_field(NULL), to_needs_confirm(false) {} ~Copy_field() = default; void set(Field *to,Field *from,bool save); // Field to field void set(uchar *to,Field *from); // Field to string diff --git a/sql/field_conv.cc b/sql/field_conv.cc index b4e2c03013cd8..5d1c692245666 100644 --- a/sql/field_conv.cc +++ b/sql/field_conv.cc @@ -701,6 +701,7 @@ void Copy_field::set(Field *to,Field *from,bool save) } from_field=from; to_field=to; + to_needs_confirm= to->is_valid_json_static(); from_ptr=from->ptr; from_length=from->pack_length_in_rec(); to_ptr= to->ptr; @@ -909,9 +910,21 @@ static int field_conv_incompatible(Field *to, Field *from) int field_conv(Field *to,Field *from) { - return to->memcpy_field_possible(from) ? - field_conv_memcpy(to, from) : - field_conv_incompatible(to, from); + int rc= to->memcpy_field_possible(from) ? + field_conv_memcpy(to, from) : + field_conv_incompatible(to, from); + /* + A field of a temporary table the server built for itself attests to + every value it holds, and a value put here came from another FIELD + rather than from an item that could be asked. The source is asked + instead, and it is asked HERE so that the answer does not rest on a + list of callers being kept complete: the two routes that go around + this function ask for themselves, and everything that goes through it + is covered wherever it is written. + */ + if (unlikely(to->is_valid_json_static())) + to->confirm_is_valid_json_static_from(from); + return rc; } diff --git a/sql/group_by_handler.cc b/sql/group_by_handler.cc index 7b998494af9ae..2375e3da9538d 100644 --- a/sql/group_by_handler.cc +++ b/sql/group_by_handler.cc @@ -46,6 +46,15 @@ int Pushdown_query::execute(JOIN *join) TABLE *table= handler->table; DBUG_ENTER("Pushdown_query::execute"); + /* + Nothing attests to what an engine puts in a table it fills, so the + bitmaps are dropped where the table is given to the engine - see + TABLE::set_filled_by_engine(). Asserted here rather than there + because what has to hold is that EVERY way in has dropped them, and + the ways in all come through here. + */ + DBUG_ASSERT(!table->is_valid_json_static_set); + if ((err= handler->init_scan())) goto error; diff --git a/sql/item.cc b/sql/item.cc index 126c8719cce4e..7369c9bccb9e2 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -43,6 +43,7 @@ // RESOLVED_AGAINST_ALIAS, ... #include "sql_expression_cache.h" #include "sql_lex.h" // empty_clex_str +#include "item_jsonfunc.h" // json_value_reads_as_document const String my_null_string("NULL", 4, default_charset_info); const String my_default_string("DEFAULT", 7, default_charset_info); @@ -2283,13 +2284,20 @@ class Item_aggregate_ref : public Item_ref Ref_Type ref_type() override final { return AGGREGATE_REF; } /* - What this stands in front of is an aggregate, which keeps no result - field of its own, so reading through it reaches the aggregate's own - val_str() and the value arrives unaltered. Whatever the aggregate - can say about it therefore still holds here. + The standing question, and only that one. split_sum_func2() puts + this in front of whatever it moves out of an expression - "or copy + it (in case of fields)", its own comment says - so an aggregate is + not the only thing under here and an aggregate's habits cannot be + assumed of it. The three answers about a VALUE are therefore left + to Item_ref, which asks the item what its result side passes + rather than what it made. + + This one is not about a value. It is asked while a temporary table + is being built, before a row exists, and by then the item underneath + has had its result field pointed at the very column being asked + about - so there is no result side to put the question to yet, and + the producer is the only thing there is to ask. */ - bool is_valid_json() const override { return (*ref)->is_valid_json(); } - bool is_nice_json() const override { return (*ref)->is_nice_json(); } bool is_valid_json_static() const override { return (*ref)->is_valid_json_static(); } protected: @@ -5358,10 +5366,45 @@ int Item_copy_string::save_in_field(Field *field, bool no_conversions) void Item_copy_string::copy() { + bool kept= true; String *res=item->val_str(&str_value); if (res && res != &str_value) - str_value.copy(*res); + kept= !str_value.copy(*res); null_value=item->null_value; + /* + The item was evaluated just above and nothing has evaluated it + since, so this is the one moment its answers are about the bytes + that are being kept. String::copy() takes the character set along + with them, so the characters those answers are about are the + characters kept here - where it kept them at all. A copy that could + not get the room leaves whatever was here before, which the value + side has always returned and which nothing here can attest to. + + A value that carries an answer is one that gets spliced into a + document rather than quoted into one, and which of the two happens + is decided by the type rather than by the answer. This item's type + is the one it was made over, so the two agree wherever that item is + the producer; where it is a character set conversion around the + producer they do not, the conversion carrying the answer through + while the type stops at it. Asking the same question the splice + will ask is what keeps this item from attesting to a value that is + going to be quoted. + + The splice asks is_json_type(), whose first question is this one and + whose second is whether the item is a conversion to look through. + A copy is not one and real_item() of a copy is the copy, so the + second question can only ever be answered no here - and answering it + is a dynamic_cast, on a path walked once a row for every copied + field in every grouped query, JSON or not. So the first question is + asked directly, once, where the type is settled, and the two are the + same question. + */ + DBUG_EXECUTE_IF("json_copy_not_kept", kept= false;); + if (null_value || !kept || !m_is_json) + m_marks.clear(); + else + m_marks.set(&str_value, item->is_valid_json(), item->is_nice_json(), + item->last_depth()); #ifdef DBUG_ASSERT_EXISTS copied_in= 1; #endif @@ -7044,6 +7087,80 @@ void Item_field::make_send_field(THD *thd, Send_field *tmp_field) } +#ifdef DBUG_ASSERT_EXISTS +/* + What a field is attesting to, read back off the bytes it is holding. + A field that answers nothing is nothing to disagree with, and a row + with nothing in it is a field that answers nothing. + + Three bodies rather than one that settles all three at a stroke, + which is what a single walk over the value could do. The three are + asked one at a time, so one body would be walked three times rather + than once - and answering the formatting means writing the whole value + out again and comparing it, which the other two would then be paying + for as well. Reading each of them the cheapest way it can be read is + what keeps a debug build's checking proportional to what is checked. +*/ +static bool field_reads_back_as_document(Field *f) +{ + if (!f->attests_is_valid_json()) + return true; + { + StringBuffer kept; + return json_value_reads_as_document(f->val_str(&kept)); + } +} + + +static bool field_reads_back_as_nice(Field *f) +{ + if (!f->attests_is_nice_json()) + return true; + { + StringBuffer kept; + return json_value_is_nice(f->val_str(&kept)); + } +} + + +static bool field_reads_back_no_deeper_than_claimed(Field *f) +{ + if (!f->attests_is_valid_json()) + return true; + { + StringBuffer kept; + return json_value_depth(f->val_str(&kept)) <= f->attested_json_depth(); + } +} + + +/* + Asked of each of the two fields an item has: the one it names, and - + wherever a temporary table has been built out of that one - the copy + in it, which is what the _result() answers are about. A fill site + nobody taught to ask its source announces itself in whichever of the + two it got wrong. +*/ +bool Item_field::reads_back_as_document() const +{ return field_reads_back_as_document(field); } + +bool Item_field::reads_back_as_nice() const +{ return field_reads_back_as_nice(field); } + +bool Item_field::reads_back_no_deeper_than_claimed() const +{ return field_reads_back_no_deeper_than_claimed(field); } + +bool Item_field::reads_back_as_document_result() const +{ return field_reads_back_as_document(result_field); } + +bool Item_field::reads_back_as_nice_result() const +{ return field_reads_back_as_nice(result_field); } + +bool Item_field::reads_back_no_deeper_than_claimed_result() const +{ return field_reads_back_no_deeper_than_claimed(result_field); } +#endif + + /** Save a field value in another field @@ -7087,6 +7204,12 @@ static int save_field_in_field(Field *from, bool *null_value, if (to == from) DBUG_RETURN(0); + /* + field_conv() confirms the destination against what the source + attests to - see there. A value that arrives off an ITEM goes through + Item::save_str_in_field instead, which asks the store the same + question this asks the source. + */ res= field_conv(to, from); DBUG_RETURN(res); } @@ -7128,6 +7251,16 @@ void Item_field::save_org_in_field(Field *to, DBUG_VOID_RETURN; } (*fast_field_copier_func)(to, field); + /* + One of the two routes that go around field_conv(), reaching what it + dispatches to without passing through it. It does write fields of + the server's own temporary tables, so the question belongs here; no + field carrying an answer is known to arrive, which is a fact about + the queries anybody has written rather than about the route, so it + is asked and not assumed. + */ + if (unlikely(to->is_valid_json_static())) + to->confirm_is_valid_json_static_from(field); } else save_field_in_field(field, &null_value, to, TRUE); @@ -7203,6 +7336,24 @@ int Item::save_str_in_field(Field *field, bool no_conversions) field->set_notnull(); int error= field->store(result->ptr(),result->length(),cs); + /* + A value that reaches a field carrying the standing answer off a + string ITEM comes through here, that being the one way one is put + into a field. A value that comes off another FIELD does not, and + is asked about at the three places save_field_in_field() names. + The answer is about characters, so it is kept only while the store + keeps them. + + How they are formatted is asked of the item rather than of the store, + and it is asked HERE rather than said at build time because it is + about the value just written and not about the item that wrote it. + How deep the value goes is asked here for the same reason and taken + the same way, an item that counted nothing answering with the + largest figure there is. + */ + if (unlikely(field->is_valid_json_static())) + field->confirm_is_valid_json_static(result->length(), is_nice_json(), + last_depth(), cs); str_value.set_buffer_if_not_allocated(0, 0, cs); return error; } diff --git a/sql/item.h b/sql/item.h index 2213f8eb9171a..adee5db9194aa 100644 --- a/sql/item.h +++ b/sql/item.h @@ -27,6 +27,7 @@ #include "sql_const.h" /* RAND_TABLE_BIT, MAX_FIELD_NAME */ #include "field.h" /* Derivation */ #include "sql_type.h" +#include "sql_type_json.h" /* Type_handler_json_common */ #include "sql_time.h" #include "sql_schema.h" #include "mem_root_array.h" @@ -861,6 +862,19 @@ static inline item_with_t operator~(const item_with_t a) } +/* + What a function answers when it cannot say how many structures deep + the value it returned goes. It is a number no document reaches, + so a caller weighing it against the limit turns the value down + whatever it is being put inside, which is the answer that costs a + reading and nothing else. + + A caller with a bound of its own takes the smaller of the two before + adding anything to it, so this never enters a sum. +*/ +#define JSON_DEPTH_UNKNOWN UINT_MAX + + /* What a JSON function says about the value it has just returned: is_valid, that the value is a document, and is_nice, that it is @@ -885,8 +899,17 @@ class Json_result_marks { bool m_valid; bool m_nice; + /* + How deep the value goes, where the function that wrote it worked + that out on the way; JSON_DEPTH_UNKNOWN where it did not. Kept + with the other two because it is answered under the same rules and + over the same value, and because a class picking up one picks up + all three. + */ + uint m_depth; public: - Json_result_marks(): m_valid(false), m_nice(false) {} + Json_result_marks() + : m_valid(false), m_nice(false), m_depth(JSON_DEPTH_UNKNOWN) {} /* A copy is a new item that has not been evaluated, so it has produced no value and there is nothing for it to say. What the item it was @@ -894,17 +917,21 @@ class Json_result_marks to write over. */ Json_result_marks(const Json_result_marks &) - : m_valid(false), m_nice(false) {} + : m_valid(false), m_nice(false), m_depth(JSON_DEPTH_UNKNOWN) {} bool valid() const { return m_valid; } bool nice() const { return m_nice; } - void clear() { m_valid= m_nice= false; } - /* - 'str' is the value 'valid' and 'nice' are being set for. A debug - build reads it back and stops if it is not what they say it is. - Nothing else ever would: the marks appear in no result, so a claim - that was not true would go unnoticed until something acted on it. - */ - void set(const String *str, bool valid, bool nice); + uint depth() const { return m_depth; } + void clear() + { m_valid= m_nice= false; m_depth= JSON_DEPTH_UNKNOWN; } + /* + 'str' is the value 'valid', 'nice' and 'depth' are being set for. A + debug build reads it back and stops if it is not what they say it + is. Nothing else ever would: the marks appear in no result, so a + claim that was not true would go unnoticed until something acted on + it. + */ + void set(const String *str, bool valid, bool nice, + uint depth= JSON_DEPTH_UNKNOWN); }; @@ -1784,6 +1811,27 @@ class Item :public Value_source, */ virtual bool is_nice_json() const { return false; } + /* + How many structures that same value nests: 0 for a scalar, 1 for an + array or object holding nothing but scalars, and so on. Asked and + answered under the same rules as the two above, and read only + together with is_valid - a value that is not known to be a document + has no depth worth speaking of. + + An item works this out only where it was going to walk the value + anyway, so most of them say nothing and answer JSON_DEPTH_UNKNOWN. + That is the safe answer here as false is there: a caller that is + told nothing reads the value, which is what every caller did before + there was anything to ask. + + A number too small is the one answer that must never be given. The + caller adds it to the depth the value is being spliced at and holds + the total against the limit, so an item that under-counts lets a + document past that limit be composed and passed on as one inside + it. + */ + virtual uint last_depth() const { return JSON_DEPTH_UNKNOWN; } + /* Whether EVERY evaluation of this item returns a document or nothing at all. A property of the class, so it can be asked before @@ -1792,6 +1840,30 @@ class Item :public Value_source, */ virtual bool is_valid_json_static() const { return false; } + /* + The same three questions, asked of the value the RESULT side of this + item returns - str_result(), which is how an Item_ref reads it. + + For nearly every item the two sides are one function: + Item::str_result() IS val_str(), so the value asked about is the + same value and the answers are the same answers, which is what the + defaults here say. An item that keeps a result field of its own + returns a copy sitting in a record buffer instead, and what was + said about the value it made is not said about the copy. Those + items - the five that override str_result() - attest to whatever + their own str_result() reads, by putting the question to it. + + Which of the two sides a reference forwards is not a judgement it + makes. Item_ref::val_str() calls str_result(), so it asks these; + Item_direct_ref::val_str() calls val_str(), so it asks the three + above. Nothing is dropped for safety and nothing is guessed: the + marks describe the bytes because they are asked of whatever handed + the bytes over. + */ + virtual bool is_valid_json_result() const { return is_valid_json(); } + virtual bool is_nice_json_result() const { return is_nice_json(); } + virtual uint last_depth_result() const { return last_depth(); } + /* Return decimal representation of item with fixed point. @@ -4029,6 +4101,111 @@ class Item_field :public Item_ident, return field->table->pos_in_table_list->outer_join; } bool check_index_dependence(void *arg) override; + /* + A column says nothing about itself: a row can hold bytes no check ever + saw, and nothing in it records where they came from. The one + exception is a field of a temporary table the server built for + itself, which was told at build time what would be written into it - + see TABLE::is_valid_json_static_set. Nobody can address such a table, + so nothing can put anything else in it. + + One bit answers both questions, and that is what it was given to + mean: it was said of every row the column will ever hold, so what is + true of this evaluation is true of all of them. Answering the + standing question is what lets a column of one such table be written + into a column of the next - a table built out of an earlier one is + built before either has a row, so what the earlier column says THEN + is an upper bound on what it says by the time the row arrives, and + the later column asks it again at every fill. See + Field::confirm_is_valid_json_static_from(). + */ + bool is_valid_json() const override + { + DBUG_ASSERT(reads_back_as_document()); + return field->attests_is_valid_json(); + } + bool is_valid_json_static() const override + { return field->is_valid_json_static(); } + /* + Formatting, unlike the two above, is nothing the column was promised. + It was granted along with them and spent by whatever has been + written since, so what it answers is about the rows that are there - + which is what a reader with a row in front of it is asking about. + + There is no standing companion to this one and there could not be: + the standing question is asked while a table is being built out of an + earlier one, and what the earlier column will come to be written like + is not a thing anybody knows then. A column of the later table is + granted the formatting like any other and gets it from its source at + each fill - see Field::confirm_is_valid_json_static_from(). + */ + bool is_nice_json() const override + { + DBUG_ASSERT(reads_back_as_nice()); + return field->attests_is_nice_json(); + } + /* + And how deep it goes, which is what a caller splicing this value + into one of its own adds its own levels to. It is the deepest of + the rows written so far rather than the depth of this one, which is + more than this value needs and never less - and more is the + direction a depth is allowed to be wrong in, costing a reading that + could have been skipped rather than admitting a document nothing + can read back. + */ + uint last_depth() const override + { + DBUG_ASSERT(reads_back_no_deeper_than_claimed()); + return field->attested_json_depth(); + } + /* + str_result() reads result_field rather than field, so these read + result_field too. Which of the two an Item_field is asked for is + decided by whoever reads it: everything that goes through a value + side reaches the first, and an Item_ref reaches the second, the two + being the same object until a temporary table has been built out of + this one and the copy put in it - which is exactly the case where + they can disagree, a store into the copy being able to shorten or + reformat what the original holds. + */ + bool is_valid_json_result() const override + { + DBUG_ASSERT(reads_back_as_document_result()); + return result_field->attests_is_valid_json(); + } + bool is_nice_json_result() const override + { + DBUG_ASSERT(reads_back_as_nice_result()); + return result_field->attests_is_nice_json(); + } + uint last_depth_result() const override + { + DBUG_ASSERT(reads_back_no_deeper_than_claimed_result()); + return result_field->attested_json_depth(); + } +#ifdef DBUG_ASSERT_EXISTS + /* + Detectors rather than checks, and the only ones that do not care + which way the value arrived: whatever filled this field, if the + column attests to its values then the bytes in the row have to read + as a document, and if it attests to their formatting they have to be + formatted that way. A fill site nobody taught to ask its source + announces itself here, at the first read, rather than wherever + something later acts on the answer. + */ + bool reads_back_as_document() const; + bool reads_back_as_nice() const; + bool reads_back_no_deeper_than_claimed() const; + /* + And the same three over the other field, for the three answers that + are about it. A twin rather than a parameter because the assertions + read as the accessors do, each naming the field its own answer came + from. + */ + bool reads_back_as_document_result() const; + bool reads_back_as_nice_result() const; + bool reads_back_no_deeper_than_claimed_result() const; +#endif void set_refers_to_temp_table(); friend class Item_default_value; friend class Item_insert_value; @@ -6062,6 +6239,41 @@ class Item_ref :public Item_ident my_decimal *val_decimal_result(my_decimal *) override; bool val_bool_result() override; bool is_null_result() override; + /* + val_str() here reads the referenced item through str_result(), so + what is asked of that item is what IT says about the value its + result side passes - which for most items is the value it made, + and for the few that keep a copy is the copy. The item is the one + that knows which; nothing is decided here. + */ + bool is_valid_json() const override + { return (*ref)->is_valid_json_result(); } + bool is_nice_json() const override + { return (*ref)->is_nice_json_result(); } + uint last_depth() const override + { return (*ref)->last_depth_result(); } + /* + And this item's own result side, which str_result() below reads out + of a result field where there is one and through val_str() where + there is not. Whether there is one is a fact about this instance + rather than about the class, which is why the condition is written + out here rather than answered once and for all somewhere. + */ + bool is_valid_json_result() const override + { + return result_field ? result_field->attests_is_valid_json() + : (*ref)->is_valid_json_result(); + } + bool is_nice_json_result() const override + { + return result_field ? result_field->attests_is_nice_json() + : (*ref)->is_nice_json_result(); + } + uint last_depth_result() const override + { + return result_field ? result_field->attested_json_depth() + : (*ref)->last_depth_result(); + } bool send(Protocol *prot, st_value *buffer) override; void make_send_field(THD *thd, Send_field *field) override; bool fix_fields(THD *, Item **) override; @@ -6294,6 +6506,7 @@ class Item_direct_ref :public Item_ref */ bool is_valid_json() const override { return (*ref)->is_valid_json(); } bool is_nice_json() const override { return (*ref)->is_nice_json(); } + uint last_depth() const override { return (*ref)->last_depth(); } bool is_valid_json_static() const override { return (*ref)->is_valid_json_static(); } @@ -6584,6 +6797,22 @@ class Item_direct_view_ref :public Item_direct_ref { return !null_row_ref() && Item_direct_ref::is_valid_json(); } bool is_nice_json() const override { return !null_row_ref() && Item_direct_ref::is_nice_json(); } + uint last_depth() const override + { + return null_row_ref() ? JSON_DEPTH_UNKNOWN + : Item_direct_ref::last_depth(); + } + /* + str_result() below passes on whatever the referenced item's result + side gave, and does not look at the row the way the val_XXX above + it do, so neither does this. + */ + bool is_valid_json_result() const override + { return (*ref)->is_valid_json_result(); } + bool is_nice_json_result() const override + { return (*ref)->is_nice_json_result(); } + uint last_depth_result() const override + { return (*ref)->last_depth_result(); } void save_val(Field *to) override { @@ -6920,11 +7149,21 @@ class Item_copy :public Item, Type_std_attributes::set(item); name= item->name; set_handler(item->type_handler()); + m_is_json= Type_handler_json_common::is_json_type_handler(type_handler()); #ifdef DBUG_ASSERT_EXISTS copied_in= 0; #endif } + /* + Whether a value kept here is one a splice puts into a document rather + than quotes into one. The type settles that and the type is settled + here, so it is worked out once: the row path would otherwise ask two + virtuals per copied field per row to find out something that was true + before the query began. + */ + bool m_is_json; + /* Read by a DBUG_ASSERT and by nothing else, so it is compiled wherever a DBUG_ASSERT is rather than wherever a debug build is. Those are not @@ -6994,6 +7233,20 @@ class Item_copy :public Item, */ class Item_copy_string : public Item_copy { + /* + What the item this was made over said about the value kept here, + taken at the moment it was kept. + + The three questions are about the last value an item produced, and + this item's values are produced elsewhere and at another time: what + copy() puts away stands until the next copy(), while the item it + came from goes on evaluating. Asking that item when somebody asks + this one would pair an answer about a value it has since produced + with the bytes sitting here - which is why the answers are taken + once, beside the bytes they are about, exactly as + Item_func_conv_charset does over the value it freezes. + */ + Json_result_marks m_marks; public: Item_copy_string(THD *thd, Item *item_arg): Item_copy(thd, item_arg) {} @@ -7006,6 +7259,25 @@ class Item_copy_string : public Item_copy DBUG_ASSERT(copied_in); return get_date_from_string(thd, ltime, fuzzydate); } + /* + Asked next to val_str(), which returns the same bytes these are + about and which asserts the same thing about having been filled. + */ + bool is_valid_json() const override + { + DBUG_ASSERT(copied_in); + return m_marks.valid(); + } + bool is_nice_json() const override + { + DBUG_ASSERT(copied_in); + return m_marks.nice(); + } + uint last_depth() const override + { + DBUG_ASSERT(copied_in); + return m_marks.depth(); + } void copy() override; int save_in_field(Field *field, bool no_conversions) override; protected: @@ -7300,7 +7572,14 @@ class Item_default_value : public Item_field longlong val_time_packed(THD *thd) override { return Item::val_time_packed(thd); } - /* Result variants */ + /* + Result variants. Each of these puts the default in place and then + does what Item_field does, and the marks that go with them need no + variant of their own: what they add is the putting in place, and by + the time anybody asks what a value was, the value has been made. + An accessor that called calculate() again would be making a second + one. + */ double val_result() override; longlong val_int_result() override; String *str_result(String* tmp) override; @@ -8325,11 +8604,14 @@ class Item_type_holder: public Item, public Type_handler_hybrid_field_type { protected: const TYPELIB *enum_set_typelib; + bool m_is_valid_json_static; public: Item_type_holder(THD *thd, Item *item, const Type_handler *handler, - const Type_all_attributes *attr, bool maybe_null_arg) + const Type_all_attributes *attr, bool maybe_null_arg, + bool is_valid_json_static_arg) :Item(thd), Type_handler_hybrid_field_type(handler), - enum_set_typelib(attr->get_typelib()) + enum_set_typelib(attr->get_typelib()), + m_is_valid_json_static(is_valid_json_static_arg) { name= item->name; Type_std_attributes::set(*attr); @@ -8349,6 +8631,17 @@ class Item_type_holder: public Item, public Type_handler_hybrid_field_type } Type type() const override { return TYPE_HOLDER; } + /* + A holder stands in for a column of a UNION, which has one producer + per branch and a type they agree on. What is answered here is what + every one of those producers answers, asked of them all while the + types are being agreed - the branches being the one place they are + all in reach. Where they were not all reached, the caller hands + down a no rather than letting the ones it did reach speak for the + rest. + */ + bool is_valid_json_static() const override + { return m_is_valid_json_static; } const TYPELIB *get_typelib() const override { return enum_set_typelib; } /* When handling a query like this: diff --git a/sql/item_func.h b/sql/item_func.h index c668bbfe76826..73330d0820154 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -3602,6 +3602,18 @@ class Item_func_set_user_var :public Item_func_user_var String *str_result(String *str) override; my_decimal *val_decimal_result(my_decimal *) override; bool is_null_result() override; + /* + str_result() above is the one of the five that does not merely read + somewhere else: it assigns the variable and then returns what is + in it. So these cannot be written the way the other four are, by + putting the question to whatever that reads - putting a question to + it would perform the assignment. They say nothing instead, which + is what there is to say: nothing keeps marks for a user variable, + so there would be nothing to pass on even if asking were free. + */ + bool is_valid_json_result() const override { return false; } + bool is_nice_json_result() const override { return false; } + uint last_depth_result() const override { return JSON_DEPTH_UNKNOWN; } bool update_hash(void *ptr, size_t length, const Type_handler *th, CHARSET_INFO *cs); bool send(Protocol *protocol, st_value *buffer) override; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index bf96f9d1fe2a3..53013cfb3ca39 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -633,10 +633,19 @@ static int json_walk_nice_value(json_engine_t *je, String *to, int &max_level); +/* + 'deepest', where a caller asks for it, comes back holding how many + structures the deepest part of what was written nests inside: 0 for a + scalar, 1 for an array of scalars, and so on. The count is free here - + the walk keeps it anyway, to know how far to indent - and it is the + only place a whole document is measured without being walked twice. +*/ static int json_nice(json_engine_t *je, String *nice_js, - Item_func_json_format::formats mode, int tab_size=4) + Item_func_json_format::formats mode, + uint *deepest= NULL, int tab_size=4) { int depth= 0; + int reached= 0; static const char *comma= json_loose_comma, *colon= json_loose_colon; uint comma_len, colon_len; int first_value= 1; @@ -669,8 +678,6 @@ static int json_nice(json_engine_t *je, String *nice_js, value ended. A document is a value with nothing after it, so what is left is read through to find out whether anything is. */ - int reached= 0; - if (json_read_value(je) || json_walk_nice_value(je, nice_js, reached)) { /* @@ -779,6 +786,8 @@ static int json_nice(json_engine_t *je, String *nice_js, first_value= 1; value_size= (je->value_type == JSON_VALUE_OBJECT) ? -1: 0; depth++; + if (depth > reached) + reached= depth; } break; @@ -815,6 +824,13 @@ static int json_nice(json_engine_t *je, String *nice_js, DBUG_SET("-d,simulate_realloc_out_of_memory");); done: + /* + Said only where the walk finished, a count off a walk that stopped + early being about the part of the document it got through rather + than about the document. + */ + if (deepest && !(je->s.error || *je->killed_ptr)) + *deepest= (uint) reached; return je->s.error || *je->killed_ptr; error: @@ -998,21 +1014,13 @@ static int json_walk_nice_value(json_engine_t *je, String *to, } -/* - The two below are read by DBUG_ASSERT and by nothing else, so they are - here wherever a DBUG_ASSERT is compiled rather than wherever a debug - build is. Those are not the same set: DBUG_ASSERT_AS_PRINTF turns the - assertion into a printed complaint and leaves its expression standing, - while setting DBUG_OFF, so a guard written the other way round leaves - those expressions with nothing to call and that build stops compiling. -*/ #ifdef DBUG_ASSERT_EXISTS /* Reads a value the way whoever receives it will read it: in the character set the value says it is written in, from one end of it to the other. */ -static bool json_value_reads_as_document(const String *str) +bool json_value_reads_as_document(const String *str) { /* Which is the question json_valid() is, so it is asked rather than @@ -1064,7 +1072,7 @@ class Json_kill_unsimulated changed anything. A value already written that way comes back byte for byte; one written any other way does not. */ -static bool json_value_is_nice(const String *str) +bool json_value_is_nice(const String *str) { Json_kill_unsimulated unkilled; json_engine_t je; @@ -1077,6 +1085,30 @@ static bool json_value_is_nice(const String *str) return !stringcmp(&nice, str); } + + +/* + How deep the value actually goes, read off the value itself. A + claimed depth is only ever compared against this one - a claim that + is larger than the truth is allowed, that being what an item saying + nothing amounts to, and one that is smaller is the failure this + exists to catch. +*/ +uint json_value_depth(const String *str) +{ + json_engine_t je; + uint deepest= 0; + + json_scan_start_unbilled(&je, str->charset(), (const uchar *) str->ptr(), + (const uchar *) str->ptr() + str->length()); + while (json_scan_next(&je) == 0) + { + if ((uint) je.stack_p > deepest) + deepest= (uint) je.stack_p; + } + + return deepest; +} #endif @@ -1140,7 +1172,8 @@ class Json_source_watch something starts acting on it - and by then the function that made the promise is a long way from the code that believed it. */ -void Json_result_marks::set(const String *str, bool valid, bool nice) +void Json_result_marks::set(const String *str, bool valid, bool nice, + uint depth) { /* The loose form is a formatting of a document, so there is no such @@ -1152,8 +1185,15 @@ void Json_result_marks::set(const String *str, bool valid, bool nice) DBUG_ASSERT(!nice || valid); DBUG_ASSERT(!valid || json_value_reads_as_document(str)); DBUG_ASSERT(!nice || json_value_is_nice(str)); + /* + A depth is only about a value that reads as a document, and is only + ever wrong in one direction - see json_value_depth(). + */ + DBUG_ASSERT(depth == JSON_DEPTH_UNKNOWN || !valid || + json_value_depth(str) <= depth); m_valid= valid; m_nice= nice; + m_depth= depth; } @@ -1535,6 +1575,10 @@ bool Json_path_extractor::extract(String *str, Item *item_js, Item *item_jp, int array_counters[JSON_DEPTH_LIMIT]= {0}; Json_source_watch watch; + /* Taken here rather than where they are used - see m_js_nice. */ + m_js_nice= item_js->is_nice_json(); + m_js_depth= item_js->last_depth(); + watch.take(js); if (!parsed) { @@ -2028,6 +2072,8 @@ String *Item_func_json_extract::read_json(String *str, int possible_multiple_values; /* How far down the deepest of the values written out goes. */ int deepest_value= 0; + /* The same, where the answer is read back instead of counted. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; int array_size_counter[JSON_DEPTH_LIMIT]; uint has_negative_path= 0; THD *thd; @@ -2246,14 +2292,15 @@ String *Item_func_json_extract::read_json(String *str, (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, &tmp_js, Item_func_json_format::LOOSE)) + if (json_nice(&je, &tmp_js, Item_func_json_format::LOOSE, &read_back_depth)) goto error; /* Both marks come from the reading back, for the reason given where - Item_func_json_insert::val_str() marks the same answer. + Item_func_json_insert::val_str() marks the same answer, and so + does the depth. */ - m_marks.set(&tmp_js, true, true); + m_marks.set(&tmp_js, true, true, read_back_depth); return &tmp_js; } @@ -2278,8 +2325,13 @@ String *Item_func_json_extract::read_json(String *str, scratch the argument may have been read into - so the answer was being written over the document it was made from, and only got away with it because the document had been finished with. + + How deep it goes was counted as it was written: each value put in + was walked, and the deepest of them is the deepest the answer goes, + plus the bracket written round them where there is one. */ - m_marks.set(str, true, true); + m_marks.set(str, true, true, + (uint) deepest_value + (possible_multiple_values ? 1 : 0)); return str; return_ok: @@ -2966,11 +3018,12 @@ bool is_json_type(const Item *item) the loose form is a form OF a document, so nothing clears the second without clearing the first. - A caller that reads the whole of its own result back afterwards learns - nothing from these and passes NULL instead - the reading back is a - stronger answer than anything collected on the way. That is every - function which EDITS a document today; when the reading back goes, - each of them will want a set of these of its own. + Every caller keeps a set, so every value put in is accounted for + whether or not the caller ends up believing what they say. A caller + that reads its whole answer back afterwards has a stronger answer than + anything collected on the way and takes that instead; it still hands + one of these over, there being nowhere else for a value's own answer + to go. */ struct Json_splice_marks { @@ -2993,9 +3046,23 @@ struct Json_splice_marks #ifdef DBUG_ASSERT_EXISTS bool is_deep; #endif + /* + The deepest any value put in reaches, counted from the outside of + the document being composed rather than from the value itself. A + caller starts it at the depth of whatever it is writing the values + inside, so that it holds for a document with no values in it too, + and it is then the depth of the whole composed answer - every other + part of that answer being punctuation the caller wrote itself. + + Wrong only ever upwards, like the answers it is made of: a value + taken on trust contributes the bound the trusting was done against + rather than a measurement, and a bound is not smaller than the + truth. + */ + uint deepest; - Json_splice_marks() - : is_valid(true), is_nice(true) + Json_splice_marks(uint depth) + : is_valid(true), is_nice(true), deepest(depth) { #ifdef DBUG_ASSERT_EXISTS is_deep= false; @@ -3048,12 +3115,17 @@ struct Json_splice_marks left undone; is_nice false is what the writing out further down is for. 'need_nice' is the caller saying that what it composes is what it returns, so a value that is not is_nice has to be made so. + + 'value_depth' is how deep the item says the value goes, or + JSON_DEPTH_UNKNOWN where it does not say. It is only ever read + alongside is_valid. */ static int append_json_typed_value(String *str, const String *sv, uint depth, const char *fname, int n_param, - Json_splice_marks *marks, + Json_splice_marks &marks, bool caller_reads_back, bool is_valid, - bool is_nice, bool need_nice) + bool is_nice, bool need_nice, + uint value_depth) { StringBuffer cnv; const char *ptr= sv->ptr(); @@ -3132,23 +3204,43 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, And it must not make the result too deep to read back. The value's own nesting is what the parse below measures, and the shortcut is not - doing that - but it does not have to, because a value can only nest - as deeply as it is long: every level takes a character to open and - one to close, so a value of n characters reaches at most n/2 levels, - and there are at most length/mbminlen characters in it. Where that - much nesting still leaves the result inside the limit, the parse - could not have found a breach to report and skipping it takes nothing - away. The bound is exact for the values that matter here - a run of - nothing but brackets is all opening and closing - and gives up on - long ones, which is the safe direction: giving up costs a reading and - nothing else. + doing that - so it takes the smallest thing it has that the nesting + cannot be more than. + + There are two such things and either alone would do. The item may + have worked the depth out while it was writing the value, in which + case it says so. And a value can only nest as deeply as it is long, + whatever it holds: every level takes a character to open and one to + close, so a value of n characters reaches at most n/2 levels, and + there are at most length/mbminlen characters in it. That second one + is exact for the values where being wrong would matter most - a run + of nothing but brackets is all opening and closing - and hopeless for + a long shallow document, which is what the first one is for. + + Where the smaller of them still leaves the result inside the limit, + the parse could not have found a breach to report and skipping it + takes nothing away. Where it does not, the value is read as before: + giving up costs a reading and nothing else. */ - if (is_valid && (is_nice || !need_nice) && in_result_charset && - depth + length / (2 * str->charset()->mbminlen) < JSON_DEPTH_LIMIT) + if (is_valid && (is_nice || !need_nice) && in_result_charset) { - if (marks && !is_nice) - marks->is_nice= false; - return append_simple(str, ptr, length); + /* The most levels the value can turn out to have - see above. */ + uint bound= MY_MIN(value_depth, + (uint) (length / (2 * str->charset()->mbminlen))); + + if (depth + bound < JSON_DEPTH_LIMIT) + { + if (!is_nice) + marks.is_nice= false; + /* + What the value was let in on, rather than what it turned out to + be - nothing measured it. A bound is not smaller than the + truth, which is all a caller composing from these needs of it. + */ + if (depth + bound > marks.deepest) + marks.deepest= depth + bound; + return append_simple(str, ptr, length); + } } /* @@ -3254,8 +3346,8 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, out - the item was asked, and one that says nothing is taken at its word. */ - if (!reformat && marks && !is_nice) - marks->is_nice= false; + if (!reformat && !is_nice) + marks.is_nice= false; /* Too deep only once it is in place. There is nothing wrong with the @@ -3270,16 +3362,22 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, */ if (je.s.error == 0 && max_level + (int) depth >= JSON_DEPTH_LIMIT) { - if (marks) - { - marks->is_valid= marks->is_nice= false; + marks.is_valid= marks.is_nice= false; #ifdef DBUG_ASSERT_EXISTS - marks->is_deep= true; + marks.is_deep= true; #endif - } if (!caller_reads_back) json_error(&je.s, JE_DEPTH); } + /* + Measured rather than reckoned, this being the reading the shortcut + above was for going without. Only where the reading finished: what + a walk that stopped early reached is about the part of the value it + got through, and there is nothing to compose out of a value that + did not read as one anyway. + */ + else if (je.s.error == 0 && depth + (uint) max_level > marks.deepest) + marks.deepest= depth + (uint) max_level; /* Said here rather than left to the caller: the callers report through @@ -3295,8 +3393,7 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, */ if (je.s.error) { - if (marks) - marks->is_valid= marks->is_nice= false; + marks.is_valid= marks.is_nice= false; report_json_error_ex(ptr, &je, fname, n_param, Sql_condition::WARN_LEVEL_NOTE); } @@ -3359,17 +3456,34 @@ __attribute__((warn_unused_result)) */ static int append_json_value(String *str, Item *item, String *tmp_val, uint depth, const char *fname, int n_param, - Json_splice_marks *marks, + Json_splice_marks &marks, bool caller_reads_back, bool need_nice) { /* - Too deep is settled before anything is looked at, because a value - put where the limit has already been reached is past it whatever it - turns out to be: the arms below add whatever nesting the value has - of its own and none of them takes any away. Only the arm that - READS the value can say how much further down it reaches, so this - is the whole of the question for the three that read nothing - a - bare word, a null, and a value written out as a string. + Said before anything is looked at, because it holds for whatever the + value turns out to be: it is going in where the caller says, so the + answer reaches at least that far down. Only a value that is a + document of its own reaches further, and that is the one arm below + that has anything to add. + + It is not the caller's own depth restated. A caller that wraps - + putting a new array where a scalar was and the new value beside it - + passes a depth one further down than anything it has written, + and a value QUOTED at that depth would otherwise be counted nowhere + at all. + */ + if (depth > marks.deepest) + marks.deepest= depth; + + /* + And too deep before anything is looked at either, for the same + reason. A value put where the limit has already been reached is + past it whatever it turns out to be: the arms below add whatever + nesting the value has of its own and none of them takes any away. + Only the arm that READS the value can say how much further down it + reaches, so this is the whole of the question for the three that + read nothing - a bare word, a null, and a value written out as a + string. Saying so is all that is owed here. A value only arrives this deep through a function that EDITS a document, by putting a new array @@ -3380,11 +3494,11 @@ static int append_json_value(String *str, Item *item, String *tmp_val, The functions that BUILD a document put their values one level down and cannot get here. */ - if (marks && depth >= JSON_DEPTH_LIMIT) + if (depth >= JSON_DEPTH_LIMIT) { - marks->is_valid= marks->is_nice= false; + marks.is_valid= marks.is_nice= false; #ifdef DBUG_ASSERT_EXISTS - marks->is_deep= true; + marks.is_deep= true; #endif } @@ -3419,7 +3533,8 @@ static int append_json_value(String *str, Item *item, String *tmp_val, if (is_json_type(item)) return append_json_typed_value(str, sv, depth, fname, n_param, marks, caller_reads_back, item->is_valid_json(), - item->is_nice_json(), need_nice); + item->is_nice_json(), need_nice, + item->last_depth()); /* Being a document is what gets a value spliced rather than quoted, @@ -3436,8 +3551,8 @@ static int append_json_value(String *str, Item *item, String *tmp_val, here does not care; one that carries on is building something that no longer reads as a document. */ - if (marks && rc == JSON_APPEND_BAD_CHR) - marks->is_valid= marks->is_nice= false; + if (rc == JSON_APPEND_BAD_CHR) + marks.is_valid= marks.is_nice= false; return rc; } @@ -3450,8 +3565,12 @@ static int append_json_value(String *str, Item *item, String *tmp_val, __attribute__((warn_unused_result)) static int append_json_value_from_field(String *str, Item *i, Field *f, const uchar *key, size_t offset, String *tmp_val, - uint depth, const char *fname, int n_param, Json_splice_marks *marks) + uint depth, const char *fname, int n_param, Json_splice_marks &marks) { + /* Said up front for the reason given in append_json_value(). */ + if (depth > marks.deepest) + marks.deepest= depth; + if (i->type_handler()->is_bool_type()) { longlong v_int= f->val_int(key + offset); @@ -3482,25 +3601,39 @@ static int append_json_value_from_field(String *str, goto append_null; if (is_json_type(i)) /* - The bytes come out of a record rather than off an item, and a - record says nothing about how what is in it was written, so the - document being built cannot be said to be written the loose way. - Nothing has attested to them either: what an item says is about - the value its last evaluation returned, and this one was read - straight out of a record instead. - - Nor is anything written out again here. The only callers that - read a value out of a record are building a document and handing - back what they built, and a value put in as it stands is what a + The bytes come out of a record rather than off an item, so there + is nobody here to ask about them - but the column they came out + of was asked, once for all its rows. A column of a table the + server built for itself can say that its values read as + documents, one producer having filled every row of it and every + store into it having kept what it was given, and it can say that + none of them arrived written any way but the loose one. A column + of any other table says neither, having nowhere to keep a yes. + + Nothing is written out again here. The only callers that read a + value out of a record are building a document and returning + what they built, and a value put in as it stands is what a released server returns. + + It can say how deep they go as well, and that answer is a figure + rather than a yes: the deepest any value written into the column + has gone, which is at least as deep as this one. What it saves + is the only thing left to read a trusted value for - the reading + that is skipped elsewhere is a validating one, and this one + counts brackets and nothing else. Without it the caller falls + back on how long the value is, nothing nesting deeper than half + its length, which is exact for a run of brackets and says + nothing whatever about a long shallow document. */ return append_json_typed_value(str, sv, depth, fname, n_param, marks, - false, false, false, false); + false, f->is_valid_json_static(), + f->is_nice_json_static(), false, + f->json_static_depth()); rc= append_escaped_value(str, sv, i->result_type() == STRING_RESULT, fname, n_param); - if (marks && rc == JSON_APPEND_BAD_CHR) - marks->is_valid= marks->is_nice= false; + if (rc == JSON_APPEND_BAD_CHR) + marks.is_valid= marks.is_nice= false; return rc; } @@ -3646,7 +3779,12 @@ String *Item_func_json_array::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; - Json_splice_marks marks; + /* + The array written here is one structure of its own, which is where + the values go inside - so that is where the reckoning starts, and an + array with nothing in it is one deep for the brackets alone. + */ + Json_splice_marks marks(1); m_marks.clear(); @@ -3655,7 +3793,7 @@ String *Item_func_json_array::val_str(String *str) if (str->append('[') || ((arg_count > 0) && - append_json_value(str, args[0], &tmp_val, 1, func_name(), 0, &marks, + append_json_value(str, args[0], &tmp_val, 1, func_name(), 0, marks, false, false))) goto err_return; @@ -3663,7 +3801,7 @@ String *Item_func_json_array::val_str(String *str) { if (str->append(STRING_WITH_LEN(json_loose_comma)) || append_json_value(str, args[n_arg], &tmp_val, 1, func_name(), - (int) n_arg, &marks, false, false)) + (int) n_arg, marks, false, false)) goto err_return; } @@ -3692,7 +3830,7 @@ String *Item_func_json_array::val_str(String *str) bool is_json_compatible= is_json_compatible_charset(str->charset()); m_marks.set(str, is_json_compatible && marks.is_valid, - is_json_compatible && marks.is_nice); + is_json_compatible && marks.is_nice, marks.deepest); return str; } @@ -3746,20 +3884,23 @@ bool Item_func_json_array_append::fix_length_and_dec(THD *thd) the answer is already sitting in is the one thing the callers differ in, so the copy is made only where it is owed. - 'to' is what comes back, or NULL where the buffer would not grow. - Reporting that is the caller's as well: they do not all report it in - the same place, some of them having a document engine to complain - through and some not. + The depth is the caller's to work out - it is the only part of this + that depends on what the function did to the document - and 'to' is + what comes back, or NULL where the buffer would not grow. Reporting + that is the caller's as well: they do not all report it in the same + place, some of them having a document engine to complain through and + some not. */ -String *Item_json_func::return_json(String *to, const String *from) +String *Item_json_func::return_json(String *to, const String *from, + uint depth) { if ((from != to && to->copy(from->ptr(), from->length(), from->charset())) || DBUG_IF("json_return_out_of_memory")) return NULL; null_value= 0; - m_marks.set(to, true, true); + m_marks.set(to, true, true, depth); return to; } @@ -3776,12 +3917,18 @@ String *Item_func_json_array_append::val_str(String *str) String *const to= str; bool compose_final; uint depth; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; Json_source_watch watch; /* Cleared by anything spliced in that leaves is_valid or is_nice - false for the answer, the depth a path reaches among it. + false for the answer, the depth a path reaches among it. See + Item_func_json_insert::val_str() for what the fourth of them + starts at and why. */ - Json_splice_marks splice; + Json_splice_marks splice(0); DBUG_ASSERT(fixed()); m_marks.clear(); @@ -3799,6 +3946,8 @@ String *Item_func_json_array_append::val_str(String *str) Item_func_json_insert::val_str() asks it. */ compose_final= document_arg_composes_final(args[0], js); + /* Taken here for the reason given at the same place there. */ + js_depth= args[0]->last_depth(); for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { @@ -3885,7 +4034,7 @@ String *Item_func_json_array_append::val_str(String *str) if (n_items) str->append(", ", 2); if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), - (int) n_arg + 1, &splice, + (int) n_arg + 1, splice, !compose_final, compose_final)) goto return_null; /* Out of memory. */ @@ -3930,7 +4079,7 @@ String *Item_func_json_array_append::val_str(String *str) append_simple(str, c_from, c_to - c_from) || str->append(", ", 2) || append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), - (int) n_arg + 1, &splice, + (int) n_arg + 1, splice, !compose_final, compose_final) || str->append(']') || append_simple(str, je.s.c_str, @@ -3957,11 +4106,15 @@ String *Item_func_json_array_append::val_str(String *str) A document that was already written the loose way, with a value written that way put inside it, is written that way already. See Item_func_json_insert::val_str() for why the answer is in js, and - for what makes this safe to believe. + for what makes this safe to believe - and for how deep it goes, + which is worked out the same way here. Wrapping puts a new array + where the value it holds already sat, and the value going in beside + it is counted from inside that array, so the new level is counted + with it. */ if (compose_final && splice.is_valid && splice.is_nice) { - if (!return_json(to, js)) + if (!return_json(to, js, MY_MAX(js_depth, splice.deepest))) goto return_null; /* Out of memory. */ return to; @@ -3970,14 +4123,15 @@ String *Item_func_json_array_append::val_str(String *str) json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; /* - Both marks come from the reading back, for the reason given where - Item_func_json_insert::val_str() marks the same answer. + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. */ - m_marks.set(str, true, true); + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -3999,12 +4153,18 @@ String *Item_func_json_array_insert::val_str(String *str) String *const to= str; bool compose_final; uint depth; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; Json_source_watch watch; /* Cleared by anything spliced in that leaves is_valid or is_nice - false for the answer, the depth a path reaches among it. + false for the answer, the depth a path reaches among it. See + Item_func_json_insert::val_str() for what the fourth of them + starts at and why. */ - Json_splice_marks splice; + Json_splice_marks splice(0); DBUG_ASSERT(fixed()); m_marks.clear(); @@ -4022,6 +4182,8 @@ String *Item_func_json_array_insert::val_str(String *str) Item_func_json_insert::val_str() asks it. */ compose_final= document_arg_composes_final(args[0], js); + /* Taken here for the reason given at the same place there. */ + js_depth= args[0]->last_depth(); for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { @@ -4150,7 +4312,7 @@ String *Item_func_json_array_insert::val_str(String *str) goto return_null; /* Out of memory. */ } if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), - (int) n_arg + 1, &splice, + (int) n_arg + 1, splice, !compose_final, compose_final)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); @@ -4191,7 +4353,7 @@ String *Item_func_json_array_insert::val_str(String *str) goto return_null; /* Out of memory. */ } if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), - (int) n_arg + 1, &splice, + (int) n_arg + 1, splice, !compose_final, compose_final)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); @@ -4225,11 +4387,15 @@ String *Item_func_json_array_insert::val_str(String *str) A document that was already written the loose way, with a value written that way put inside it, is written that way already. See Item_func_json_insert::val_str() for why the answer is in js, and - for what makes this safe to believe. + for what makes this safe to believe - and for how deep it goes, + which is worked out the same way here. Wrapping puts a new array + where the value it holds already sat, and the value going in beside + it is counted from inside that array, so the new level is counted + with it. */ if (compose_final && splice.is_valid && splice.is_nice) { - if (!return_json(to, js)) + if (!return_json(to, js, MY_MAX(js_depth, splice.deepest))) goto return_null; /* Out of memory. */ return to; @@ -4238,14 +4404,15 @@ String *Item_func_json_array_insert::val_str(String *str) json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; /* - Both marks come from the reading back, for the reason given where - Item_func_json_insert::val_str() marks the same answer. + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. */ - m_marks.set(str, true, true); + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -4261,7 +4428,8 @@ String *Item_func_json_object::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; - Json_splice_marks marks; + /* One deep for the braces alone - see Item_func_json_array::val_str(). */ + Json_splice_marks marks(1); m_marks.clear(); @@ -4271,7 +4439,7 @@ String *Item_func_json_object::val_str(String *str) if (str->append('{') || (arg_count > 0 && (append_json_keyname(str, args[0], &tmp_val, func_name(), 0) || - append_json_value(str, args[1], &tmp_val, 1, func_name(), 1, &marks, + append_json_value(str, args[1], &tmp_val, 1, func_name(), 1, marks, false, false)))) goto err_return; @@ -4281,7 +4449,7 @@ String *Item_func_json_object::val_str(String *str) append_json_keyname(str, args[n_arg], &tmp_val, func_name(), (int) n_arg) || append_json_value(str, args[n_arg+1], &tmp_val, 1, func_name(), - (int) n_arg + 1, &marks, false, false)) + (int) n_arg + 1, marks, false, false)) goto err_return; } @@ -4297,7 +4465,7 @@ String *Item_func_json_object::val_str(String *str) bool is_json_compatible= is_json_compatible_charset(str->charset()); m_marks.set(str, is_json_compatible && marks.is_valid, - is_json_compatible && marks.is_nice); + is_json_compatible && marks.is_nice, marks.deepest); return str; } @@ -4560,6 +4728,36 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2, } +/* + How deep the deepest of the document arguments goes, for the two + functions whose answer is composed out of nothing else. Taken one + argument at a time, beside where that argument is evaluated. + + What an argument answers is about the value it has just passed, + and by the end of the loop it need not be about that value any more. + An argument that reads a variable answers out of what is in the + variable now, so a later argument that assigns to the same variable + moves the first one's answer out from under the bytes that went into + the merge - and a document written over by a shallower one then + answers a depth smaller than the answer really goes, which is the one + direction a depth must never be wrong in. The other two marks are + already taken this way; this is the third of them, and was the only + one left to the end. + + One argument that will not say makes the answer not say either: what + is being asked is how deep the deepest part of the result is, and a + part nothing measured could be anywhere. +*/ +static uint deepest_document_argument(uint deepest, Item *arg) +{ + uint arg_depth= arg->last_depth(); + + if (deepest == JSON_DEPTH_UNKNOWN || arg_depth == JSON_DEPTH_UNKNOWN) + return JSON_DEPTH_UNKNOWN; + return MY_MAX(deepest, arg_depth); +} + + /* Says what is wrong with a document argument that was read only as far as its first value, when anything is. @@ -4616,6 +4814,10 @@ String *Item_func_json_merge::val_str(String *str) uint colon_len; /* Raised where the merging puts what it was given inside a new array. */ bool wrapped= false; + /* How deep the deepest document argument goes, taken as each is read. */ + uint deepest= 0; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; Json_source_watch watch; m_marks.clear(); @@ -4639,6 +4841,7 @@ String *Item_func_json_merge::val_str(String *str) rather than into anything a caller wrote. */ compose_final= document_arg_composes_final(args[0], js1); + deepest= deepest_document_argument(deepest, args[0]); colon_len= compose_final ? 3 : 2; for (n_arg=1; n_arg < arg_count; n_arg++) @@ -4663,6 +4866,7 @@ String *Item_func_json_merge::val_str(String *str) */ if (!args[n_arg]->is_valid_json() || !args[n_arg]->is_nice_json()) compose_final= false; + deepest= deepest_document_argument(deepest, args[n_arg]); json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); @@ -4720,10 +4924,24 @@ String *Item_func_json_merge::val_str(String *str) argument, and each of them has already had its say in compose_final as it came round the loop - so there is no set of splice marks to ask, and the one condition is the whole of it. + + Which is also what says how deep the answer goes, ONE LEVEL OUT. + Merging two arrays lays their members side by side and merging two + objects puts their keys together, and neither of those moves + anything further inside than it was. But merging an array with + anything else makes that other thing a MEMBER of the array, and a + member sits one level inside - so the deepest of the arguments is + not enough on its own, which the reading back said the first time + this was written without the level. + + Which of the two happened is not worth telling apart here: a level + more than the answer needs turns down a splice that could have been + taken, and costs a reading nobody is owed an answer about. */ if (compose_final) { - if (!return_json(to, js1)) + if (!return_json(to, js1, + deepest == JSON_DEPTH_UNKNOWN ? deepest : deepest + 1)) goto error_return; return to; @@ -4733,15 +4951,16 @@ String *Item_func_json_merge::val_str(String *str) (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je1, str, Item_func_json_format::LOOSE)) + if (json_nice(&je1, str, Item_func_json_format::LOOSE, &read_back_depth)) goto error_return; null_value= 0; /* - Both marks come from the reading back, for the reason given where - Item_func_json_insert::val_str() marks the same answer. + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. */ - m_marks.set(str, true, true); + m_marks.set(str, true, true, read_back_depth); return str; error_return: @@ -5018,12 +5237,19 @@ String *Item_func_json_merge_patch::val_str(String *str) bool compose_final; uint colon_len; Json_source_watch watch; + /* How deep the deepest document argument goes, taken as each is read. */ + uint deepest= 0; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; /* Cleared by anything that goes into the answer that the answer cannot be said to hold: a document taken over without having been read on - the way in, and a document answering is_nice false. + the way in, and a document whose item attests is_nice false. Nothing is + spliced into a patched document - every piece of the answer is a + document argument - so the fourth of them is never moved off nothing + here. */ - Json_splice_marks splice; + Json_splice_marks splice(0); m_marks.clear(); @@ -5041,6 +5267,7 @@ String *Item_func_json_merge_patch::val_str(String *str) */ compose_final= !merge_to_null && document_arg_composes_final(args[0], js1); colon_len= compose_final ? 3 : 2; + deepest= deepest_document_argument(deepest, args[0]); if (!compose_final) splice.is_nice= false; @@ -5056,6 +5283,16 @@ String *Item_func_json_merge_patch::val_str(String *str) watch.take(js1); js2= args[n_arg]->val_json(&tmp_js2); DBUG_ASSERT(merge_to_null || watch.unchanged(js1)); + /* + Taken before the arms below part, an argument that is dropped + being asked here just as one that goes in is. A depth folded in + from an argument the answer does not hold is a depth larger than + the answer needs, which is the side a depth is allowed to be + wrong on; asking only the ones that stay would make the reading + turn on which arm was taken, and the arm is settled by later + arguments. + */ + deepest= deepest_document_argument(deepest, args[n_arg]); if (args[n_arg]->null_value) { merge_to_null= true; @@ -5197,10 +5434,18 @@ String *Item_func_json_merge_patch::val_str(String *str) be written - and anything else clears is_nice on the spot. A document taken over whole clears both marks where it was taken. + + How deep it goes is read off the arguments, and WITHOUT the extra + level Item_func_json_merge::val_str() takes. Patching is not + merging: the two documents are walked in step, a key of one going + among the keys of the other at the level both were at, and a value + replacing a value where that value stood. Nothing is ever made a + member of anything, which is the case that costs the level there, + and a key the patch drops only takes levels away. */ if (splice.is_valid && splice.is_nice) { - if (!return_json(to, js1)) + if (!return_json(to, js1, deepest)) goto error_return; return to; @@ -5209,15 +5454,16 @@ String *Item_func_json_merge_patch::val_str(String *str) json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je1, str, Item_func_json_format::LOOSE)) + if (json_nice(&je1, str, Item_func_json_format::LOOSE, &read_back_depth)) goto error_return; null_value= 0; /* - Both marks come from the reading back, for the reason given where - Item_func_json_insert::val_str() marks the same answer. + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. */ - m_marks.set(str, true, true); + m_marks.set(str, true, true, read_back_depth); return str; error_return: @@ -5521,13 +5767,19 @@ String *Item_func_json_insert::val_str(String *str) String *const to= str; bool compose_final; uint colon_len; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; Json_source_watch watch; /* Cleared by anything spliced in that leaves is_valid or is_nice false - for the answer, the depth a path reaches among it. Nothing reads it yet; the reading - back below is still what settles the answer. + for the answer, the depth a path reaches among it. The deepest a + spliced value ends up starts at nothing: this writes no structure of + its own, so where the answer is deepest is either somewhere a value + went in or somewhere the document already was. */ - Json_splice_marks splice; + Json_splice_marks splice(0); DBUG_ASSERT(fixed()); m_marks.clear(); @@ -5553,6 +5805,21 @@ String *Item_func_json_insert::val_str(String *str) */ compose_final= document_arg_composes_final(args[0], js); colon_len= compose_final ? 3 : 2; + /* + And how deep the document goes, taken here rather than where it is + used, which is after every other argument has been worked out. + + THIS IS THE CANONICAL SITE for this too. The three answers are + about one value and have to be taken over one moment: an argument + can be any expression a caller cares to write, a stored function + among them, and one of those can assign the very thing args[0] reads + - a stored program's variable, say - between the two readings. The + document in hand is still the one this composed from, so a depth + read afterwards can be the depth of a value this answer is not + about, and too small is the one direction a depth must never be + wrong in. + */ + js_depth= args[0]->last_depth(); thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); @@ -5687,7 +5954,7 @@ String *Item_func_json_insert::val_str(String *str) str->append(", ", 2))) || append_json_value(str, args[n_arg+1], &tmp_val, (uint) je.stack_p + (do_array_autowrap ? 1 : 0), - func_name(), (int) n_arg + 1, &splice, + func_name(), (int) n_arg + 1, splice, !compose_final, compose_final) || (do_array_autowrap && str->append(']')) || append_simple(str, je.s.c_str, js_end - (const char *) je.s.c_str)) @@ -5741,7 +6008,7 @@ String *Item_func_json_insert::val_str(String *str) (n_item > 0 && str->append(", ", 2)) || append_json_value(str, args[n_arg+1], &tmp_val, (uint) je.stack_p + 1, - func_name(), (int) n_arg + 1, &splice, + func_name(), (int) n_arg + 1, splice, !compose_final, compose_final) || append_simple(str, v_to, js_end - v_to)) goto js_error; /* Out of memory. */ @@ -5821,7 +6088,7 @@ String *Item_func_json_insert::val_str(String *str) if (str->append(json_loose_colon, colon_len) || append_json_value(str, args[n_arg+1], &tmp_val, (uint) je.stack_p + 1, - func_name(), (int) n_arg + 1, &splice, + func_name(), (int) n_arg + 1, splice, !compose_final, compose_final) || append_simple(str, v_to, js_end - v_to)) goto js_error; /* Out of memory. */ @@ -5847,7 +6114,7 @@ String *Item_func_json_insert::val_str(String *str) if (append_simple(str, js->ptr(), v_to - js->ptr()) || append_json_value(str, args[n_arg+1], &tmp_val, (uint) je.stack_p, - func_name(), (int) n_arg + 1, &splice, + func_name(), (int) n_arg + 1, splice, !compose_final, compose_final) || append_simple(str, je.s.c_str, js_end - (const char *) je.s.c_str)) goto js_error; /* Out of memory. */ @@ -5907,10 +6174,22 @@ String *Item_func_json_insert::val_str(String *str) then swaps the two, so after an odd number of passes str is the scratch. It is returned in the buffer the caller supplied, which every path through this function has always done. + + HOW DEEP IT GOES, when nothing read it back to find out. Editing a + document puts a value somewhere inside it and copies the rest of it + across untouched, so the answer is deepest either where a value went + in - which splice.deepest holds, counted from the outside - or + somewhere the document already went, which is what the document was + able to say about itself. Neither of those can be short: the first + is a bound the splice was let through on, and the second is + whatever args[0] attested, which is nothing at all unless something + measured it. An item that attests to nothing leaves the answer + attesting to nothing, which is where this stood before there was + anything to ask. */ if (compose_final && splice.is_valid && splice.is_nice) { - if (!return_json(to, js)) + if (!return_json(to, js, MY_MAX(js_depth, splice.deepest))) goto js_error; /* Out of memory. */ return to; @@ -5919,7 +6198,7 @@ String *Item_func_json_insert::val_str(String *str) json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; /* @@ -5939,9 +6218,10 @@ String *Item_func_json_insert::val_str(String *str) what it read, which is what makes it written the loose way. This is the reading that stays wherever the document handed in was nobody's word - it is not a leftover, and taking it out would take 10.11's - answers with it. + answers with it. The depth comes from there too, and is the one + measurement rather than a bound. */ - m_marks.set(str, true, true); + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -5982,6 +6262,10 @@ String *Item_func_json_remove::val_str(String *str) bool compose_final; uint comma_len; Json_source_watch watch; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; DBUG_ASSERT(fixed()); m_marks.clear(); @@ -6002,6 +6286,8 @@ String *Item_func_json_remove::val_str(String *str) */ compose_final= document_arg_composes_final(args[0], js); comma_len= compose_final ? 2 : 1; + /* Taken here for the reason given at the same place there. */ + js_depth= args[0]->last_depth(); for (n_arg=1, n_path=0; n_arg < arg_count; n_arg++, n_path++) { @@ -6213,10 +6499,14 @@ String *Item_func_json_remove::val_str(String *str) reading it back would write it exactly as it stands. See Item_func_json_insert::val_str() for why the answer is in js, and for what makes this safe to believe. + + Nothing goes in, so nothing can go deeper: whatever the document + said about its own depth is still true of it with a piece taken + out, and taking one out is the only thing that happens here. */ if (compose_final) { - if (!return_json(to, js)) + if (!return_json(to, js, js_depth)) goto js_error; /* Out of memory. */ return to; @@ -6225,15 +6515,16 @@ String *Item_func_json_remove::val_str(String *str) json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; null_value= 0; /* - Both marks come from the reading back, for the reason given where - Item_func_json_insert::val_str() marks the same answer. + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. */ - m_marks.set(str, true, true); + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -6645,6 +6936,14 @@ String *Item_func_json_search::val_str(String *str) that quote escaped, or the document would not have read. Several paths are put in an array with the loose formatting between them. + How deep it goes is settled by which of those two was written and by + nothing else: one path is a string and nests nothing, several are an + array of strings and nest one. That is a figure this function has + in hand rather than one it would have to go and measure, and without + it a caller splicing the answer falls back to guessing the depth + from the length - which for paths of any size says more levels than + a document is allowed and sends the whole thing to be read again. + None of that is worth anything if the bytes do not read at all, and a path carrying an array index in a set that writes no character in one byte does not - see the number written into it. A set that can @@ -6656,7 +6955,8 @@ String *Item_func_json_search::val_str(String *str) bool result_is_document= is_json_compatible_charset(str->charset()) && path_bytes_well_formed; - m_marks.set(str, result_is_document, result_is_document); + m_marks.set(str, result_is_document, result_is_document, + n_path_found == 1 ? 0 : 1); } null_value= 0; return str; @@ -6717,6 +7017,7 @@ String *Item_func_json_format::val_str(String *str) json_engine_t je; Json_source_watch watch; int tab_size= 4; + uint deepest= JSON_DEPTH_UNKNOWN; THD *thd; m_marks.clear(); @@ -6750,7 +7051,7 @@ String *Item_func_json_format::val_str(String *str) (const uchar *) js->ptr()+js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, fmt, tab_size)) + if (json_nice(&je, str, fmt, &deepest, tab_size)) { null_value= 1; report_json_error(js, &je, 0); @@ -6761,8 +7062,11 @@ String *Item_func_json_format::val_str(String *str) Read back and written out again, so is_valid holds whatever the argument was - but only json_loose() writes the loose form, the other two formats being the whole point of the other two names. + The depth is what that reading measured, and no formatting changes + it: all three write the same structures and differ only in what + they put between them. */ - m_marks.set(str, true, fmt == LOOSE); + m_marks.set(str, true, fmt == LOOSE, deepest); return str; } @@ -6778,7 +7082,8 @@ String *Item_func_json_format::val_json(String *str) m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; - m_marks.set(js, args[0]->is_valid_json(), args[0]->is_nice_json()); + m_marks.set(js, args[0]->is_valid_json(), args[0]->is_nice_json(), + args[0]->last_depth()); return js; } @@ -6923,6 +7228,7 @@ void Item_func_json_arrayagg::clear() m_bad_element= false; m_closed= false; m_elements_valid= true; + m_elements_depth= 1; Item_func_group_concat::clear(); } @@ -6947,13 +7253,21 @@ void Item_func_json_arrayagg::clear() String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) { int rc; - Json_splice_marks marks; + /* + One deep for the brackets the group is put inside, which is where + every element written here sits - see Item_func_json_array::val_str(). + Carried across the elements rather than kept per element: the group + is as deep as its deepest element, and the elements are written out + one at a time long before the brackets go on. + */ + Json_splice_marks marks(1); m_tmp_json.length(0); - rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0, &marks, + rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0, marks, false, false); if (!marks.is_valid) m_elements_valid= false; + m_elements_depth= MY_MAX(m_elements_depth, marks.deepest); if (rc) { @@ -6969,14 +7283,16 @@ String *Item_func_json_arrayagg::get_str_from_field(Item *i,Field *f, String *tmp, const uchar *key, size_t offset) { int rc; - Json_splice_marks marks; + /* One deep for the brackets - see get_str_from_item() just above. */ + Json_splice_marks marks(1); m_tmp_json.length(0); rc= append_json_value_from_field(&m_tmp_json, i, f, key, offset, tmp, - 1, json_arrayagg_name, 0, &marks); + 1, json_arrayagg_name, 0, marks); if (!marks.is_valid) m_elements_valid= false; + m_elements_depth= MY_MAX(m_elements_depth, marks.deepest); if (rc) { @@ -7087,7 +7403,8 @@ String* Item_func_json_arrayagg::val_str(String *str) grammar takes no SEPARATOR here and writes that comma itself. */ m_marks.set(str, is_json_compatible_charset(str->charset()) && - m_elements_valid && !warning_for_row, false); + m_elements_valid && !warning_for_row, false, + m_elements_depth); } return str; @@ -7100,7 +7417,7 @@ String* Item_func_json_arrayagg::val_str(String *str) Item_func_json_objectagg:: Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item) :Item_sum(thd, item), m_bad_pair(false), m_closed(false), - m_pairs_valid(true) + m_pairs_valid(true), m_pairs_depth(1) { quick_group= FALSE; result.set_charset(collation.collation); @@ -7185,6 +7502,7 @@ void Item_func_json_objectagg::clear() m_bad_pair= false; m_closed= false; m_pairs_valid= true; + m_pairs_depth= 1; if (result.append('{')) m_bad_pair= true; } @@ -7193,7 +7511,11 @@ void Item_func_json_objectagg::clear() bool Item_func_json_objectagg::add() { StringBuffer buf; - Json_splice_marks marks; + /* + One deep for the braces the group is put inside, and carried across + the pairs - see Item_func_json_arrayagg::get_str_from_item(). + */ + Json_splice_marks marks(1); String *key; int rc; @@ -7236,10 +7558,11 @@ bool Item_func_json_objectagg::add() goto bad_pair; buf.length(0); - rc= append_json_value(&result, args[1], &buf, 1, func_name(), 1, &marks, + rc= append_json_value(&result, args[1], &buf, 1, func_name(), 1, marks, false, false); if (!marks.is_valid) m_pairs_valid= false; + m_pairs_depth= MY_MAX(m_pairs_depth, marks.deepest); if (rc == JSON_APPEND_OOM) goto bad_pair; @@ -7278,7 +7601,7 @@ String* Item_func_json_objectagg::val_str(String* str) between the two. */ m_marks.set(&result, is_json_compatible_charset(result.charset()) && - m_pairs_valid, false); + m_pairs_valid, false, m_pairs_depth); return &result; } @@ -7322,8 +7645,15 @@ String *Item_func_json_normalize::val_str(String *buf) and written in the character set this function declares, whatever the argument arrived in. Not written the loose way, though: the normal form puts nothing after a comma or a colon. + + As deep as what it was made from. Normalising sorts the keys of an + object and takes the spacing out; it puts nothing inside anything + and takes nothing out of anything, so every structure that was there + is there still and no other is. So whatever the argument could say + about its own depth is said about this, and where it said nothing + this says nothing, which is where it stood. */ - m_marks.set(buf, true, false); + m_marks.set(buf, true, false, args[0]->last_depth()); goto end; null_return: diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 4868a53a12ffc..2396696207960 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -26,6 +26,38 @@ #include "item_sum.h" #include "sql_type_json.h" +/* + The three below are read by DBUG_ASSERT and by nothing else, so they + are here wherever a DBUG_ASSERT is compiled rather than wherever a + debug build is. Those are not the same set: DBUG_ASSERT_AS_PRINTF + turns the assertion into a printed complaint and leaves its expression + standing, while setting DBUG_OFF, so a guard written the other way + round leaves those expressions with nothing to call and that build + stops compiling. +*/ +#ifdef DBUG_ASSERT_EXISTS +/* + Reads a value the way whoever receives it will read it, and says + whether it got to the end. For the debug checks that hold the marks to + what they say; the reading is not counted, being the debug build's work + and not the server's. +*/ +bool json_value_reads_as_document(const String *str); +/* + Writes the value out again in the loose form and says whether that + changed anything. For the same debug checks, and uncounted for the + same reason. +*/ +bool json_value_is_nice(const String *str); +/* + Reads the value and says how deep it actually goes. For the checks + that hold a claimed depth to the truth - a claim larger than this is + what saying nothing amounts to, and one smaller is the failure worth + catching. Uncounted for the same reason as the two above. +*/ +uint json_value_depth(const String *str); +#endif + #ifndef DBUG_OFF /* Holds the reading count still across a reading the released server does @@ -120,6 +152,15 @@ class Json_path_extractor: public json_path_with_flags { protected: String tmp_js, tmp_path; + /* + What the document said about itself, taken while extract() had just + evaluated it and before it worked the path out. A path is any + expression a caller cares to write, so the two readings would + otherwise be a stored function apart, and what the document answers + can change over that distance while the document in hand does not. + */ + bool m_js_nice; + uint m_js_depth; virtual ~Json_path_extractor() { } virtual bool check_and_get_value(Json_engine_scan *je, String *to, int *error)=0; @@ -210,7 +251,7 @@ class Item_json_func: public Item_str_func again to find out what it is - out of line, where the rule it stands on is written. */ - String *return_json(String *to, const String *from); + String *return_json(String *to, const String *from, uint depth); public: Item_json_func(THD *thd) :Item_str_func(thd) { } @@ -226,6 +267,7 @@ class Item_json_func: public Item_str_func } bool is_valid_json() const override { return m_marks.valid(); } bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } }; @@ -287,8 +329,16 @@ class Item_func_json_query: public Item_json_func, inside a document is written there the way it would be written on its own, and the two ends of the cut are the two ends of the value with no spacing of the document's left on either side. + + And a piece of a document does not nest deeper than the document + it was cut out of, wherever inside it the cut was made - so what + the document could say about its own depth is said about the + piece as well. + + Both of them as the document answered them when it was read, not + as it answers them now - see Json_path_extractor::m_js_nice. */ - m_marks.set(to, true, args[0]->is_nice_json()); + m_marks.set(to, true, m_js_nice, m_js_depth); return to; } bool check_and_get_value(Json_engine_scan *je, @@ -911,6 +961,13 @@ class Item_func_json_arrayagg : public Item_func_group_concat still not a document. */ bool m_elements_valid; + /* + How deep the deepest element written so far reaches, counted from + outside the brackets that go round the group. Accumulated over the + elements for the same reason as the mark above: the group is written + out a row at a time, and the answer is the deepest of them. + */ + uint m_elements_depth; Json_result_marks m_marks; public: String m_tmp_json; /* Used in get_str_from_*.. */ @@ -920,7 +977,8 @@ class Item_func_json_arrayagg : public Item_func_group_concat bool limit_clause, Item *row_limit, Item *offset_limit): Item_func_group_concat(thd, context_arg, is_distinct, is_select, is_order, is_separator, limit_clause, row_limit, offset_limit), - m_bad_element(false), m_closed(false), m_elements_valid(true) + m_bad_element(false), m_closed(false), m_elements_valid(true), + m_elements_depth(1) { } /* @@ -932,7 +990,7 @@ class Item_func_json_arrayagg : public Item_func_group_concat */ Item_func_json_arrayagg(THD *thd, Item_func_json_arrayagg *item) : Item_func_group_concat(thd, item), m_bad_element(false), - m_closed(false), m_elements_valid(true) + m_closed(false), m_elements_valid(true), m_elements_depth(1) { m_tmp_json.set_charset(collation.collation); } @@ -942,6 +1000,7 @@ class Item_func_json_arrayagg : public Item_func_group_concat } bool is_valid_json() const override { return m_marks.valid(); } bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } LEX_CSTRING func_name_cstring() const override { @@ -989,6 +1048,12 @@ class Item_func_json_objectagg : public Item_sum a document. */ bool m_pairs_valid; + /* + How deep the deepest value written so far reaches, counted from + outside the braces that go round the group - the sister aggregate's + m_elements_depth, for the same reason. + */ + uint m_pairs_depth; Json_result_marks m_marks; public: /* @@ -1000,7 +1065,7 @@ class Item_func_json_objectagg : public Item_sum */ Item_func_json_objectagg(THD *thd, Item *key, Item *value) : Item_sum(thd, key, value), m_bad_pair(false), m_closed(false), - m_pairs_valid(true) + m_pairs_valid(true), m_pairs_depth(1) { quick_group= FALSE; } @@ -1020,6 +1085,7 @@ class Item_func_json_objectagg : public Item_sum } bool is_valid_json() const override { return m_marks.valid(); } bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } void clear() override; bool add() override; void reset_field() override { DBUG_ASSERT(0); } // not used diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index c572d1d1547fd..00d0bfc0b1d07 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -3896,11 +3896,22 @@ String *Item_func_conv_charset::val_str(String *str) /* Asked of the argument only now, after it has been evaluated: what it answers is about the value it has just passed, which is the one - that was converted. + that was converted. A conversion that lost nothing kept the + characters it was given, and how deeply a value nests is a property + of characters like the other two, so all three carry across. + + Where the copy kept the BYTES and called them something else it + kept no such thing, and that is what the second test is for - see + String_copier::conversion_keeps_characters(), which reads the branch + this very copier took. Either end being the binary set is enough: + bytes going into it are relabelled, and bytes coming out of it are + read as whatever the other set makes of them. */ if (!copier.most_important_error_pos() && - collation.collation != &my_charset_bin) - m_marks.set(str, args[0]->is_valid_json(), args[0]->is_nice_json()); + String_copier::conversion_keeps_characters(collation.collation, + arg->charset())) + m_marks.set(str, args[0]->is_valid_json(), args[0]->is_nice_json(), + args[0]->last_depth()); return str; } diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index fc7432a4f292f..1419d25e7c2ec 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2025,9 +2025,10 @@ class Item_func_conv_charset :public Item_str_func later would pair an answer about a value it has since gone on to produce with the one frozen above. */ - if (!null_value && safe && cs != &my_charset_bin) + if (!null_value && safe && + String_copier::conversion_keeps_characters(cs, str->charset())) m_marks.set(&str_value, args[0]->is_valid_json(), - args[0]->is_nice_json()); + args[0]->is_nice_json(), args[0]->last_depth()); } else { @@ -2095,6 +2096,7 @@ class Item_func_conv_charset :public Item_str_func } bool is_valid_json() const override { return m_marks.valid(); } bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } void print(String *str, enum_query_type query_type) override; int save_in_field(Field*, bool) override; diff --git a/sql/select_handler.cc b/sql/select_handler.cc index b0b8e58623de7..21126fb631780 100644 --- a/sql/select_handler.cc +++ b/sql/select_handler.cc @@ -64,6 +64,9 @@ TABLE *select_handler::create_tmp_table(THD *thd, SELECT_LEX *select) (ORDER *) 0, false, 0, TMP_TABLE_ALL_COLUMNS, 1, &empty_clex_str, true, false); + /* The engine the select was pushed down to writes the rows of this one. */ + if (table) + table->set_filled_by_engine(); DBUG_RETURN(table); } @@ -75,8 +78,28 @@ bool select_handler::prepare() Some engines (e.g. XPand) initialize "table" on their own. So we need to create a temporary table only if "table" is NULL. */ - if (!table && !(table= create_tmp_table(thd, select))) - DBUG_RETURN(true); + if (!table) + { + if (!(table= create_tmp_table(thd, select))) + DBUG_RETURN(true); + } + else + { + /* + A table the engine brought with it is a table the engine fills, so + it drops the attestations about its columns for the same reason + one built here does - see TABLE::set_filled_by_engine(). Done on + this arm too, and not only where the table is built, because what + has to hold is that EVERY way in has dropped them. + + No engine in the tree takes this arm, every one of them leaving + the table to be built above, so nothing here is exercised by a + test. It is written all the same: the arm exists for engines that + are not in the tree, and an answer that is only right because + nobody takes the arm is not an answer. + */ + table->set_filled_by_engine(); + } DBUG_RETURN(table->fill_item_list(&result_columns)); } @@ -128,6 +151,15 @@ int select_handler::execute() DBUG_ENTER("select_handler::execute"); + /* + Nothing attests to what an engine puts in a table it fills, so the + bitmaps are dropped where the table is given to the engine - see + TABLE::set_filled_by_engine(). Asserted here rather than there + because what has to hold is that EVERY way in has dropped them, and + the ways in all come through here. + */ + DBUG_ASSERT(!table->is_valid_json_static_set); + if ((err= init_scan())) goto error; diff --git a/sql/sql_class.h b/sql/sql_class.h index 84602e6679e93..81aa78226c0ea 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -8248,15 +8248,25 @@ class Type_holder: public Sql_alloc, { const TYPELIB *m_typelib; bool m_maybe_null; + bool m_is_valid_json_static; public: Type_holder() :m_typelib(NULL), - m_maybe_null(false) + m_maybe_null(false), + m_is_valid_json_static(false) { } void set_type_maybe_null(bool maybe_null_arg) override { m_maybe_null= maybe_null_arg; } bool get_maybe_null() const { return m_maybe_null; } + /* + Whether every producer gathered here returns a document each time + it is evaluated. One column is written by all of them and read as + one, so what can be said about it is what they all say - and an + answer about no producers at all is about nothing, which is why + having gathered none is a no rather than a yes about an empty set. + */ + bool get_is_valid_json_static() const { return m_is_valid_json_static; } decimal_digits_t decimal_precision() const override { @@ -8282,8 +8292,28 @@ class Type_holder: public Sql_alloc, bool aggregate_attributes(THD *thd) { static LEX_CSTRING union_name= { STRING_WITH_LEN("UNION") }; + m_is_valid_json_static= arg_count > 0; for (uint i= 0; i < arg_count; i++) + { m_maybe_null|= args[i]->maybe_null(); + /* + What this producer always answers, and not what the column's + store will keep of it. Only the first of those can be asked + here: the set the column will be in is settled by the call + below, so there is nothing yet to hold a branch's own set up + against - and a branch made to move into a set that keeps its + bytes without keeping its characters passes a document that + nothing reads as one, however faithfully it attests to what it + wrote. + + Asked at the store instead, where both sets are in hand - see + Field::confirm_is_valid_json_static(). Asked in both places it + would be one rule written twice, and this is the copy that could + be wrong: the field is made by create_tmp_field(), which is free + to land somewhere other than the set agreed on here. + */ + m_is_valid_json_static&= args[i]->is_valid_json_static(); + } return type_handler()->Item_hybrid_func_fix_attributes(thd, union_name, this, this, diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index aba2b41a4c9b4..49ae781b8a49f 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -3129,6 +3129,14 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) (my_bitmap_map*) (bitmap + (bitmaps_used + 1)*share->column_bitmap_size), share->fields); + /* + The table this was copied from is a base table and so has none, but + the copy is made by memcpy and would carry the pointers over rather + than have any storage of its own. Said outright, so that a table + which is not the server's own can never come to attest to what is + written into it. + */ + copy->set_filled_by_engine(); copy->tmp_set.bitmap= 0; // To catch errors bzero((char*) bitmap, share->column_bitmap_size * bitmaps_used); copy->read_set= ©->def_read_set; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 6edfcc72b17ee..3f68ce970ad59 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3797,6 +3797,13 @@ bool JOIN::make_aggr_tables_info() /* Give storage engine access to temporary table */ gbh->table= table; + /* + And it is the engine that writes its rows - see + TABLE::set_filled_by_engine(), and select_handler:: + create_tmp_table(), which says the same of the table it + builds out of the same call. + */ + table->set_filled_by_engine(); pushdown_query->store_data_in_temp_table= need_tmp; pushdown_query->having= having; @@ -20896,6 +20903,11 @@ TABLE *Create_tmp_table::start(THD *thd, Field **reg_field; uint *blob_field; key_part_map *const_key_parts; + MY_BITMAP *json_static_valid_map; + uchar *json_static_valid_bits; + MY_BITMAP *json_static_nice_map; + uchar *json_static_nice_bits; + uint *json_static_depth; /* Treat sum functions as normal ones when loose index scan is used. */ m_save_sum_fields|= param->precomputed_group_by; DBUG_ENTER("Create_tmp_table::start"); @@ -20993,6 +21005,11 @@ TABLE *Create_tmp_table::start(THD *thd, &m_group_buff, (m_group && ! m_using_unique_constraint ? param->group_length : 0), &m_bitmaps, bitmap_buffer_size(field_count)*6, + &json_static_valid_map, sizeof(MY_BITMAP), + &json_static_valid_bits, bitmap_buffer_size(field_count), + &json_static_nice_map, sizeof(MY_BITMAP), + &json_static_nice_bits, bitmap_buffer_size(field_count), + &json_static_depth, sizeof(uint) * field_count, &const_key_parts, sizeof(*const_key_parts), NullS)) { @@ -21013,6 +21030,7 @@ TABLE *Create_tmp_table::start(THD *thd, bzero((char*) m_from_field, sizeof(Field*) * field_count); /* const_key_parts is used in sort_and_filter_keyuse */ bzero((char*) const_key_parts, sizeof(*const_key_parts)); + bzero((char*) json_static_depth, sizeof(uint) * field_count); table->mem_root= own_root; mem_root_save= thd->mem_root; @@ -21027,6 +21045,35 @@ TABLE *Create_tmp_table::start(THD *thd, table->temp_pool_slot= m_temp_pool_slot; table->copy_blobs= 1; table->in_use= thd; + + /* + A table built here is the server's own, and each of its fields has one + producer, known while the field is being made. So this is a table + that can attest to what will be written into it, and the bitmap + being here at all is what says so - see + TABLE::is_valid_json_static_set. It is sized for every field the + table may come to have, the fields being added after this. + */ + table->is_valid_json_static_set= json_static_valid_map; + my_bitmap_init(table->is_valid_json_static_set, + (my_bitmap_map*) json_static_valid_bits, field_count); + /* + The formatting of what is written there, kept beside it and made here + for the same reason - see TABLE::is_nice_json_static_set. The two + are made together and given up together, so nothing has to ask which + of them a table has. + */ + table->is_nice_json_static_set= json_static_nice_map; + my_bitmap_init(table->is_nice_json_static_set, + (my_bitmap_map*) json_static_nice_bits, field_count); + /* + And how deep it goes, kept beside the two of them and for the same + reason - see TABLE::json_static_depth. An entry is written when the + field it belongs to is added, so what is here now is only what a + field never added would have been read as, and nothing reads one of + those. + */ + table->json_static_depth= json_static_depth; table->no_rows_with_nulls= param->force_not_null_cols; table->group_concat= param->group_concat; table->expr_arena= thd; @@ -21144,6 +21191,7 @@ bool Create_tmp_table::add_fields(THD *thd, Item_field(thd, new_field))) goto err; ((Item_field*) tmp_item)->set_refers_to_temp_table(); + Item *producer= arg; arg= sum_item->set_arg(i, thd, tmp_item); thd->mem_root= &table->mem_root; @@ -21152,6 +21200,13 @@ bool Create_tmp_table::add_fields(THD *thd, m_field_count[current_counter]++; m_uneven_bit[current_counter]+= (m_uneven_bit_length - uneven_delta); + /* + Said after add_field() for the reason given at the other such + call below. The item that fills the field is kept aside above, + set_arg() having by now put the replacement in its place. + */ + new_field->set_is_valid_json_static(producer); + if (!(new_field->flags & NOT_NULL_FLAG)) { /* @@ -21237,6 +21292,13 @@ bool Create_tmp_table::add_fields(THD *thd, m_field_count[current_counter]++; m_uneven_bit[current_counter]+= (m_uneven_bit_length - uneven_delta); + /* + Said here rather than where the field was made, add_field() being + where the field learns its own number and so the first place it + can be spoken about. + */ + new_field->set_is_valid_json_static(item); + if (item->marker == MARKER_NULL_KEY && item->maybe_null()) { m_group_null_items++; @@ -28445,7 +28507,17 @@ copy_fields(TMP_TABLE_PARAM *param) DBUG_ASSERT((ptr != NULL && end >= ptr) || (ptr == NULL && end == NULL)); for (; ptr != end; ptr++) + { (*ptr->do_copy)(ptr); + /* + The other route that goes around field_conv() - see there. Whether + there is anything to confirm was settled where the entry was made: + an entry set up by Copy_field::set(uchar*, Field*) records neither + field and answers no, both kinds of entry sharing this array. + */ + if (unlikely(ptr->to_needs_confirm)) + ptr->to_field->confirm_is_valid_json_static_from(ptr->from_field); + } List_iterator_fast it(param->copy_funcs); Item_copy *item; diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc index 3fa4675574c48..deeaa1b4c0233 100644 --- a/sql/sql_tvc.cc +++ b/sql/sql_tvc.cc @@ -292,7 +292,9 @@ bool table_value_constr::prepare(THD *thd, SELECT_LEX *sl, Item_type_holder *new_holder= new (thd->mem_root) Item_type_holder(thd, item, holders[pos].type_handler(), &holders[pos]/*Type_all_attributes*/, - holders[pos].get_maybe_null()); + holders[pos].get_maybe_null(), + holders[pos]. + get_is_valid_json_static()); sl->item_list.push_back(new_holder); } if (arena) diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 0d87c830adf38..f5031cf86a3da 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -1261,13 +1261,22 @@ bool st_select_lex_unit::join_union_item_types(THD *thd_arg, */ bool pos_maybe_null= is_recursive ? true : holders[pos].get_maybe_null(); + /* + Only the non-recursive parts were joined above, so for a recursive + CTE the producers of the recursive parts were never asked, and the + ones that were must not answer in their place. + */ + bool pos_is_valid_json_static= is_recursive ? false : + holders[pos].get_is_valid_json_static(); + /* Error's in 'new' will be detected after loop */ types.push_back(new (thd_arg->mem_root) Item_type_holder(thd_arg, item_tmp, holders[pos].type_handler(), &holders[pos]/*Type_all_attributes*/, - pos_maybe_null)); + pos_maybe_null, + pos_is_valid_json_static)); } if (unlikely(thd_arg->is_fatal_error)) DBUG_RETURN(true); // out of memory diff --git a/sql/table.h b/sql/table.h index 75c985f2278f1..4e58dd77eb078 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1380,6 +1380,81 @@ struct TABLE can ever be set and nothing reads one. */ bool has_own_json_valid_check; + /* + Fields of a temporary table the server built for itself, every value + of which came from an item that answers is_valid_json_static() and + was put down whole. A reader of such a field is reading a document, + whatever row it has in front of it. + + Unlike the bitmap above, these bits say nothing about one row image. + They are written once, while the table is being built and each + field's one producer is known, and they hold for every row the field + will ever be given - which is why a reader can still believe one long + after the row was written and the row image it was written in is + gone. + + A bit is taken back, for good, if a store into the field ever puts + down less than it was handed. That is the one way the promise could + come apart, and it is not a way the store will own up to: truncation + is reported only where count_cuted_fields is raised, which is not + where a temporary table is written. So the store is asked what it + kept rather than whether it minded. + + Only Create_tmp_table leaves a bitmap here. A base table and a table + the user asked for have nowhere to keep a yes, and so answer no by + construction - which is the whole of why their columns stay unread: + a row can hold bytes no check ever saw, and nothing in one records + where it came from. + */ + MY_BITMAP *is_valid_json_static_set; + /* + Of those same fields, the ones every value has so far reached + formatted the way json_nice() writes a document in its loose form. + + This one cannot be promised in advance the way the one above is. + Being a document is a property of the producer, which is why it can + be asked before a row exists; the FORMATTING is a property of each + value the producer passes, and there is nobody to ask about a + value that has not been made yet. So the bit starts as a yes + wherever the bit above was granted, and the first value that arrives + formatted otherwise takes it back for good. + + What it says when it is read is therefore "nothing put here so far + was formatted another way", which is what a reader of a row already + written needs of it: the value in front of that reader was put down + before the read, so a bit that still stands now also stood when that + value was written. + + Read only through the bitmap above, a formatting being nothing to say + about a column whose values are not known to be documents at all. + That is why is_valid_json_static_set alone decides whether a table + keeps these answers; this pointer is never tested on its own. + */ + MY_BITMAP *is_nice_json_static_set; + + /* + Of those same fields, how deep the deepest value that has reached + each of them goes. + + A number rather than a bit, because what reads this is composing: + it is putting the value inside something of its own and needs a + figure to add its own levels to, not a yes. The column is written + by many values and read as one, so what holds for all of them is the + deepest of them, and the entry only ever rises. + + Too large is the safe direction here, as saying no is for the two + above: a figure larger than the truth costs a reading that need not + have happened, while one smaller lets a document be composed that + goes past what this server can read back. JSON_DEPTH_UNKNOWN is + the largest of all and so absorbs everything - one value nobody + counted takes the column there and leaves it there, which is what + it means for a column to have stopped saying. + + Read through the first bitmap, like the second, a depth being + nothing to say about a column whose values are not known to be + documents at all. + */ + uint *json_static_depth; /* The ID of the query that opened and is using this table. Has different @@ -1684,6 +1759,31 @@ struct TABLE memcpy(record[0], from, (size_t) s->reclength); clear_is_valid_json_marks(); } + /* + Said of a table this server built and a storage engine fills: a + select, a grouping or a derived table pushed down whole borrows this + machinery for its columns and then has its rows written by the + engine, straight into record[0]. No store of ours ever sees what + goes into them, so none of the columns can attest to their values - + however well the item a column was made from attests to the values + IT makes. + + Taking the bitmaps away is how that is said, a table with nowhere to + keep a yes answering no to everything. All three go, the two behind + the first being read only through it but pointing at storage of + their own all the same. + + Said at the moment the engine is handed the table rather than when + it starts filling it: a table built out of this one asks these + columns what they promise while it is being built, which is before + any row of either exists. + */ + void set_filled_by_engine() + { + is_valid_json_static_set= NULL; + is_nice_json_static_set= NULL; + json_static_depth= NULL; + } #ifndef DBUG_OFF void check_json_valid_mark(Virtual_column_info *check); #endif diff --git a/unittest/sql/copy_field-t.cc b/unittest/sql/copy_field-t.cc index 7bd46722f0c2b..7bd0a379f943d 100644 --- a/unittest/sql/copy_field-t.cc +++ b/unittest/sql/copy_field-t.cc @@ -25,7 +25,9 @@ either. Entries of both kinds are put in one array, which callers walk with a single loop, so from_field and to_field have to read as absent where they were never filled in rather than as whatever the memory the - array was built on happened to hold. + array was built on happened to hold. The same goes for the bit the + destination carries, which the field to string overload never records + either and which decides whether the copy confirms the attestation. */ #include @@ -41,6 +43,24 @@ static const int POISON= 0xAB; +/** + Whether a bool reads back as false. + + Asked of the byte rather than of the value. A bool holding neither 0 + nor 1 makes any comparison of it undefined, and ok() takes anything + but zero for a pass, so a plain comparison would silently pass over + exactly the memory this file is about. +*/ + +static bool starts_false(const bool *flag) +{ + unsigned char raw; + + memcpy(&raw, flag, sizeof(raw)); + return raw == 0; +} + + /** Construct over memory known to be dirty. */ @@ -60,8 +80,12 @@ static void test_construction_over_dirty_memory() ok(first->from_field == NULL, "first from_field starts absent"); ok(first->to_field == NULL, "first to_field starts absent"); + ok(starts_false(&first->to_needs_confirm), + "first to_needs_confirm starts false"); ok(second->from_field == NULL, "second from_field starts absent"); ok(second->to_field == NULL, "second to_field starts absent"); + ok(starts_false(&second->to_needs_confirm), + "second to_needs_confirm starts false"); second->~Copy_field(); first->~Copy_field(); @@ -92,6 +116,8 @@ static void test_array_on_mem_root() { ok(copy[i].from_field == NULL, "copy[%u] from_field starts absent", i); ok(copy[i].to_field == NULL, "copy[%u] to_field starts absent", i); + ok(starts_false(©[i].to_needs_confirm), + "copy[%u] to_needs_confirm starts false", i); } free_root(&mem_root, MYF(0)); } @@ -100,7 +126,7 @@ static void test_array_on_mem_root() int main(int argc __attribute__((unused)), char *argv[]) { MY_INIT(argv[0]); - plan(10); + plan(15); test_construction_over_dirty_memory(); test_array_on_mem_root(); From fb59607a058b12f50b4d0a8ca073985aec2bec57 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Fri, 7 Aug 2026 19:26:59 -0400 Subject: [PATCH 18/30] MDEV-40668 Give a JSON function a document it can keep A JSON function keeps its document for the whole of its work. It walks the document to find the place it was asked about, then works out the rest of its arguments, and only then reads the pieces of the document either side of that place. Working out an argument runs whatever the caller wrote. What the caller wrote can assign to the very variable the document came out of. The variables of a package body are in reach of every routine in that package, so a function called to work out one argument can store a longer value into the variable holding another, and storing it gives that variable a new buffer and lets go of the old one. The old one is what is being read from, so the rest of the document comes out of freed heap. Nothing says so. The functions that splice a document back together re-parse what they built, and bytes off freed heap usually fail that re-parse, so the answer is `NULL` and a `Broken JSON string` warning - which is what a caller sees for an ordinary malformed document. The functions that only read have no such re-parse and answer from whatever the buffer holds by then. ## Where it comes from `Item_sp_variable::val_str()` returns a pointer INTO the variable rather than bytes of the caller's own: if (res != &str_value) str_value.set(res->ptr(), res->length(), res->charset()); That is deliberate, and the comment above it says why: a function that builds into the buffer it is offered - `CONCAT` is the example given - must not get to build into the variable. What it leaves the caller holding is a view, good only for as long as nothing writes where it points. A caller that means to keep one across working something else out cannot be given one. A user variable has never been read this way. `user_var_entry:: val_str()` copies, which is why the same statement written with one of those holds together, and it is the reason a copy here is no new kind of cost. ## What is done about it `Item_sp_variable` answers `val_json()` with a copy in the caller's own buffer. That is one place rather than one per function: it holds for every reader that asks for a document, whenever that reader was written. A reader has to ask, though. The copy is made in the answer to a question, so anything that reads the variable some other way is handed the view as before, and a reader that keeps what it read has to be the one to put the question. The document argument is worked out before the others, so the value the variable held then is the one the answer is owed about - which is what a copy gives. One place, provided everything standing between the function and the variable asks the same way. A reference did not: `Item_ref::val_str()` reads its referent through `str_result()` and `Item_direct_ref:: val_str()` through `val_str()`, but neither said anything about `val_json()`, so a request for a document fell back to `Item::val_json()`, which is `val_str()` - and the variable handed out the view again with a wrapper over it. So the two formats get result-side twins beside the marks that already had them, virtual String *val_json_result(String *str); virtual String *val_json_at_once_result(String *str); each class transcribing whichever side its own `val_str()` reads: `Item_field` and `Item_func_set_user_var` out of their `str_result()`, `Item_direct_view_ref` by passing on, `Item_ref` by the same runtime condition it uses for the marks. `Item_direct_view_ref` keeps the row check its other val_XXX have, so a row an outer join filled with NULLs is answered without the referent being worked out at all, and `Item_ref_null_helper` notes the NULL down for its owner as its `val_str()` does. Of the five shapes that put a reference in front of a variable, only `ORDER BY` over a select-list alias reaches this - each was run on its own rather than taken as equivalent to another. A call with nothing left to work out after reading the document is finished with it before anything else can run, and has no use for a copy. Those read it with `val_json_at_once()`, which returns the view `val_str()` would. Asked of the argument count rather than written into each function, so that one which gains an optional argument later goes back to being read the careful way without anyone having to remember to say so. One is left out of it. `JSON_UNQUOTE` hands its argument straight on when there is nothing to unquote, so what it would hand its own caller is the view. `JSON_COMPACT`, `JSON_LOOSE` and `JSON_DETAILED` hand their argument's value straight on as well, where one of them is read as a document rather than as a string and there is nothing to rewrite - so a caller that is finished with what they return is finished with what the argument holds. They pass the promise along rather than stopping it, asking the argument the way they were asked themselves; the body that is common to both ways of asking is written once. `JSON_NORMALIZE` writes its answer out afresh and returns nothing of what it was given, so it takes its argument the way every other such function does - through the one place that decides which way to ask, rather than by reading the value for itself. Nothing else changes. A document that arrives some other way is untouched, and so is `val_str()`. ## A walk that outlives the row A table built out of a document keeps it for longer than any function does. `ha_json_table::rnd_init()` is told where the document stands and the walk reads on from there each time a row is asked for, so that note outlives everything the query does between one row and the next. Working out a row can write to whatever the document was read from - a package body's variable is the plain case - and a value stored over it that wants a bigger buffer lets the old one go while the walk is still pointing into it. So the document is asked for the way that returns bytes the reader can keep, into a buffer the handler owns for as long as the walk lasts. A released server answers SELECT jt.v FROM JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt WHERE grow() = 7; with `4038 Syntax error in JSON text ... at position 10` in place of its rows. Under a sanitizer the same read is a use-after-free in `get_first_nonspace()`, reached from `Json_table_nested_path::scan_next()`, on the buffer that `Field_blob::store()` let go while the condition was being worked out. Both go, and the rows come back. ## What it reaches Twenty-one functions over seventeen implementations, each one shown under AddressSanitizer to read freed memory: `JSON_INSERT`, `JSON_SET`, `JSON_REPLACE`, `JSON_ARRAY_APPEND`, `JSON_ARRAY_INSERT`, `JSON_CONTAINS`, `JSON_EXISTS`, `JSON_VALUE`, `JSON_QUERY`, `JSON_EXTRACT`, `JSON_CONTAINS_PATH`, `JSON_LENGTH`, `JSON_REMOVE`, `JSON_KEYS`, `JSON_EQUALS`, `JSON_MERGE`, `JSON_MERGE_PRESERVE`, `JSON_MERGE_PATCH`, `JSON_OVERLAPS`, `JSON_SEARCH` and `JSON_DETAILED`. It is not about value arguments, and not about the functions that edit a document. Every argument is worked out just as late as every other, so a path reaches it - `JSON_REMOVE` has no value argument at all - a string to search for reaches it, and so does the number of spaces `JSON_DETAILED` is asked to indent by. Functions taking a document and nothing else have nothing to work out afterwards and are not affected. ## The test `func_json_aliasing` gained five sections. Its sections 4 to 6 already hold a user variable, a routine's variable and a column, but nothing a routine calls can reach the routine's own variables, so none of them could write where the document stands. A package body's variables are shared by everything in the package, which is what the new ones use. Two of the four are about the route above. A reference exists only for a select-list alias and a select-list entry is always printed, so no shape both routes a variable through a reference and prints what the function answered; the document is therefore grown and put back, and the row shows something short. A buffer is let go only where the next value will not fit, so each case needs a variable of its own written away exactly once, or the first grow makes room for every later one and nothing is freed at all. The fourth of those cases is quiet and earns its place by a liveness probe rather than by failing: a call finished with its document may hold a view, which is the whole of what `val_json_at_once()` is for. The last section reads a depth after the arguments have run, over a document written shallower than the one measured, which is what makes a wrong depth rather than a freed read the thing that shows. It covers the merging and the patching beside the five that ask their document early, and those two are reached differently: the five take a value or a path that can be made to write the variable on its way past, while the merging and the patching take documents, so the writer sits inside one of them - here inside a `JSON_OBJECT()` standing as the second argument. Merging makes the answer a level deeper than the deepest of its arguments and patching leaves it where it was, and both take that figure from what the arguments say rather than from a reading, which is what puts them there. The walk of a table gets a section of its own, where the writing is done once by the condition deciding whether to keep a row and once by an entry printed for it, those being different points in working a row out. The array has more than one element so that there is a row left to read after the write. Twenty cases, grouped by which argument does the writing, over a document with a nested object, an array and a string so that every one of them asserts an answer rather than `NULL` - `NULL` being what a freed read tends to produce as well. The width `JSON_DETAILED` indents by is read off the answer, so the case shows that the argument which frees the document really was worked out. The second section is the same package read the cheap way: once for the answer each gives, and once more with that answer made the document of something that keeps it across working out an argument which writes to the variable. That second group is where a function wrongly let out of the copy would show up. It has two `JSON_UNQUOTE` cases, and the character set on the variable is the whole of the difference between them. The answer is declared utf8mb4 whatever went in, so an argument in any other set is converted into a buffer belonging to the caller and no view of the variable goes anywhere; a variable declared utf8mb4 is passed through instead and the answer is a view of it. Take the careful read away and only the second stops the build, on `watch.unchanged(js)`. `func_json_variable_copy_oom` fails the copy itself. The copy is a write and can run short of room like any other, and no ordinary question reaches the arm that copes with that; what the arm has to do is refuse, rather than fall back on the view this commit exists to stop handing out. Reaching it means making the copy fail, and the only thing that can make it fail is an injection point a release build does not have, set through a variable a release build does not know - so it is a file of its own, sourcing `have_debug.inc`, rather than a section of `func_json_aliasing`, which asks nothing about how the server was built and runs wherever it was. `func_json_result_side` says which side of a select list entry a reference passes. A name repeated in `HAVING` or in `ORDER BY` is a reference, and once a temporary table has been built for the grouping or the sorting it reads a column of that table rather than the thing that made the value, so a store that shortened the value or wrote it in another character set is what every question about the value has to be put to. No query shape prints what a reference returned, so the outcomes are told apart by which side of a comparison the rows come back on: a value attested joins the array it is put in, and one that is not is written out as a string and quoted. A column comes back attested and a view column passes the question to the column underneath it, while a user variable assigned in the same select list does not - reading it back would perform the assignment again, and nothing keeps marks for one in any event. --- mysql-test/main/func_json_aliasing.result | 492 ++++++++++++++++++ mysql-test/main/func_json_aliasing.test | 412 +++++++++++++++ mysql-test/main/func_json_result_side.result | 171 ++++++ mysql-test/main/func_json_result_side.test | 130 +++++ .../main/func_json_variable_copy_oom.result | 36 ++ .../main/func_json_variable_copy_oom.test | 39 ++ sql/item.cc | 151 ++++++ sql/item.h | 82 ++- sql/item_func.h | 11 + sql/item_jsonfunc.cc | 64 ++- sql/item_jsonfunc.h | 2 + sql/json_table.cc | 10 +- 12 files changed, 1589 insertions(+), 11 deletions(-) create mode 100644 mysql-test/main/func_json_result_side.result create mode 100644 mysql-test/main/func_json_result_side.test create mode 100644 mysql-test/main/func_json_variable_copy_oom.result create mode 100644 mysql-test/main/func_json_variable_copy_oom.test diff --git a/mysql-test/main/func_json_aliasing.result b/mysql-test/main/func_json_aliasing.result index 5d8e44683174f..739cd743553f7 100644 --- a/mysql-test/main/func_json_aliasing.result +++ b/mysql-test/main/func_json_aliasing.result @@ -225,3 +225,495 @@ JSON_ARRAY((@d := REPEAT('z', 30)), 2)) AS typed_uvar; typed_uvar {"a": 1111111111, "b": ["zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 2]} DROP TABLE t1; +# +# 8. The document is a routine's variable, and working out +# another argument writes to that same variable +# +# Section 5 calls something to work out the value, but nothing +# it calls can reach the document: a routine's own variables +# are its own, and what it calls cannot see them. A package +# body's variables are shared by everything in the package, so +# there the expression working out another argument can assign +# to the variable the document is being read from, and storing +# a longer value there gives the variable a new buffer and +# lets go of the old one. +# +# The document is worked out before any of the other +# arguments, so the answer each of these owes is the one built +# from the value the variable held then, not from the value it +# holds by the time the answer is put together. +# +# Every argument is worked out just as late as every other, so +# this is not about values: a path reaches it, so does a +# string to search for, so does a width to indent by. The +# cases below are grouped by which argument does the writing. +# +SET @save_sql_mode= @@sql_mode; +SET sql_mode='ORACLE'; +CREATE PACKAGE pkg AS +PROCEDURE p; +PROCEDURE p1; +PROCEDURE p2; +PROCEDURE p3; +PROCEDURE p4; +END; +$$ +CREATE PACKAGE BODY pkg AS +d TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +/* +Section 10 has a variable of its own for each of its cases, and +writes each of them away exactly once. Both halves of that are +needed. A buffer is let go only where the value being stored will +not fit in it, so a variable that has already been written away +once is holding a buffer big enough for the next time and every +case after the first would reach nothing; and the rows these cases +select show the variable, so what is written away has to be put +back or a row would show one thing or another according to when it +was read. Putting it back settles nothing the cases are about - +the buffer the document was read from has gone by then. +*/ +e1 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +e2 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +e3 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +e4 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +/* +The variable section 11 uses. Its value comes from a function so +that the value is attested - a string written out by hand is +not - and from the function that writes it the plain way, because +a document not formatted that way is read back rather than handed +over and nothing is then said about how deep it is. The value is +deep, so that writing a shallow one over it makes the difference +the section is about. +*/ +g TEXT := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); +/* +The second JSON_UNQUOTE variable of section 9, and the character +set on it is the whole reason it exists. JSON_UNQUOTE passes its +argument straight on only where the argument is already in the set +the answer is declared to be in, and that set is utf8mb4 whatever +the argument was; a variable left to take the set the connection +is using is therefore converted into a buffer belonging to the +caller instead, and no view of it goes anywhere. Every other +variable here is left as it comes, so that the ordinary case is +what the rest of the file reads. +*/ +u TEXT CHARACTER SET utf8mb4 := '{"a":{"k":"zz"},"arr":[7,8]}'; +PROCEDURE reset AS +BEGIN +d := '{"a":{"k":"zz"},"arr":[7,8]}'; +END; +PROCEDURE grew AS +BEGIN +d := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +END; +FUNCTION grow RETURN INT AS +BEGIN +grew(); +RETURN 7; +END; +FUNCTION grow_path RETURN TEXT AS +BEGIN +grew(); +RETURN '$.a'; +END; +FUNCTION grow_scalar_path RETURN TEXT AS +BEGIN +grew(); +RETURN '$.arr[0]'; +END; +FUNCTION grow_needle RETURN TEXT AS +BEGIN +grew(); +RETURN 'zz'; +END; +PROCEDURE reset_u AS +BEGIN +u := '{"a":{"k":"zz"},"arr":[7,8]}'; +END; +FUNCTION grow_u RETURN INT AS +BEGIN +u := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +RETURN 7; +END; +FUNCTION grow1 RETURN INT AS +BEGIN +e1 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e1 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN 7; +END; +FUNCTION grow2_path RETURN TEXT AS +BEGIN +e2 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e2 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN '$.a'; +END; +FUNCTION grow3 RETURN INT AS +BEGIN +e3 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e3 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN 7; +END; +FUNCTION grow4 RETURN INT AS +BEGIN +e4 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e4 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN 7; +END; +PROCEDURE reset_g AS +BEGIN +g := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); +END; +FUNCTION flat RETURN INT AS +BEGIN +g := JSON_LOOSE('{"z":1}'); +RETURN 7; +END; +FUNCTION flat_path_rem RETURN TEXT AS +BEGIN +g := JSON_LOOSE('{"z":1}'); +RETURN '$.a.b.c[1]'; +END; +FUNCTION flat_path_qry RETURN TEXT AS +BEGIN +g := JSON_LOOSE('{"z":1}'); +RETURN '$.a'; +END; +PROCEDURE p AS +BEGIN +reset(); SELECT JSON_INSERT(d, '$.b', grow()) AS pkg_ins; +reset(); SELECT JSON_SET(d, '$.a', grow()) AS pkg_set; +reset(); SELECT JSON_REPLACE(d, '$.a', grow()) AS pkg_rep; +reset(); SELECT JSON_ARRAY_APPEND(d, '$.arr', grow()) AS pkg_app; +reset(); SELECT JSON_ARRAY_INSERT(d, '$.arr[0]', grow()) AS pkg_ari; +reset(); SELECT JSON_CONTAINS(d, CONCAT('', grow()), '$.arr') AS pkg_has; +reset(); SELECT JSON_REMOVE(d, grow_path()) AS pkg_rem; +reset(); SELECT JSON_EXISTS(d, grow_path()) AS pkg_exi; +reset(); SELECT JSON_EXTRACT(d, grow_path()) AS pkg_ext; +reset(); SELECT JSON_QUERY(d, grow_path()) AS pkg_qry; +reset(); SELECT JSON_VALUE(d, grow_scalar_path()) AS pkg_val; +reset(); SELECT JSON_CONTAINS_PATH(d, 'one', grow_path()) AS pkg_cpa; +reset(); SELECT JSON_LENGTH(d, grow_path()) AS pkg_len; +reset(); SELECT JSON_KEYS(d, grow_path()) AS pkg_key; +reset(); SELECT JSON_MERGE(d, JSON_OBJECT('b', grow())) AS pkg_mrg; +reset(); SELECT JSON_MERGE_PATCH(d, JSON_OBJECT('b', grow())) AS pkg_mpt; +reset(); SELECT JSON_EQUALS(d, JSON_OBJECT('b', grow())) AS pkg_eqs; +reset(); SELECT JSON_OVERLAPS(d, JSON_OBJECT('arr', +JSON_ARRAY(7, grow() + 1))) AS pkg_ovl; +reset(); SELECT JSON_SEARCH(d, 'one', grow_needle()) AS pkg_sea; +reset(); SELECT JSON_DETAILED(d, grow()) AS pkg_det; +END; +PROCEDURE p1 AS +BEGIN +reset(); SELECT JSON_VALID(d) AS pkg1_vld, JSON_TYPE(d) AS pkg1_typ, +JSON_DEPTH(d) AS pkg1_dep, JSON_LENGTH(d) AS pkg1_len; +reset(); SELECT JSON_KEYS(d) AS pkg1_key; +reset(); SELECT JSON_COMPACT(d) AS pkg1_cmp; +reset(); SELECT JSON_LOOSE(d) AS pkg1_lse; +reset(); SELECT JSON_DETAILED(d) AS pkg1_det; +reset(); SELECT JSON_UNQUOTE(d) AS pkg1_unq; +reset(); SELECT JSON_INSERT(JSON_KEYS(d), '$[2]', grow()) AS pkg1_key_kept; +reset(); SELECT JSON_INSERT(JSON_COMPACT(d), '$.b', grow()) AS pkg1_cmp_kept; +reset(); SELECT JSON_INSERT(JSON_LOOSE(d), '$.b', grow()) AS pkg1_lse_kept; +reset(); SELECT JSON_INSERT(JSON_DETAILED(d), '$.b', grow()) AS pkg1_det_kept; +reset(); SELECT JSON_INSERT(JSON_UNQUOTE(d), '$.b', grow()) AS pkg1_unq_kept; +reset_u(); SELECT JSON_INSERT(JSON_UNQUOTE(u), '$.b', grow_u()) +AS pkg1_unq4_kept; +END; +PROCEDURE p2 AS +BEGIN +SELECT id, e1 AS a FROM t2 +ORDER BY JSON_INSERT(a, '$.b', grow1() * id) DESC; +SELECT id, e2 AS a FROM t2 +ORDER BY JSON_REMOVE(a, grow2_path()) DESC, id; +SELECT id, e3 AS a FROM t2 +ORDER BY JSON_INSERT(JSON_KEYS(a), '$[2]', grow3() * id) DESC; +SELECT id, e4 AS a, COUNT(*) FROM t2 GROUP BY id, a +ORDER BY JSON_INSERT(a, '$.b', grow4() * id) DESC; +END; +PROCEDURE p3 AS +BEGIN +reset_g(); SELECT JSON_SET(g, '$.z', flat()) AS dep_set; +reset_g(); SELECT JSON_ARRAY_APPEND(g, '$.arr', flat()) AS dep_app; +reset_g(); SELECT JSON_ARRAY_INSERT(g, '$.arr[0]', flat()) AS dep_ari; +reset_g(); SELECT JSON_REMOVE(g, flat_path_rem()) AS dep_rem; +reset_g(); SELECT JSON_QUERY(g, flat_path_qry()) AS dep_qry; +reset_g(); SELECT JSON_MERGE(g, JSON_OBJECT('m', flat())) AS dep_mrg; +reset_g(); SELECT JSON_MERGE_PATCH(g, JSON_OBJECT('m', flat())) AS dep_mpt; +END; +PROCEDURE p4 AS +BEGIN +reset(); SELECT jt.v AS pkg4_where FROM +JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt +WHERE grow() = 7; +reset(); SELECT jt.v AS pkg4_list, grow() AS g FROM +JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt; +END; +END; +$$ +CALL pkg.p(); +pkg_ins +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg_set +{"a": 7, "arr": [7, 8]} +pkg_rep +{"a": 7, "arr": [7, 8]} +pkg_app +{"a": {"k": "zz"}, "arr": [7, 8, 7]} +pkg_ari +{"a": {"k": "zz"}, "arr": [7, 7, 8]} +pkg_has +1 +pkg_rem +{"arr": [7, 8]} +pkg_exi +1 +pkg_ext +{"k": "zz"} +pkg_qry +{"k":"zz"} +pkg_val +7 +pkg_cpa +1 +pkg_len +1 +pkg_key +["k"] +pkg_mrg +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg_mpt +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg_eqs +0 +pkg_ovl +1 +pkg_sea +"$.a.k" +pkg_det +{ + "a": + { + "k": "zz" + }, + "arr": + [ + 7, + 8 + ] +} +# +# 9. The document is a routine's variable, and the call has +# nothing left to work out after reading it +# +# A call given nothing but a document is finished with it +# before anything else can run, so it is answered with a +# view into the variable rather than with bytes of its own. +# The first group reads each of them that way. +# +# That holds only while none of the document comes back out +# of the call, so the second group makes each answer the +# document of something that keeps it across working out an +# argument that writes to the variable. JSON_UNQUOTE is +# among them because it passes its argument straight on when +# there is nothing to unquote, and is read the careful way +# for that reason. +# +# Straight on is not what it does with every argument, and +# the difference decides whether there is anything to be +# careful about. The answer is declared to be in one +# particular character set whatever went in, so an argument +# in some other set is converted into a buffer of the +# caller's own on the way out and the variable is left where +# it stands. Only an argument already in that set is the one +# passed on, and only then is the answer a view of the +# variable. So the group ends with the same case twice over, +# once on a variable that takes the connection's set and once +# on one declared in the answer's. +# +CALL pkg.p1(); +pkg1_vld pkg1_typ pkg1_dep pkg1_len +1 OBJECT 3 2 +pkg1_key +["a", "arr"] +pkg1_cmp +{"a":{"k":"zz"},"arr":[7,8]} +pkg1_lse +{"a": {"k": "zz"}, "arr": [7, 8]} +pkg1_det +{ + "a": + { + "k": "zz" + }, + "arr": + [ + 7, + 8 + ] +} +pkg1_unq +{"a":{"k":"zz"},"arr":[7,8]} +pkg1_key_kept +["a", "arr", 7] +pkg1_cmp_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_lse_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_det_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_unq_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_unq4_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +# +# 10. The document is reached through a reference standing in +# front of the thing that holds it +# +# An expression in an ORDER BY that names an entry of the +# select list does not get that entry. It gets a reference, +# and a reference passes the value on by asking the entry for +# it - so which of the two ways of asking it uses is the +# whole of what these cases are about. +# +# Asked for a plain string, a routine's variable answers with +# a view of itself, on the footing that whoever wanted a +# string may build over the buffer it was offered and must +# not be allowed to build over the variable. Asked for a +# document it answers with bytes of the caller's own, for the +# reason section 8 gives. A reference that puts the first +# question on behalf of a caller that asked the second lets +# the view out again, and section 8's hazard is back with a +# wrapper in front of it. +# +# A value writes the variable in the first case and a path in +# the second, for the reason section 8 gives: an argument is +# an argument, and which one it is decides nothing. +# +# The third asks for the document the cheap way. A call given +# nothing but a document is answered with a view, section 9's +# subject, and a reference has to pass that formatting on as +# the same formatting rather than quietly making it the other +# one; the answer is then kept across a write to show that +# what came back out is nobody's view of anything. +# +# The last case is the same question answered the other way. +# Once the rows are gathered into a temporary table the +# reference reads that table's column rather than the entry, +# and a column of a row that is being read is nobody's to +# free while it is being read - so there a plain string is +# the right thing to ask for and no copy is owed. +# +CREATE TABLE t2 (id INT); +INSERT INTO t2 VALUES (1), (2); +CALL pkg.p2(); +id a +1 {"a":{"k":"zz"},"arr":[7,8]} +2 {"a":{"k":"zz"},"arr":[7,8]} +id a +1 {"a":{"k":"zz"},"arr":[7,8]} +2 {"a":{"k":"zz"},"arr":[7,8]} +id a +1 {"a":{"k":"zz"},"arr":[7,8]} +2 {"a":{"k":"zz"},"arr":[7,8]} +id a COUNT(*) +1 {"a":{"k":"zz"},"arr":[7,8]} 1 +2 {"a":{"k":"zz"},"arr":[7,8]} 1 +DROP TABLE t2; +# +# 11. How deep the document goes, asked after the other arguments +# have been worked out +# +# A function that edits a document asks it three things. Two +# of them - whether it reads as a document at all, and whether +# it is written the plain way - are asked before anything is +# composed, because the answers are about the document that is +# about to be read. How deep it goes is the same kind of +# question and has to be asked at the same time. +# +# Asked at the far end it is a question about whatever the +# variable holds by then, and by then an argument has run. So +# these write a SHALLOWER document over the variable while the +# deeper one is being read: the answer returned is as deep +# as the document it was composed from, and the depth said +# about it would be the shallow one. +# +# Too small is the one direction a depth must never be wrong +# in. A caller splicing this answer into one of its own adds +# its own levels to what it is told, and a total that comes +# out under the limit on a document that is over it is how a +# document nothing can read back gets attested to. A debug +# build reads the answer and stops. +# +# This cannot be shown from any commit below the one that +# gives a variable's reader bytes of its own: without that, +# writing over the variable takes the document away as well as +# the depth, and what shows is section 8's fault instead. +# +# A value writes the variable in the first three and a path in +# the next two, the path being the argument in between for +# JSON_REMOVE and for JSON_QUERY. +# +# The last two are the pair whose answer is composed out of +# document arguments and nothing else, so there the variable is +# written by a value inside another document argument rather +# than by an argument of the function itself. Merging makes +# the answer a level deeper than the deepest of them and +# patching leaves it where it was, and both take the deepest +# from what the arguments say rather than from a reading, so +# both have to have asked while the arguments were still +# talking about the documents that went in. +# +CALL pkg.p3(); +dep_set +{"a": {"b": {"c": [1, 2]}}, "arr": [9], "z": 7} +dep_app +{"a": {"b": {"c": [1, 2]}}, "arr": [9, 7]} +dep_ari +{"a": {"b": {"c": [1, 2]}}, "arr": [7, 9]} +dep_rem +{"a": {"b": {"c": [1]}}, "arr": [9]} +dep_qry +{"b": {"c": [1, 2]}} +dep_mrg +{"a": {"b": {"c": [1, 2]}}, "arr": [9], "m": 7} +dep_mpt +{"a": {"b": {"c": [1, 2]}}, "arr": [9], "m": 7} +# +# 12. The document is walked a row at a time, with something +# else in the query writing to it between rows +# +# Everything above reads a document, works an answer out of +# it and is finished with it inside the one call. A table +# built out of a document is not: it is told where the +# document stands when the walk begins and reads on from +# there each time a row is asked for, so the note of where it +# stands outlives everything the query does between one row +# and the next. +# +# What the query does between rows is work out the rest of +# the row - the condition deciding whether to keep it, and +# the entries printed for it - and either of those can write +# to the variable the document was read from. So the first +# row is read from a document that is still there and every +# row after it from one that has been let go. +# +# Both places are shown because they are different points in +# working a row out, and the array has more than one element +# so that there is a row left to read after the write. The +# rows owed either way are the elements of the array the +# document held when the walk began. +# +CALL pkg.p4(); +pkg4_where +7 +8 +pkg4_list g +7 7 +8 7 +DROP PACKAGE pkg; +SET sql_mode= @save_sql_mode; +# +# The arm that copes with there being no room for the copy a +# variable is passed as is reached only by a debug build, +# and is in func_json_variable_copy_oom. +# diff --git a/mysql-test/main/func_json_aliasing.test b/mysql-test/main/func_json_aliasing.test index 521ac8ad902ce..87a6aea3930ea 100644 --- a/mysql-test/main/func_json_aliasing.test +++ b/mysql-test/main/func_json_aliasing.test @@ -171,3 +171,415 @@ SELECT JSON_INSERT(@d, '$.b', JSON_ARRAY((@d := REPEAT('z', 30)), 2)) AS typed_uvar; DROP TABLE t1; + +--echo # +--echo # 8. The document is a routine's variable, and working out +--echo # another argument writes to that same variable +--echo # +--echo # Section 5 calls something to work out the value, but nothing +--echo # it calls can reach the document: a routine's own variables +--echo # are its own, and what it calls cannot see them. A package +--echo # body's variables are shared by everything in the package, so +--echo # there the expression working out another argument can assign +--echo # to the variable the document is being read from, and storing +--echo # a longer value there gives the variable a new buffer and +--echo # lets go of the old one. +--echo # +--echo # The document is worked out before any of the other +--echo # arguments, so the answer each of these owes is the one built +--echo # from the value the variable held then, not from the value it +--echo # holds by the time the answer is put together. +--echo # +--echo # Every argument is worked out just as late as every other, so +--echo # this is not about values: a path reaches it, so does a +--echo # string to search for, so does a width to indent by. The +--echo # cases below are grouped by which argument does the writing. +--echo # +SET @save_sql_mode= @@sql_mode; +SET sql_mode='ORACLE'; + +--delimiter $$ +CREATE PACKAGE pkg AS + PROCEDURE p; + PROCEDURE p1; + PROCEDURE p2; + PROCEDURE p3; + PROCEDURE p4; +END; +$$ +CREATE PACKAGE BODY pkg AS + d TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + /* + Section 10 has a variable of its own for each of its cases, and + writes each of them away exactly once. Both halves of that are + needed. A buffer is let go only where the value being stored will + not fit in it, so a variable that has already been written away + once is holding a buffer big enough for the next time and every + case after the first would reach nothing; and the rows these cases + select show the variable, so what is written away has to be put + back or a row would show one thing or another according to when it + was read. Putting it back settles nothing the cases are about - + the buffer the document was read from has gone by then. + */ + e1 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + e2 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + e3 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + e4 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + /* + The variable section 11 uses. Its value comes from a function so + that the value is attested - a string written out by hand is + not - and from the function that writes it the plain way, because + a document not formatted that way is read back rather than handed + over and nothing is then said about how deep it is. The value is + deep, so that writing a shallow one over it makes the difference + the section is about. + */ + g TEXT := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); + /* + The second JSON_UNQUOTE variable of section 9, and the character + set on it is the whole reason it exists. JSON_UNQUOTE passes its + argument straight on only where the argument is already in the set + the answer is declared to be in, and that set is utf8mb4 whatever + the argument was; a variable left to take the set the connection + is using is therefore converted into a buffer belonging to the + caller instead, and no view of it goes anywhere. Every other + variable here is left as it comes, so that the ordinary case is + what the rest of the file reads. + */ + u TEXT CHARACTER SET utf8mb4 := '{"a":{"k":"zz"},"arr":[7,8]}'; + + PROCEDURE reset AS + BEGIN + d := '{"a":{"k":"zz"},"arr":[7,8]}'; + END; + + PROCEDURE grew AS + BEGIN + d := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + END; + + FUNCTION grow RETURN INT AS + BEGIN + grew(); + RETURN 7; + END; + + FUNCTION grow_path RETURN TEXT AS + BEGIN + grew(); + RETURN '$.a'; + END; + + FUNCTION grow_scalar_path RETURN TEXT AS + BEGIN + grew(); + RETURN '$.arr[0]'; + END; + + FUNCTION grow_needle RETURN TEXT AS + BEGIN + grew(); + RETURN 'zz'; + END; + + PROCEDURE reset_u AS + BEGIN + u := '{"a":{"k":"zz"},"arr":[7,8]}'; + END; + + FUNCTION grow_u RETURN INT AS + BEGIN + u := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + RETURN 7; + END; + + FUNCTION grow1 RETURN INT AS + BEGIN + e1 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e1 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN 7; + END; + + FUNCTION grow2_path RETURN TEXT AS + BEGIN + e2 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e2 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN '$.a'; + END; + + FUNCTION grow3 RETURN INT AS + BEGIN + e3 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e3 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN 7; + END; + + FUNCTION grow4 RETURN INT AS + BEGIN + e4 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e4 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN 7; + END; + + PROCEDURE reset_g AS + BEGIN + g := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); + END; + + FUNCTION flat RETURN INT AS + BEGIN + g := JSON_LOOSE('{"z":1}'); + RETURN 7; + END; + + FUNCTION flat_path_rem RETURN TEXT AS + BEGIN + g := JSON_LOOSE('{"z":1}'); + RETURN '$.a.b.c[1]'; + END; + + FUNCTION flat_path_qry RETURN TEXT AS + BEGIN + g := JSON_LOOSE('{"z":1}'); + RETURN '$.a'; + END; + + PROCEDURE p AS + BEGIN + reset(); SELECT JSON_INSERT(d, '$.b', grow()) AS pkg_ins; + reset(); SELECT JSON_SET(d, '$.a', grow()) AS pkg_set; + reset(); SELECT JSON_REPLACE(d, '$.a', grow()) AS pkg_rep; + reset(); SELECT JSON_ARRAY_APPEND(d, '$.arr', grow()) AS pkg_app; + reset(); SELECT JSON_ARRAY_INSERT(d, '$.arr[0]', grow()) AS pkg_ari; + reset(); SELECT JSON_CONTAINS(d, CONCAT('', grow()), '$.arr') AS pkg_has; + + reset(); SELECT JSON_REMOVE(d, grow_path()) AS pkg_rem; + reset(); SELECT JSON_EXISTS(d, grow_path()) AS pkg_exi; + reset(); SELECT JSON_EXTRACT(d, grow_path()) AS pkg_ext; + reset(); SELECT JSON_QUERY(d, grow_path()) AS pkg_qry; + reset(); SELECT JSON_VALUE(d, grow_scalar_path()) AS pkg_val; + reset(); SELECT JSON_CONTAINS_PATH(d, 'one', grow_path()) AS pkg_cpa; + reset(); SELECT JSON_LENGTH(d, grow_path()) AS pkg_len; + reset(); SELECT JSON_KEYS(d, grow_path()) AS pkg_key; + + reset(); SELECT JSON_MERGE(d, JSON_OBJECT('b', grow())) AS pkg_mrg; + reset(); SELECT JSON_MERGE_PATCH(d, JSON_OBJECT('b', grow())) AS pkg_mpt; + reset(); SELECT JSON_EQUALS(d, JSON_OBJECT('b', grow())) AS pkg_eqs; + reset(); SELECT JSON_OVERLAPS(d, JSON_OBJECT('arr', + JSON_ARRAY(7, grow() + 1))) AS pkg_ovl; + + reset(); SELECT JSON_SEARCH(d, 'one', grow_needle()) AS pkg_sea; + reset(); SELECT JSON_DETAILED(d, grow()) AS pkg_det; + END; + + PROCEDURE p1 AS + BEGIN + reset(); SELECT JSON_VALID(d) AS pkg1_vld, JSON_TYPE(d) AS pkg1_typ, + JSON_DEPTH(d) AS pkg1_dep, JSON_LENGTH(d) AS pkg1_len; + reset(); SELECT JSON_KEYS(d) AS pkg1_key; + reset(); SELECT JSON_COMPACT(d) AS pkg1_cmp; + reset(); SELECT JSON_LOOSE(d) AS pkg1_lse; + reset(); SELECT JSON_DETAILED(d) AS pkg1_det; + reset(); SELECT JSON_UNQUOTE(d) AS pkg1_unq; + + reset(); SELECT JSON_INSERT(JSON_KEYS(d), '$[2]', grow()) AS pkg1_key_kept; + reset(); SELECT JSON_INSERT(JSON_COMPACT(d), '$.b', grow()) AS pkg1_cmp_kept; + reset(); SELECT JSON_INSERT(JSON_LOOSE(d), '$.b', grow()) AS pkg1_lse_kept; + reset(); SELECT JSON_INSERT(JSON_DETAILED(d), '$.b', grow()) AS pkg1_det_kept; + reset(); SELECT JSON_INSERT(JSON_UNQUOTE(d), '$.b', grow()) AS pkg1_unq_kept; + reset_u(); SELECT JSON_INSERT(JSON_UNQUOTE(u), '$.b', grow_u()) + AS pkg1_unq4_kept; + END; + + PROCEDURE p2 AS + BEGIN + SELECT id, e1 AS a FROM t2 + ORDER BY JSON_INSERT(a, '$.b', grow1() * id) DESC; + SELECT id, e2 AS a FROM t2 + ORDER BY JSON_REMOVE(a, grow2_path()) DESC, id; + SELECT id, e3 AS a FROM t2 + ORDER BY JSON_INSERT(JSON_KEYS(a), '$[2]', grow3() * id) DESC; + SELECT id, e4 AS a, COUNT(*) FROM t2 GROUP BY id, a + ORDER BY JSON_INSERT(a, '$.b', grow4() * id) DESC; + END; + + PROCEDURE p3 AS + BEGIN + reset_g(); SELECT JSON_SET(g, '$.z', flat()) AS dep_set; + reset_g(); SELECT JSON_ARRAY_APPEND(g, '$.arr', flat()) AS dep_app; + reset_g(); SELECT JSON_ARRAY_INSERT(g, '$.arr[0]', flat()) AS dep_ari; + reset_g(); SELECT JSON_REMOVE(g, flat_path_rem()) AS dep_rem; + reset_g(); SELECT JSON_QUERY(g, flat_path_qry()) AS dep_qry; + reset_g(); SELECT JSON_MERGE(g, JSON_OBJECT('m', flat())) AS dep_mrg; + reset_g(); SELECT JSON_MERGE_PATCH(g, JSON_OBJECT('m', flat())) AS dep_mpt; + END; + + PROCEDURE p4 AS + BEGIN + reset(); SELECT jt.v AS pkg4_where FROM + JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt + WHERE grow() = 7; + reset(); SELECT jt.v AS pkg4_list, grow() AS g FROM + JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt; + END; +END; +$$ +--delimiter ; + +CALL pkg.p(); + +--echo # +--echo # 9. The document is a routine's variable, and the call has +--echo # nothing left to work out after reading it +--echo # +--echo # A call given nothing but a document is finished with it +--echo # before anything else can run, so it is answered with a +--echo # view into the variable rather than with bytes of its own. +--echo # The first group reads each of them that way. +--echo # +--echo # That holds only while none of the document comes back out +--echo # of the call, so the second group makes each answer the +--echo # document of something that keeps it across working out an +--echo # argument that writes to the variable. JSON_UNQUOTE is +--echo # among them because it passes its argument straight on when +--echo # there is nothing to unquote, and is read the careful way +--echo # for that reason. +--echo # +--echo # Straight on is not what it does with every argument, and +--echo # the difference decides whether there is anything to be +--echo # careful about. The answer is declared to be in one +--echo # particular character set whatever went in, so an argument +--echo # in some other set is converted into a buffer of the +--echo # caller's own on the way out and the variable is left where +--echo # it stands. Only an argument already in that set is the one +--echo # passed on, and only then is the answer a view of the +--echo # variable. So the group ends with the same case twice over, +--echo # once on a variable that takes the connection's set and once +--echo # on one declared in the answer's. +--echo # +CALL pkg.p1(); + +--echo # +--echo # 10. The document is reached through a reference standing in +--echo # front of the thing that holds it +--echo # +--echo # An expression in an ORDER BY that names an entry of the +--echo # select list does not get that entry. It gets a reference, +--echo # and a reference passes the value on by asking the entry for +--echo # it - so which of the two ways of asking it uses is the +--echo # whole of what these cases are about. +--echo # +--echo # Asked for a plain string, a routine's variable answers with +--echo # a view of itself, on the footing that whoever wanted a +--echo # string may build over the buffer it was offered and must +--echo # not be allowed to build over the variable. Asked for a +--echo # document it answers with bytes of the caller's own, for the +--echo # reason section 8 gives. A reference that puts the first +--echo # question on behalf of a caller that asked the second lets +--echo # the view out again, and section 8's hazard is back with a +--echo # wrapper in front of it. +--echo # +--echo # A value writes the variable in the first case and a path in +--echo # the second, for the reason section 8 gives: an argument is +--echo # an argument, and which one it is decides nothing. +--echo # +--echo # The third asks for the document the cheap way. A call given +--echo # nothing but a document is answered with a view, section 9's +--echo # subject, and a reference has to pass that formatting on as +--echo # the same formatting rather than quietly making it the other +--echo # one; the answer is then kept across a write to show that +--echo # what came back out is nobody's view of anything. +--echo # +--echo # The last case is the same question answered the other way. +--echo # Once the rows are gathered into a temporary table the +--echo # reference reads that table's column rather than the entry, +--echo # and a column of a row that is being read is nobody's to +--echo # free while it is being read - so there a plain string is +--echo # the right thing to ask for and no copy is owed. +--echo # +CREATE TABLE t2 (id INT); +INSERT INTO t2 VALUES (1), (2); +CALL pkg.p2(); +DROP TABLE t2; + +--echo # +--echo # 11. How deep the document goes, asked after the other arguments +--echo # have been worked out +--echo # +--echo # A function that edits a document asks it three things. Two +--echo # of them - whether it reads as a document at all, and whether +--echo # it is written the plain way - are asked before anything is +--echo # composed, because the answers are about the document that is +--echo # about to be read. How deep it goes is the same kind of +--echo # question and has to be asked at the same time. +--echo # +--echo # Asked at the far end it is a question about whatever the +--echo # variable holds by then, and by then an argument has run. So +--echo # these write a SHALLOWER document over the variable while the +--echo # deeper one is being read: the answer returned is as deep +--echo # as the document it was composed from, and the depth said +--echo # about it would be the shallow one. +--echo # +--echo # Too small is the one direction a depth must never be wrong +--echo # in. A caller splicing this answer into one of its own adds +--echo # its own levels to what it is told, and a total that comes +--echo # out under the limit on a document that is over it is how a +--echo # document nothing can read back gets attested to. A debug +--echo # build reads the answer and stops. +--echo # +--echo # This cannot be shown from any commit below the one that +--echo # gives a variable's reader bytes of its own: without that, +--echo # writing over the variable takes the document away as well as +--echo # the depth, and what shows is section 8's fault instead. +--echo # +--echo # A value writes the variable in the first three and a path in +--echo # the next two, the path being the argument in between for +--echo # JSON_REMOVE and for JSON_QUERY. +--echo # +--echo # The last two are the pair whose answer is composed out of +--echo # document arguments and nothing else, so there the variable is +--echo # written by a value inside another document argument rather +--echo # than by an argument of the function itself. Merging makes +--echo # the answer a level deeper than the deepest of them and +--echo # patching leaves it where it was, and both take the deepest +--echo # from what the arguments say rather than from a reading, so +--echo # both have to have asked while the arguments were still +--echo # talking about the documents that went in. +--echo # +CALL pkg.p3(); + +--echo # +--echo # 12. The document is walked a row at a time, with something +--echo # else in the query writing to it between rows +--echo # +--echo # Everything above reads a document, works an answer out of +--echo # it and is finished with it inside the one call. A table +--echo # built out of a document is not: it is told where the +--echo # document stands when the walk begins and reads on from +--echo # there each time a row is asked for, so the note of where it +--echo # stands outlives everything the query does between one row +--echo # and the next. +--echo # +--echo # What the query does between rows is work out the rest of +--echo # the row - the condition deciding whether to keep it, and +--echo # the entries printed for it - and either of those can write +--echo # to the variable the document was read from. So the first +--echo # row is read from a document that is still there and every +--echo # row after it from one that has been let go. +--echo # +--echo # Both places are shown because they are different points in +--echo # working a row out, and the array has more than one element +--echo # so that there is a row left to read after the write. The +--echo # rows owed either way are the elements of the array the +--echo # document held when the walk began. +--echo # +CALL pkg.p4(); + +DROP PACKAGE pkg; +SET sql_mode= @save_sql_mode; + +--echo # +--echo # The arm that copes with there being no room for the copy a +--echo # variable is passed as is reached only by a debug build, +--echo # and is in func_json_variable_copy_oom. +--echo # diff --git a/mysql-test/main/func_json_result_side.result b/mysql-test/main/func_json_result_side.result new file mode 100644 index 0000000000000..4f63109d243de --- /dev/null +++ b/mysql-test/main/func_json_result_side.result @@ -0,0 +1,171 @@ +# +# What a reference passes when a document is asked of it. +# +# A name written in HAVING or in ORDER BY that repeats a select +# list entry does not hold a value of its own. It is a reference, +# and it reads the entry's RESULT side - which, once a temporary +# table has been built for the grouping or the sorting, is a +# column of that table rather than the thing that made the value. +# +# The two sides can disagree, which is the whole reason they are +# asked separately: a store into the column can shorten a value or +# write it in another character set, so what the column holds may +# be a document where the producer's answer was about something +# else, or may be nothing of the sort where the producer's was. +# Every question a JSON function asks of an argument - is it a +# document, is it written the loose way, how deep does it go - is +# therefore put to the same side the value itself came from. +# +# A user variable assigned in the select list is the case where +# the answer must be no. Reading it back performs the assignment, +# so the value cannot be asked about without being made again, and +# nothing keeps marks for a user variable in any event. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'), (2, '{"a":2,"b":[3,4]}'); +CREATE VIEW v1 AS SELECT id AS vid, j AS vj FROM t1; +# +# 1. A column read through a reference +# +# One argument is passed as a view of what is already +# there and several are each given a copy, so both formats +# of the read are put through a reference here. +# +SELECT j AS a FROM t1 ORDER BY JSON_COMPACT(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 ORDER BY JSON_LOOSE(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a ORDER BY JSON_EXTRACT(a, '$.a'); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_EXTRACT(a, '$.a') > 1; +a +{"a":2,"b":[3,4]} +# +# 2. The same value put inside another document +# +# Splicing asks how the value is formatted as well as whether it +# is a document, and it is the reference that is asked. +# +SELECT j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_INSERT('{}','$.x',a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a +ORDER BY JSON_ARRAY_APPEND('{"c":[1]}', '$.c', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +# +# 3. A view's column read through a reference +# +# A view column is itself a reference, and it hands the +# question on rather than answering it. +# +SELECT vj AS a FROM v1 ORDER BY JSON_COMPACT(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT vj AS a FROM v1 ORDER BY JSON_INSERT('{}', '$.x', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT vj AS a FROM v1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT JSON_TYPE(vj) AS ty FROM v1 GROUP BY vj; +ty +OBJECT +OBJECT +# +# 4. A user variable read through a reference +# +SELECT @v := j AS a FROM t1 ORDER BY JSON_COMPACT(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT @w := j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT @u := j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +# +# 5. Which of them is attested, shown by the answer itself +# +# A value attested is put inside the array it is joining; +# one that is not is written out as a string and quoted. No +# query shape prints what a reference passed, so the two +# outcomes are told apart by whether the row is there. +# +# The column is attested, so its rows come back under +# OBJECT and none comes back under STRING. +# +SELECT j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; +a +# The user variable is not, so the rows change sides. +SELECT @u := j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +a +SELECT @u := j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +# A view column is attested, the view passing the question +# on to the column underneath it. +SELECT vj AS a FROM v1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT vj AS a FROM v1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; +a +# +# 6. The reading that is not done +# +# Json_scans is the only place the saved reading shows. The +# query below reads a document back out of a temporary table +# and puts it inside a new one; with the column's answer taken +# away the same query reads that document again. +# +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; +DROP VIEW v1; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_result_side.test b/mysql-test/main/func_json_result_side.test new file mode 100644 index 0000000000000..97fe00fd9c832 --- /dev/null +++ b/mysql-test/main/func_json_result_side.test @@ -0,0 +1,130 @@ +--source include/have_debug.inc + +--echo # +--echo # What a reference passes when a document is asked of it. +--echo # +--echo # A name written in HAVING or in ORDER BY that repeats a select +--echo # list entry does not hold a value of its own. It is a reference, +--echo # and it reads the entry's RESULT side - which, once a temporary +--echo # table has been built for the grouping or the sorting, is a +--echo # column of that table rather than the thing that made the value. +--echo # +--echo # The two sides can disagree, which is the whole reason they are +--echo # asked separately: a store into the column can shorten a value or +--echo # write it in another character set, so what the column holds may +--echo # be a document where the producer's answer was about something +--echo # else, or may be nothing of the sort where the producer's was. +--echo # Every question a JSON function asks of an argument - is it a +--echo # document, is it written the loose way, how deep does it go - is +--echo # therefore put to the same side the value itself came from. +--echo # +--echo # A user variable assigned in the select list is the case where +--echo # the answer must be no. Reading it back performs the assignment, +--echo # so the value cannot be asked about without being made again, and +--echo # nothing keeps marks for a user variable in any event. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'), (2, '{"a":2,"b":[3,4]}'); + +CREATE VIEW v1 AS SELECT id AS vid, j AS vj FROM t1; + +--echo # +--echo # 1. A column read through a reference +--echo # +--echo # One argument is passed as a view of what is already +--echo # there and several are each given a copy, so both formats +--echo # of the read are put through a reference here. +--echo # +SELECT j AS a FROM t1 ORDER BY JSON_COMPACT(a); +SELECT j AS a FROM t1 ORDER BY JSON_LOOSE(a); +SELECT j AS a FROM t1 GROUP BY a ORDER BY JSON_EXTRACT(a, '$.a'); +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_EXTRACT(a, '$.a') > 1; + +--echo # +--echo # 2. The same value put inside another document +--echo # +--echo # Splicing asks how the value is formatted as well as whether it +--echo # is a document, and it is the reference that is asked. +--echo # +SELECT j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_INSERT('{}','$.x',a)); +SELECT j AS a FROM t1 GROUP BY a + ORDER BY JSON_ARRAY_APPEND('{"c":[1]}', '$.c', a); + +--echo # +--echo # 3. A view's column read through a reference +--echo # +--echo # A view column is itself a reference, and it hands the +--echo # question on rather than answering it. +--echo # +SELECT vj AS a FROM v1 ORDER BY JSON_COMPACT(a); +SELECT vj AS a FROM v1 ORDER BY JSON_INSERT('{}', '$.x', a); +SELECT vj AS a FROM v1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +SELECT JSON_TYPE(vj) AS ty FROM v1 GROUP BY vj; + +--echo # +--echo # 4. A user variable read through a reference +--echo # +SELECT @v := j AS a FROM t1 ORDER BY JSON_COMPACT(a); +SELECT @w := j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +SELECT @u := j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); + +--echo # +--echo # 5. Which of them is attested, shown by the answer itself +--echo # +--echo # A value attested is put inside the array it is joining; +--echo # one that is not is written out as a string and quoted. No +--echo # query shape prints what a reference passed, so the two +--echo # outcomes are told apart by whether the row is there. +--echo # +--echo # The column is attested, so its rows come back under +--echo # OBJECT and none comes back under STRING. +--echo # +SELECT j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +SELECT j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; + +--echo # The user variable is not, so the rows change sides. +SELECT @u := j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +SELECT @u := j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; + +--echo # A view column is attested, the view passing the question +--echo # on to the column underneath it. +SELECT vj AS a FROM v1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +SELECT vj AS a FROM v1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; + +--echo # +--echo # 6. The reading that is not done +--echo # +--echo # Json_scans is the only place the saved reading shows. The +--echo # query below reads a document back out of a temporary table +--echo # and puts it inside a new one; with the column's answer taken +--echo # away the same query reads that document again. +--echo # +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; + +DROP VIEW v1; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_variable_copy_oom.result b/mysql-test/main/func_json_variable_copy_oom.result new file mode 100644 index 0000000000000..14d071081aab1 --- /dev/null +++ b/mysql-test/main/func_json_variable_copy_oom.result @@ -0,0 +1,36 @@ +# +# No room for the copy made of a variable's value. +# +# What a function is given is a copy and not a view, so that the +# bytes it works on cannot be written over while it is working on +# them. Making that copy is a write, and a write can run short of +# room. There is no ordinary question that reaches the arm which +# copes with that, so it is reached here, and what it must do is +# refuse rather than return the view it was avoiding. +# +# The calls the copy is made for, and everything else about +# reading a document from a variable while the variable is being +# written, are in func_json_aliasing. +# +CREATE PROCEDURE p_variable_copy() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v = JSON_OBJECT('a', 1, 'b', 2); +SELECT JSON_INSERT(v, '$.c', 3) AS added, +JSON_REMOVE(v, '$.b') AS removed, +JSON_ARRAY_APPEND(v, '$', 9) AS appended; +END $$ +CALL p_variable_copy(); +added removed appended +{"a": 1, "b": 2, "c": 3} {"a": 1} [{"a": 1, "b": 2}, 9] +SET SESSION debug_dbug = '+d,json_variable_copy_out_of_memory'; +CALL p_variable_copy(); +added removed appended +NULL NULL NULL +SET SESSION debug_dbug = DEFAULT; +# and again with the room there, which is what says the refusals +# are about the copy rather than about the calls +CALL p_variable_copy(); +added removed appended +{"a": 1, "b": 2, "c": 3} {"a": 1} [{"a": 1, "b": 2}, 9] +DROP PROCEDURE p_variable_copy; diff --git a/mysql-test/main/func_json_variable_copy_oom.test b/mysql-test/main/func_json_variable_copy_oom.test new file mode 100644 index 0000000000000..b22c7ad3e0df5 --- /dev/null +++ b/mysql-test/main/func_json_variable_copy_oom.test @@ -0,0 +1,39 @@ +--source include/have_debug.inc + +--echo # +--echo # No room for the copy made of a variable's value. +--echo # +--echo # What a function is given is a copy and not a view, so that the +--echo # bytes it works on cannot be written over while it is working on +--echo # them. Making that copy is a write, and a write can run short of +--echo # room. There is no ordinary question that reaches the arm which +--echo # copes with that, so it is reached here, and what it must do is +--echo # refuse rather than return the view it was avoiding. +--echo # +--echo # The calls the copy is made for, and everything else about +--echo # reading a document from a variable while the variable is being +--echo # written, are in func_json_aliasing. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_variable_copy() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v = JSON_OBJECT('a', 1, 'b', 2); + SELECT JSON_INSERT(v, '$.c', 3) AS added, + JSON_REMOVE(v, '$.b') AS removed, + JSON_ARRAY_APPEND(v, '$', 9) AS appended; +END $$ +--delimiter ; + +CALL p_variable_copy(); + +SET SESSION debug_dbug = '+d,json_variable_copy_out_of_memory'; +CALL p_variable_copy(); +SET SESSION debug_dbug = DEFAULT; + +--echo # and again with the room there, which is what says the refusals +--echo # are about the copy rather than about the calls +CALL p_variable_copy(); + +DROP PROCEDURE p_variable_copy; diff --git a/sql/item.cc b/sql/item.cc index 7369c9bccb9e2..abd9697cf2ed1 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1739,6 +1739,78 @@ String *Item_sp_variable::val_str(String *sp) } +/* + A JSON function keeps its document for the whole of its work. It + walks the document to find the place it was asked about, then works + out the rest of its arguments, and only then reads the pieces of the + document either side of that place. Working out an argument runs + whatever the caller wrote, and what the caller wrote can assign to + this very variable - the variables of a package body being in reach + of every routine in that package. Storing a longer value there + gives the variable a new buffer and lets go of the old one, which is + the buffer being read from. + + val_str() above returns a pointer INTO the variable rather than + bytes of the caller's own, and does so on purpose: a function that + builds into the buffer it is offered must not get to build into the + variable. What that leaves the caller holding is a view, good only + for as long as nothing writes where it points, so a caller that + means to keep it across working something else out cannot be given + one. Give that caller a copy. + + A user variable has always been read this way - user_var_entry:: + val_str() copies - which is why the same shape written with one of + those holds together. +*/ + +String *Item_sp_variable::val_json(String *str) +{ + DBUG_ASSERT(fixed()); + Item *it= this_item(); + String *res= it->val_json(str); + + null_value= it->null_value; + + /* + Nothing to copy, or worked out into the buffer that was offered + instead of read off the variable, in which case the bytes are the + caller's own already. + */ + if (!res || res == str) + return res; + + if (str->copy(res->ptr(), res->length(), res->charset()) || + DBUG_IF("json_variable_copy_out_of_memory")) + { + null_value= true; /* Out of memory. */ + return NULL; + } + + return str; +} + + +/* + A view into the variable is what val_str() above returns, and it + is enough for a caller that reads the bytes and is done with them + before anything else can run: nothing can write where they point + while nothing is running. So this is the one JSON caller that need + not be given a copy. See val_json() above for the caller that must + be. +*/ + +String *Item_sp_variable::val_json_at_once(String *str) +{ + DBUG_ASSERT(fixed()); + Item *it= this_item(); + String *res= it->val_json(str); + + null_value= it->null_value; + + return res; +} + + bool Item_sp_variable::val_native(THD *thd, Native *to) { return val_native_from_item(thd, this_item(), to); @@ -5558,6 +5630,24 @@ String* Item_ref_null_helper::val_str(String* s) } +String* Item_ref_null_helper::val_json(String* s) +{ + DBUG_ASSERT(fixed()); + String* tmp= (*ref)->val_json_result(s); + owner->was_null|= null_value= (*ref)->null_value; + return tmp; +} + + +String* Item_ref_null_helper::val_json_at_once(String* s) +{ + DBUG_ASSERT(fixed()); + String* tmp= (*ref)->val_json_at_once_result(s); + owner->was_null|= null_value= (*ref)->null_value; + return tmp; +} + + bool Item_ref_null_helper::val_native(THD *thd, Native *to) { return (owner->was_null|= val_native_from_item(thd, *ref, to)); @@ -9154,6 +9244,33 @@ String *Item_ref::val_str(String* tmp) } +/* + The two below are written for a caller that wants a document, reaching + the same side of the referenced item that val_str() does. Without + them Item::val_json() would send the request back through val_str(), + and an item that answers those two differently - a routine's variable + returns a view of itself from one and a copy from the other - would + have the difference thrown away by the reference in front of it. +*/ + +String *Item_ref::val_json(String* tmp) +{ + DBUG_ASSERT(fixed()); + tmp=(*ref)->val_json_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + +String *Item_ref::val_json_at_once(String* tmp) +{ + DBUG_ASSERT(fixed()); + tmp=(*ref)->val_json_at_once_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + bool Item_ref::is_null() { DBUG_ASSERT(fixed()); @@ -9302,6 +9419,24 @@ String *Item_direct_ref::val_str(String* tmp) } +/* The value side, val_str() above being of the value side here. */ + +String *Item_direct_ref::val_json(String* tmp) +{ + tmp=(*ref)->val_json(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + +String *Item_direct_ref::val_json_at_once(String* tmp) +{ + tmp=(*ref)->val_json_at_once(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + my_decimal *Item_direct_ref::val_decimal(my_decimal *decimal_value) { my_decimal *tmp= (*ref)->val_decimal(decimal_value); @@ -10146,6 +10281,22 @@ String *Item_direct_view_ref::str_result(String* tmp) } +String *Item_direct_view_ref::val_json_result(String* tmp) +{ + tmp=(*ref)->val_json_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + +String *Item_direct_view_ref::val_json_at_once_result(String* tmp) +{ + tmp=(*ref)->val_json_at_once_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + my_decimal *Item_direct_view_ref::val_decimal_result(my_decimal *val) { my_decimal *tmp= (*ref)->val_decimal_result(val); diff --git a/sql/item.h b/sql/item.h index adee5db9194aa..3ad5ab8b4b8e5 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1783,6 +1783,41 @@ class Item :public Value_source, virtual String *val_json(String *str) { return val_str(str); } + /* + The same value, for a caller that is finished with it before + anything else can run. + + val_json() may have to return bytes of the caller's own so that + they outlive whatever the caller does next. A caller that does + nothing next has no use for them, and this variant lets the item + say so cheaply. + + Only ask for it when BOTH hold: nothing else is worked out between + this call and the last read of what it returns, and no part of + what it returns reaches the caller's own caller. If either + fails, val_json() is what is wanted. Answering with val_json() is + always allowed, and is what an item that has no cheaper answer + does. + */ + virtual String *val_json_at_once(String *str) { return val_json(str); } + + /* + The same two of the result side, which is what a reference reads - + Item_ref::val_str() calls str_result(), so Item_ref::val_json() + calls these. Where an item has no result side of its own the two + sides are the same object and the defaults here say so; the five + that override str_result() attest to whatever their own + str_result() reads, exactly as they do for the marks below. + + Without them a reference would answer a request for a document by + falling back to Item::val_json(), which is val_str() - and an item + that hands out a view of somebody else's bytes from val_str() and a + copy from val_json() would be back to handing out the view. + */ + virtual String *val_json_result(String *str) { return val_json(str); } + virtual String *val_json_at_once_result(String *str) + { return val_json_at_once(str); } + /* Whether the value returned by the LAST evaluation of this item is guaranteed to be a JSON document, written in the character set the @@ -3354,6 +3389,8 @@ class Item_sp_variable :public Item_fixed_hybrid double val_real() override; longlong val_int() override; String *val_str(String *sp) override; + String *val_json(String *str) override; + String *val_json_at_once(String *str) override; my_decimal *val_decimal(my_decimal *decimal_value) override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; bool val_native(THD *thd, Native *to) override; @@ -4158,6 +4195,18 @@ class Item_field :public Item_ident, DBUG_ASSERT(reads_back_no_deeper_than_claimed()); return field->attested_json_depth(); } + /* + A field has no document of its own to pass - val_json() here is + Item::val_json(), which is val_str() - so its result side is + str_result() and nothing more. The bytes it reads are a row's, and + a row outlives the working out of an argument, so there is nothing + for val_json_at_once() to be careful about, and the two are one + call. Item_default_value inherits both: str_result() is virtual, + so its own is the one reached and the value gets put in place first. + */ + String *val_json_result(String *str) override { return str_result(str); } + String *val_json_at_once_result(String *str) override + { return str_result(str); } /* str_result() reads result_field rather than field, so these read result_field too. Which of the two an Item_field is asked for is @@ -6227,6 +6276,8 @@ class Item_ref :public Item_ident my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; String *val_str(String* tmp) override; + String *val_json(String* tmp) override; + String *val_json_at_once(String* tmp) override; bool val_native(THD *thd, Native *to) override; bool is_null() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; @@ -6259,6 +6310,10 @@ class Item_ref :public Item_ident rather than about the class, which is why the condition is written out here rather than answered once and for all somewhere. */ + String *val_json_result(String* tmp) override + { return result_field ? str_result(tmp) : val_json(tmp); } + String *val_json_at_once_result(String* tmp) override + { return result_field ? str_result(tmp) : val_json_at_once(tmp); } bool is_valid_json_result() const override { return result_field ? result_field->attests_is_valid_json() @@ -6489,6 +6544,8 @@ class Item_direct_ref :public Item_ref my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; String *val_str(String* tmp) override; + String *val_json(String* tmp) override; + String *val_json_at_once(String* tmp) override; bool val_native(THD *thd, Native *to) override; bool is_null() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; @@ -6805,8 +6862,10 @@ class Item_direct_view_ref :public Item_direct_ref /* str_result() below passes on whatever the referenced item's result side gave, and does not look at the row the way the val_XXX above - it do, so neither does this. + it do, so neither do these. */ + String *val_json_result(String* tmp) override; + String *val_json_at_once_result(String* tmp) override; bool is_valid_json_result() const override { return (*ref)->is_valid_json_result(); } bool is_nice_json_result() const override @@ -6842,6 +6901,20 @@ class Item_direct_view_ref :public Item_direct_ref else return Item_direct_ref::val_str(tmp); } + String *val_json(String* tmp) override + { + if (check_null_ref()) + return NULL; + else + return Item_direct_ref::val_json(tmp); + } + String *val_json_at_once(String* tmp) override + { + if (check_null_ref()) + return NULL; + else + return Item_direct_ref::val_json_at_once(tmp); + } bool val_native(THD *thd, Native *to) override { if (check_null_ref()) @@ -7044,6 +7117,13 @@ class Item_ref_null_helper: public Item_ref double val_real() override; longlong val_int() override; String* val_str(String* s) override; + /* + Every val_XXX here notes a NULL down for the owner, so these have to + as well: they are how val_str() used to be reached, and a NULL that + goes unnoted is a subquery answering the wrong way. + */ + String* val_json(String* s) override; + String* val_json_at_once(String* s) override; my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; diff --git a/sql/item_func.h b/sql/item_func.h index 73330d0820154..bbb95e61d4f3a 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -3602,6 +3602,17 @@ class Item_func_set_user_var :public Item_func_user_var String *str_result(String *str) override; my_decimal *val_decimal_result(my_decimal *) override; bool is_null_result() override; + /* + Reading the document out of str_result() is what happens today and + is what has to keep happening: the assignment it performs is this + item's whole point, and a document must not be fetched by any road + that performs it a second time or not at all. A user variable is + read out by copying, so what comes back is the caller's own bytes + and val_json_at_once() has nothing to save. + */ + String *val_json_result(String *str) override { return str_result(str); } + String *val_json_at_once_result(String *str) override + { return str_result(str); } /* str_result() above is the one of the five that does not merely read somewhere else: it assigns the variable and then returns what is diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 53013cfb3ca39..056d299f0affb 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -1289,6 +1289,34 @@ static void report_bad_chr_note(const char *fname, int n_arg) } +/* + The document argument of a call that has no other argument to work + out after it. + + Such a call reads the document and is finished with it before + anything else can run, so it can be answered with a view rather than + with bytes of its own - see Item::val_json_at_once(). A call that + does have another argument to work out cannot: working one out runs + whatever the caller wrote, and what the caller wrote can take the + document away. + + Asked of the argument count rather than written into each function, + so that a function which gains an optional argument later goes back + to being answered the careful way without anyone having to remember + to say so. + + Only for a function that returns nothing of the document: one + that passes it straight on, as JSON_UNQUOTE and JSON_COMPACT do when + there is nothing to unquote or to rewrite, gives its own caller a + view and has to read the document with val_json(). +*/ + +static inline String *val_json_arg0(Item **args, uint arg_count, String *str) +{ + return arg_count == 1 ? args[0]->val_json_at_once(str) + : args[0]->val_json(str); +} + #define NO_WILDCARD_ALLOWED 1 #define SHOULD_END_WITH_ARRAY 2 @@ -1378,7 +1406,7 @@ CHARSET_INFO *def_path_charset(CHARSET_INFO *cs, CHARSET_INFO *alt) bool Item_func_json_valid::val_bool() { - String *js= args[0]->val_json(&tmp_value); + String *js= val_json_arg0(args, arg_count, &tmp_value); THD *thd; json_engine_t je; @@ -5489,7 +5517,7 @@ bool Item_func_json_length::fix_length_and_dec(THD *thd) longlong Item_func_json_length::val_int() { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; Json_source_watch watch; uint length= 0; @@ -5588,7 +5616,7 @@ longlong Item_func_json_length::val_int() longlong Item_func_json_depth::val_int() { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; uint depth= 0, c_depth= 0; bool inc_depth= TRUE; @@ -5653,7 +5681,7 @@ bool Item_func_json_type::fix_length_and_dec(THD *thd) String *Item_func_json_type::val_str(String *str) { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; const char *type; THD *thd; @@ -6585,7 +6613,7 @@ String *Item_func_json_keys::val_str(String *str) { json_engine_t je; Json_source_watch watch; - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); uint n_keys= 0; int array_counters[JSON_DEPTH_LIMIT]= {0}; THD *thd; @@ -7013,7 +7041,7 @@ bool Item_func_json_format::fix_length_and_dec(THD *thd) String *Item_func_json_format::val_str(String *str) { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; Json_source_watch watch; int tab_size= 4; @@ -7076,9 +7104,8 @@ String *Item_func_json_format::val_str(String *str) straight on. So whatever the argument was able to say about it is still true of it here, and this function says neither more nor less. */ -String *Item_func_json_format::val_json(String *str) +String *Item_func_json_format::forward_json(String *js) { - String *js= args[0]->val_json(&tmp_js); m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; @@ -7087,6 +7114,25 @@ String *Item_func_json_format::val_json(String *str) return js; } + +String *Item_func_json_format::val_json(String *str) +{ + return forward_json(args[0]->val_json(&tmp_js)); +} + + +/* + What is returned IS the argument's value, so a caller that is + finished with it before anything else can run leaves the argument free + to answer with a view of what it already holds - see + Item::val_json_at_once(). The promise is passed along with the value + rather than stopping here. +*/ +String *Item_func_json_format::val_json_at_once(String *str) +{ + return forward_json(args[0]->val_json_at_once(&tmp_js)); +} + int Arg_comparator::compare_json_str_basic(Item *j, Item *s) { String *js,*str; @@ -7611,7 +7657,7 @@ String *Item_func_json_normalize::val_str(String *buf) String tmp; json_engine_t je; THD *thd; - String *raw_json= args[0]->val_str(&tmp); + String *raw_json= val_json_arg0(args, arg_count, &tmp); m_marks.clear(); diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 2396696207960..54ecf2ddc8ede 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -903,6 +903,7 @@ class Item_func_json_format: public Item_json_func protected: formats fmt; String tmp_js; + String *forward_json(String *js); public: Item_func_json_format(THD *thd, Item *js, formats format): Item_json_func(thd, js), fmt(format) {} @@ -913,6 +914,7 @@ class Item_func_json_format: public Item_json_func bool fix_length_and_dec(THD *thd) override; String *val_str(String *str) override; String *val_json(String *str) override; + String *val_json_at_once(String *str) override; protected: Item *shallow_copy(THD *thd) const override diff --git a/sql/json_table.cc b/sql/json_table.cc index e5ac60085bf19..97679d9755858 100644 --- a/sql/json_table.cc +++ b/sql/json_table.cc @@ -347,7 +347,15 @@ int ha_json_table::rnd_init(bool scan) Json_table_nested_path &p= m_jt->m_nested_path; DBUG_ENTER("ha_json_table::rnd_init"); - if ((m_js= m_jt->m_json->val_str(&m_tmps))) + /* + The scan is told where the document stands and reads from there + until it runs out of rows, so the bytes it is given have to outlive + everything the query does between one row and the next. Working + out a row can write to whatever the document was read from - a + package body's variable is the plain case - so the document is + asked for the way that returns bytes of the reader's own. + */ + if ((m_js= m_jt->m_json->val_json(&m_tmps))) { p.scan_start(m_js->charset(), (const uchar *) m_js->ptr(), (const uchar *) m_js->end()); From 8bbd1a13ea71279d95fd982dc08094bdc9f235b5 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Fri, 7 Aug 2026 20:59:31 -0400 Subject: [PATCH 19/30] MDEV-40673 CONCAT drops the value of its first argument CONCAT works out its first argument into the buffer it will answer with, and every argument after that into a buffer of its own, appending each to the first as it goes. An argument that answers in the buffer it was handed rather than in one of its own leaves CONCAT holding that very buffer, and CONCAT keeps it as its accumulator without copying anything - which is the reason for handing the buffer down at all. Growing that accumulator is where the value goes. `realloc_result()` asks the buffer how much room it has, and reads none-at-all as nothing-in-it: it calls `alloc()`, which empties the string it allocates for. That reading holds for the buffer CONCAT starts with, which really is empty, and stops holding once an argument has returned it pointing at bytes it does not own - a static string, say, whose room to grow is none and whose length is six. So `CONCAT(JSON_TYPE('{"a":1}'), 'X')` answers `X`. Nothing is raised; the answer is a string of the right type and the wrong value. Appending an empty string is enough to lose it, and only the first argument can be lost, every later one being appended to an accumulator that owns its bytes by then. ## What is done about it The shortcut is removed. What is left asks for twice the room or the room needed, whichever is more, and `realloc()` carries the bytes over into the new buffer. With no room at all those come to the same thing, so the buffer is no larger than the shortcut made it; what is already in it survives the move. A buffer that really is empty is not copied out of - `realloc_raw()` skips that - so nothing is paid for by the case that was right all along. `Item_func_concat_operator_oracle` is the `||` operator of sql_mode=ORACLE, and appends through the same two functions, so the one change attests to it too. ## What reaches it `JSON_TYPE` and `GET_FORMAT` are the two functions in the tree that finish this way. `ENCRYPT` finishes the same way and then copies the bytes into a buffer of its own, which is what keeps it out of this. ## The test `func_concat` gained a section over both functions: first argument with one, two and no further arguments after it; a later argument, which was always right; over a `JSON` column, so the case is not one of literals; and through the Oracle `||` operator. `CONCAT_WS`, which accumulates differently and was never affected, is pinned beside them. --- mysql-test/main/func_concat.result | 69 ++++++++++++++++++++++++++++++ mysql-test/main/func_concat.test | 44 +++++++++++++++++++ sql/item_strfunc.cc | 6 +-- 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/mysql-test/main/func_concat.result b/mysql-test/main/func_concat.result index 4b91fb97c6c8c..d31eaf0942eb3 100644 --- a/mysql-test/main/func_concat.result +++ b/mysql-test/main/func_concat.result @@ -304,3 +304,72 @@ f NULL DROP TABLE t1; # End of 10.5 tests +# Start of 10.11 tests +# +# CONCAT keeps the value of its first argument when that argument +# answers in the buffer it was handed, pointing at bytes it does +# not own +# +SELECT JSON_TYPE('{"a":1}'); +JSON_TYPE('{"a":1}') +OBJECT +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'X'); +CONCAT(JSON_TYPE('{"a":1}'), 'X') +OBJECTX +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B'); +CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B') +OBJECTAB +SELECT CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]')); +CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]')) +OBJECTARRAY +SELECT CONCAT(JSON_TYPE('{"a":1}'), ''); +CONCAT(JSON_TYPE('{"a":1}'), '') +OBJECT +SELECT LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X')); +LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X')) +7 +SELECT GET_FORMAT(DATE, 'USA'); +GET_FORMAT(DATE, 'USA') +%m.%d.%Y +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), 'X'); +CONCAT(GET_FORMAT(DATE, 'USA'), 'X') +%m.%d.%YX +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO')); +CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO')) +%m.%d.%Y%Y-%m-%d +# Every other argument position was always right +SELECT CONCAT('P', JSON_TYPE('{"a":1}')); +CONCAT('P', JSON_TYPE('{"a":1}')) +POBJECT +SELECT CONCAT('P', GET_FORMAT(DATE, 'USA')); +CONCAT('P', GET_FORMAT(DATE, 'USA')) +P%m.%d.%Y +SELECT CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X'); +CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X') +OBJECT-X +# The same over table data +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'), ('[1,2]'); +SELECT CONCAT(JSON_TYPE(j), '.') AS c FROM t1; +c +OBJECT. +ARRAY. +SELECT CONCAT(JSON_TYPE(j), JSON_TYPE(j)) AS c FROM t1; +c +OBJECTOBJECT +ARRAYARRAY +DROP TABLE t1; +# The || operator in sql_mode=ORACLE concatenates the same way +SET @save_sql_mode=@@sql_mode; +SET sql_mode=ORACLE; +SELECT JSON_TYPE('{"a":1}') || 'X'; +JSON_TYPE('{"a":1}') || 'X' +OBJECTX +SELECT GET_FORMAT(DATE, 'USA') || 'X'; +GET_FORMAT(DATE, 'USA') || 'X' +%m.%d.%YX +SELECT 'P' || JSON_TYPE('{"a":1}'); +'P' || JSON_TYPE('{"a":1}') +POBJECT +SET sql_mode=@save_sql_mode; +# End of 10.11 tests diff --git a/mysql-test/main/func_concat.test b/mysql-test/main/func_concat.test index f93b150f88f90..f9d8367b404e6 100644 --- a/mysql-test/main/func_concat.test +++ b/mysql-test/main/func_concat.test @@ -281,3 +281,47 @@ SELECT CONCAT_WS(' ', a, b, PASSWORD(c)) AS f FROM t1 GROUP BY f WITH ROLLUP; DROP TABLE t1; --echo # End of 10.5 tests + +--echo # Start of 10.11 tests + +--echo # +--echo # CONCAT keeps the value of its first argument when that argument +--echo # answers in the buffer it was handed, pointing at bytes it does +--echo # not own +--echo # + +SELECT JSON_TYPE('{"a":1}'); +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'X'); +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B'); +SELECT CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]')); +SELECT CONCAT(JSON_TYPE('{"a":1}'), ''); +SELECT LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X')); + +SELECT GET_FORMAT(DATE, 'USA'); +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), 'X'); +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO')); + +--echo # Every other argument position was always right + +SELECT CONCAT('P', JSON_TYPE('{"a":1}')); +SELECT CONCAT('P', GET_FORMAT(DATE, 'USA')); +SELECT CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X'); + +--echo # The same over table data + +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'), ('[1,2]'); +SELECT CONCAT(JSON_TYPE(j), '.') AS c FROM t1; +SELECT CONCAT(JSON_TYPE(j), JSON_TYPE(j)) AS c FROM t1; +DROP TABLE t1; + +--echo # The || operator in sql_mode=ORACLE concatenates the same way + +SET @save_sql_mode=@@sql_mode; +SET sql_mode=ORACLE; +SELECT JSON_TYPE('{"a":1}') || 'X'; +SELECT GET_FORMAT(DATE, 'USA') || 'X'; +SELECT 'P' || JSON_TYPE('{"a":1}'); +SET sql_mode=@save_sql_mode; + +--echo # End of 10.11 tests diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 00d0bfc0b1d07..2ed0911265f58 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -642,9 +642,6 @@ bool Item_func_concat::realloc_result(String *str, uint length) const if (str->alloced_length() >= length) return false; // Alloced space is big enough, nothing to do. - if (str->alloced_length() == 0) - return str->alloc(length); - /* Item_func_concat::val_str() makes sure the result length does not grow higher than max_allowed_packet. So "length" is limited to 1G here. @@ -653,6 +650,9 @@ bool Item_func_concat::realloc_result(String *str, uint length) const So multiplication by 2 can overflow, if args[0] for some reasons did not limit the result to max_alloced_packet. But it's not harmful, "str" will be reallocated exactly to "length" bytes in case of overflow. + It can even be zero, with "str" pointing to bytes that args[0] does not + own. realloc() moves those bytes into a buffer of our own, while alloc() + would throw them away: it empties the string it allocates for. */ uint new_length= MY_MAX(str->alloced_length() * 2, length); return str->realloc(new_length); From 2f815665b0ae1cda8fb57996ab105862bd614f4c Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Sat, 8 Aug 2026 01:14:47 -0400 Subject: [PATCH 20/30] Let a variable attest to the value it was given A value read out of a stored program's variable is read again to find out whether it is a document, because nothing about a variable says what was put in it. Something does: every assignment to one goes through a single funnel, and it arrives there with the expression that made the value still in hand. So what is in a variable can be said at the moment it is put there, once per assignment rather than once per reading, and a function later handed that variable as the document to work on can be spared finding out again what was already known. A stored program keeps its variables in the fields of a table it builds for itself. No statement can name such a table, and nothing writes one but the funnel, so `TABLE::json_held_marks` is left there and nowhere else - the array being there at all is the whole of the discriminator, as the bitmap beside it is for a temporary table the server built for a query. A base table and a table the user asked for answer no by having nowhere to keep a yes. The room is given where a stored program's variables are built, rather than in the base that builds every table of the kind. Two of the three callers of that base build something else - the table `SUM(DISTINCT)` counts in, and the one row-based replication converts through - and neither holds anything anybody assigns to, so neither has anything to say about a value. Both ways a stored program gets a table come through the one caller, the row a cursor is read into among them, so nothing of the kind is left to be given room afterwards. Unlike the marks a column of a query's temporary table carries, these say nothing about rows. There is one value, the item that made it is right there, and it is replaced whole by the next assignment - so the formatting and the depth are taken exactly rather than as a bound over everything the field will ever hold. The value is read back out of the field only where something reads it. Nothing but the debug check ever looks at it - the marks appear in no result - so a released server does not write it out to hand it to a check that is not there. The rule is the one every store site applies: the item said it was a document, the store kept every character of it, and nothing went wrong while it happened. It is the column's rule, so the assignment asks `Field::is_attestation_preserved()` rather than writing the same questions out beside it. What stays here is the NULL: a variable holding one is holding nothing that reads as a document, whatever the item it was assigned says about itself. That is asked of the destination as the answer is put down, and it is asked again of whatever field the answer is later read off, because a field with nothing in it attests to nothing whichever of the two channels is carrying the answer. Nothing here needs the second asking - an answer put down one value at a time is cleared for a NULL where it is set, so it never stands over a row with nothing in it - and it is asked of both anyway, so that a reader does not have to know which channel it is reading. Keeping the characters is two questions and not one. Whether the type puts down what it is given is one of them, and whether the two character sets put the same characters down is the other. A store into or out of the binary set answers the first yes and the second no - the bytes go across untouched and are then read as whatever the other set makes of them - so a document written in a wide set arrives in a BLOB as bytes that read as no document at all. Both are asked here, in the words the column stores beside this one ask them in. The store is asked what it returned and not merely whether anything went wrong, because on this path the two come apart. Assigning more than fits raises an error under the strict mode and the value never lands; under the loose one the front of it is put down and neither an error nor a warning is raised, leaving a variable holding something that is not a document and nobody saying so. Asking the store what it kept is what refuses that. Nothing is cleared on the way in. Every way out of the funnel ends either at the answer or at the clear beside it, so a value that never arrived leaves nothing standing over the bytes of the one before it. Not clearing first is also what lets a variable assigned to itself keep its answer. A row is written a member at a time whichever form the assignment takes, so each member is attested by whatever made the value that landed in it. Where a row assignment stops part of the way through, the members it never reached give their answers up: what they hold is what the assignment before this one left. The ones it wrote keep theirs, having gone through the funnel like any other. Assigning one variable to another arrives at the store as a read of the field behind the source, dereferenced on the way, so the item the funnel is handed is the same kind of item a reader of that variable reads through. Answering there answers both, and a value moved from variable to variable carries what was said about it without anything having to move it. The standing question is left unanswered on purpose. A variable holds what the last assignment put in it and the next one will put something else there, so nothing about the value in it now holds for every evaluation - which is what stops a column built out of a variable being granted an answer about values it has not been given yet. A variable is not typed as a document however it was declared, so a value out of one is quoted into a document rather than spliced into it, exactly as before. Which of the two happens is decided by the type; what is attested here is the other thing, and a value can be attested and still be quoted. The assertion that said otherwise was written when the two always agreed and is now gone: nothing in that arm reads the answer, and quoting a document writes the same bytes as quoting anything else. `func_json_sp_trust` records the answers, which do not move. Among them is the declared type that changes nothing on the way in: a conversion written out in the assignment, and a variable already holding the wide encoding, each landing in a BLOB, with a reading of the wide variable itself beside them so that what those two show is about the store and not about the set. Its last section writes a row variable from a list of values rather than from a row: `SELECT INTO` and `FETCH INTO` write the members one at a time and stop at the first that fails, leaving a mixture of what this assignment wrote, a member holding nothing, and what the assignment before left. An answer standing for either of the last two would be an answer from an assignment that did not finish. `func_json_sp_scan_count` records the readings, which do: one edit over a variable falls from two to one, five edits from ten to five, and a value moved between variables is not read at all. Both a value the store cut short and an assignment that never landed are in there, being the two ways an answer has to be refused. --- .../main/func_json_sp_scan_count.result | 274 +++++++++++ mysql-test/main/func_json_sp_scan_count.test | 275 +++++++++++ mysql-test/main/func_json_sp_trust.result | 431 ++++++++++++++++++ mysql-test/main/func_json_sp_trust.test | 366 +++++++++++++++ sql/field.cc | 118 ++++- sql/field.h | 61 ++- sql/item.cc | 4 + sql/item.h | 55 +++ sql/item_jsonfunc.cc | 13 +- sql/sql_select.cc | 47 ++ sql/sql_select.h | 27 ++ sql/table.h | 30 ++ 12 files changed, 1681 insertions(+), 20 deletions(-) create mode 100644 mysql-test/main/func_json_sp_scan_count.result create mode 100644 mysql-test/main/func_json_sp_scan_count.test create mode 100644 mysql-test/main/func_json_sp_trust.result create mode 100644 mysql-test/main/func_json_sp_trust.test diff --git a/mysql-test/main/func_json_sp_scan_count.result b/mysql-test/main/func_json_sp_scan_count.result new file mode 100644 index 0000000000000..3a4cff5492163 --- /dev/null +++ b/mysql-test/main/func_json_sp_scan_count.result @@ -0,0 +1,274 @@ +# +# How many times a value out of a stored program's variable is read +# to find out whether it is a document. +# +# No query can see the difference - the reading could only have found +# out what was already known - so nothing but Json_scans says whether +# it happened. The counts below are the record of it. A count that +# goes up is a reading that came back. +# +# A debug build reads values back to check what was claimed about +# them, and none of those readings are counted: they are the debug +# build's work and not the server's. A count of 0 below therefore +# means nothing read, not that a reading went uncounted. +# +SET NAMES utf8mb4; +# +# 1. A variable assigned by hand, then edited twice. Nobody said +# anything about what was typed, so the first edit reads it and then +# reads back what it composed. The second edit is over what the +# first one put there, which was attested to as it went in, so it +# reads once. +# +CREATE PROCEDURE p_literal() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= '{"a":1}'; +SET v= JSON_SET(v, '$.b', 2); +SET v= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_literal(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +# +# 2. The same three statements, with the first one assigning what a +# function attested rather than what somebody typed. +# +CREATE PROCEDURE p_attested() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_SET(v, '$.b', 2); +SET v= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_attested(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 3. One edit on its own, either way round. +# +CREATE PROCEDURE p_one_literal() +BEGIN +DECLARE v LONGTEXT DEFAULT '{"a":1}'; +SET v= JSON_SET(v, '$.b', 2); +END $$ +CREATE PROCEDURE p_one_attested() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_SET(v, '$.b', 2); +END $$ +FLUSH STATUS; +CALL p_one_literal(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +CALL p_one_attested(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# +# 4. One variable assigned from another. Moving a value from one +# variable to another reads nothing, and what was said about it +# arrives with it. +# +CREATE PROCEDURE p_var_to_var() +BEGIN +DECLARE a LONGTEXT DEFAULT NULL; +DECLARE b LONGTEXT DEFAULT NULL; +SET a= JSON_OBJECT('a', 1); +SET b= a; +SET b= JSON_SET(b, '$.b', 2); +END $$ +FLUSH STATUS; +CALL p_var_to_var(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# +# 5. The other mutators over the same variable, once each. +# +CREATE PROCEDURE p_mutators() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE r LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1, 'arr', JSON_ARRAY(1, 2)); +SET r= JSON_INSERT(v, '$.b', 2); +SET r= JSON_REPLACE(v, '$.a', 9); +SET r= JSON_REMOVE(v, '$.a'); +SET r= JSON_ARRAY_APPEND(v, '$.arr', 3); +SET r= JSON_ARRAY_INSERT(v, '$.arr[0]', 0); +END $$ +FLUSH STATUS; +CALL p_mutators(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 5 +# +# 6. A member of a row variable, written one member at a time and +# then a whole row at a time. A row is written a member at a time +# either way - the whole-row form walks the members and puts each +# one through the same funnel - so both say the same thing and both +# are read the same number of times. +# +CREATE PROCEDURE p_row_member() +BEGIN +DECLARE r ROW(a LONGTEXT, b LONGTEXT); +DECLARE out1 LONGTEXT DEFAULT NULL; +SET r.a= JSON_OBJECT('x', 1); +SET r.b= JSON_ARRAY(1, 2); +SET out1= JSON_SET(r.a, '$.y', 2); +SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +CREATE PROCEDURE p_row_whole() +BEGIN +DECLARE r ROW(a LONGTEXT, b LONGTEXT); +DECLARE out1 LONGTEXT DEFAULT NULL; +SET r= ROW(JSON_OBJECT('x', 1), JSON_ARRAY(1, 2)); +SET out1= JSON_SET(r.a, '$.y', 2); +SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +FLUSH STATUS; +CALL p_row_member(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +CALL p_row_whole(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 7. A value that does not fit what it is being assigned to. What +# was said about it was said about the value passed and not +# about what fitted, so the answer goes with the part that did not. +# +# Which of the two things happens depends on the mode. Under the +# strict one the assignment does not land at all and the variable is +# left holding nothing, so there is nothing to read; under the loose +# one the front of the value is put down, quietly and without so +# much as a warning, and what is there is not a document. It is the +# second that the store having to be asked what it kept is for: no +# error is raised on that path and nothing else would say. +CREATE PROCEDURE p_truncated() +BEGIN +DECLARE fits VARCHAR(64) DEFAULT NULL; +DECLARE cut VARCHAR(8) DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET fits= JSON_OBJECT('a', 1); +SET out1= JSON_SET(fits, '$.b', 2); +SET cut= JSON_OBJECT('aaaaaaaaaa', 1); +SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; +SET out1= JSON_SET(cut, '$.b', 2); +END $$ +FLUSH STATUS; +CALL p_truncated(); +what_landed reads_as_document +NULL NULL +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# the same body under the mode that lets a short store through, which +# a procedure keeps from the moment it is created and not from the +# moment it is called +SET @save_sql_mode= @@sql_mode; +SET sql_mode=''; +CREATE PROCEDURE p_truncated_loose() +BEGIN +DECLARE fits VARCHAR(64) DEFAULT NULL; +DECLARE cut VARCHAR(8) DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET fits= JSON_OBJECT('a', 1); +SET out1= JSON_SET(fits, '$.b', 2); +SET cut= JSON_OBJECT('aaaaaaaaaa', 1); +SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; +SET out1= JSON_SET(cut, '$.b', 2); +END $$ +SET sql_mode= @save_sql_mode; +FLUSH STATUS; +CALL p_truncated_loose(); +what_landed reads_as_document +{"aaaaaa 0 +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +# +# 8. An assignment that never landed. The answer the one before it +# left is given up with it, and the variable is left holding +# nothing, so what a reader finds is a NULL and not a stale +# document with a stale answer beside it. +# +CREATE PROCEDURE p_failed() +BEGIN +DECLARE v VARCHAR(20) DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_OBJECT('bbbbbbbbbbbbbbbbbbbb', 2); +SELECT v AS after_failure; +SET out1= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_failed(); +after_failure +NULL +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 9. A value out of a table by SELECT INTO, from a column that says +# nothing and from a function that does. Two edits rather than one, +# so that the reading the function's own scan pays for does not hide +# the readings the edits do or do not do. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); +CREATE PROCEDURE p_from_column() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +SELECT j INTO v FROM t1 WHERE id= 1; +SET out1= JSON_SET(v, '$.b', 2); +SET out1= JSON_SET(v, '$.c', 3); +END $$ +CREATE PROCEDURE p_from_function() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 1; +SET out1= JSON_SET(v, '$.b', 2); +SET out1= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_from_column(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +CALL p_from_function(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +DROP PROCEDURE p_literal; +DROP PROCEDURE p_attested; +DROP PROCEDURE p_one_literal; +DROP PROCEDURE p_one_attested; +DROP PROCEDURE p_var_to_var; +DROP PROCEDURE p_mutators; +DROP PROCEDURE p_row_member; +DROP PROCEDURE p_row_whole; +DROP PROCEDURE p_truncated; +DROP PROCEDURE p_truncated_loose; +DROP PROCEDURE p_failed; +DROP PROCEDURE p_from_column; +DROP PROCEDURE p_from_function; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp_scan_count.test b/mysql-test/main/func_json_sp_scan_count.test new file mode 100644 index 0000000000000..36e6872fd548c --- /dev/null +++ b/mysql-test/main/func_json_sp_scan_count.test @@ -0,0 +1,275 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a value out of a stored program's variable is read +--echo # to find out whether it is a document. +--echo # +--echo # No query can see the difference - the reading could only have found +--echo # out what was already known - so nothing but Json_scans says whether +--echo # it happened. The counts below are the record of it. A count that +--echo # goes up is a reading that came back. +--echo # +--echo # A debug build reads values back to check what was claimed about +--echo # them, and none of those readings are counted: they are the debug +--echo # build's work and not the server's. A count of 0 below therefore +--echo # means nothing read, not that a reading went uncounted. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A variable assigned by hand, then edited twice. Nobody said +--echo # anything about what was typed, so the first edit reads it and then +--echo # reads back what it composed. The second edit is over what the +--echo # first one put there, which was attested to as it went in, so it +--echo # reads once. +--echo # +--delimiter $$ +CREATE PROCEDURE p_literal() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= '{"a":1}'; + SET v= JSON_SET(v, '$.b', 2); + SET v= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_literal(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 2. The same three statements, with the first one assigning what a +--echo # function attested rather than what somebody typed. +--echo # +--delimiter $$ +CREATE PROCEDURE p_attested() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_SET(v, '$.b', 2); + SET v= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_attested(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 3. One edit on its own, either way round. +--echo # +--delimiter $$ +CREATE PROCEDURE p_one_literal() +BEGIN + DECLARE v LONGTEXT DEFAULT '{"a":1}'; + SET v= JSON_SET(v, '$.b', 2); +END $$ +CREATE PROCEDURE p_one_attested() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_SET(v, '$.b', 2); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_one_literal(); +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +CALL p_one_attested(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 4. One variable assigned from another. Moving a value from one +--echo # variable to another reads nothing, and what was said about it +--echo # arrives with it. +--echo # +--delimiter $$ +CREATE PROCEDURE p_var_to_var() +BEGIN + DECLARE a LONGTEXT DEFAULT NULL; + DECLARE b LONGTEXT DEFAULT NULL; + SET a= JSON_OBJECT('a', 1); + SET b= a; + SET b= JSON_SET(b, '$.b', 2); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_var_to_var(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 5. The other mutators over the same variable, once each. +--echo # +--delimiter $$ +CREATE PROCEDURE p_mutators() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE r LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1, 'arr', JSON_ARRAY(1, 2)); + SET r= JSON_INSERT(v, '$.b', 2); + SET r= JSON_REPLACE(v, '$.a', 9); + SET r= JSON_REMOVE(v, '$.a'); + SET r= JSON_ARRAY_APPEND(v, '$.arr', 3); + SET r= JSON_ARRAY_INSERT(v, '$.arr[0]', 0); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_mutators(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 6. A member of a row variable, written one member at a time and +--echo # then a whole row at a time. A row is written a member at a time +--echo # either way - the whole-row form walks the members and puts each +--echo # one through the same funnel - so both say the same thing and both +--echo # are read the same number of times. +--echo # +--delimiter $$ +CREATE PROCEDURE p_row_member() +BEGIN + DECLARE r ROW(a LONGTEXT, b LONGTEXT); + DECLARE out1 LONGTEXT DEFAULT NULL; + SET r.a= JSON_OBJECT('x', 1); + SET r.b= JSON_ARRAY(1, 2); + SET out1= JSON_SET(r.a, '$.y', 2); + SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +CREATE PROCEDURE p_row_whole() +BEGIN + DECLARE r ROW(a LONGTEXT, b LONGTEXT); + DECLARE out1 LONGTEXT DEFAULT NULL; + SET r= ROW(JSON_OBJECT('x', 1), JSON_ARRAY(1, 2)); + SET out1= JSON_SET(r.a, '$.y', 2); + SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_row_member(); +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +CALL p_row_whole(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 7. A value that does not fit what it is being assigned to. What +--echo # was said about it was said about the value passed and not +--echo # about what fitted, so the answer goes with the part that did not. +--echo # +--echo # Which of the two things happens depends on the mode. Under the +--echo # strict one the assignment does not land at all and the variable is +--echo # left holding nothing, so there is nothing to read; under the loose +--echo # one the front of the value is put down, quietly and without so +--echo # much as a warning, and what is there is not a document. It is the +--echo # second that the store having to be asked what it kept is for: no +--echo # error is raised on that path and nothing else would say. +--delimiter $$ +CREATE PROCEDURE p_truncated() +BEGIN + DECLARE fits VARCHAR(64) DEFAULT NULL; + DECLARE cut VARCHAR(8) DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET fits= JSON_OBJECT('a', 1); + SET out1= JSON_SET(fits, '$.b', 2); + SET cut= JSON_OBJECT('aaaaaaaaaa', 1); + SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; + SET out1= JSON_SET(cut, '$.b', 2); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_truncated(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # the same body under the mode that lets a short store through, which +--echo # a procedure keeps from the moment it is created and not from the +--echo # moment it is called +SET @save_sql_mode= @@sql_mode; +SET sql_mode=''; +--delimiter $$ +CREATE PROCEDURE p_truncated_loose() +BEGIN + DECLARE fits VARCHAR(64) DEFAULT NULL; + DECLARE cut VARCHAR(8) DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET fits= JSON_OBJECT('a', 1); + SET out1= JSON_SET(fits, '$.b', 2); + SET cut= JSON_OBJECT('aaaaaaaaaa', 1); + SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; + SET out1= JSON_SET(cut, '$.b', 2); +END $$ +--delimiter ; +SET sql_mode= @save_sql_mode; +FLUSH STATUS; +CALL p_truncated_loose(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 8. An assignment that never landed. The answer the one before it +--echo # left is given up with it, and the variable is left holding +--echo # nothing, so what a reader finds is a NULL and not a stale +--echo # document with a stale answer beside it. +--echo # +--delimiter $$ +CREATE PROCEDURE p_failed() +BEGIN + DECLARE v VARCHAR(20) DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_OBJECT('bbbbbbbbbbbbbbbbbbbb', 2); + SELECT v AS after_failure; + SET out1= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_failed(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 9. A value out of a table by SELECT INTO, from a column that says +--echo # nothing and from a function that does. Two edits rather than one, +--echo # so that the reading the function's own scan pays for does not hide +--echo # the readings the edits do or do not do. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); + +--delimiter $$ +CREATE PROCEDURE p_from_column() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + SELECT j INTO v FROM t1 WHERE id= 1; + SET out1= JSON_SET(v, '$.b', 2); + SET out1= JSON_SET(v, '$.c', 3); +END $$ +CREATE PROCEDURE p_from_function() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 1; + SET out1= JSON_SET(v, '$.b', 2); + SET out1= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_from_column(); +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +CALL p_from_function(); +SHOW STATUS LIKE 'Json_scans'; + +DROP PROCEDURE p_literal; +DROP PROCEDURE p_attested; +DROP PROCEDURE p_one_literal; +DROP PROCEDURE p_one_attested; +DROP PROCEDURE p_var_to_var; +DROP PROCEDURE p_mutators; +DROP PROCEDURE p_row_member; +DROP PROCEDURE p_row_whole; +DROP PROCEDURE p_truncated; +DROP PROCEDURE p_truncated_loose; +DROP PROCEDURE p_failed; +DROP PROCEDURE p_from_column; +DROP PROCEDURE p_from_function; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp_trust.result b/mysql-test/main/func_json_sp_trust.result new file mode 100644 index 0000000000000..a9fc0930a4c02 --- /dev/null +++ b/mysql-test/main/func_json_sp_trust.result @@ -0,0 +1,431 @@ +# +# What a JSON function reads out of a stored program's variable. +# +# A variable is written by one funnel and by nothing else, and every +# assignment to one arrives there with the expression that made the +# value still in hand. So what was put in a variable can be said at +# the moment it is put there, and a function later handed that +# variable as the document to work on can be spared finding out +# again what is in it. +# +# A variable is not typed as a document however it was declared, so +# a value out of one is still quoted into a document rather than +# spliced into it. That is decided by the type and is left alone +# here. The answers below are the same either way - the reading +# that is left out could only have found out what was already +# known - so this file is here to say that they stay the same. +# +SET NAMES utf8mb4; +# +# 1. A variable assigned by a function that attests to its result, +# then read as a document. +# +CREATE PROCEDURE p_read() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_SET('{"a":1}', '$.b', 2); +SELECT v AS held; +SELECT JSON_SET(v, '$.c', 3) AS edited; +SELECT JSON_INSERT(v, '$.c', 3) AS inserted; +SELECT JSON_REPLACE(v, '$.a', 9) AS replaced; +SELECT JSON_REMOVE(v, '$.a') AS removed; +SELECT JSON_ARRAY_APPEND(JSON_SET(v, '$.arr', JSON_ARRAY(1)), '$.arr', 2) +AS appended; +SELECT JSON_MERGE_PATCH(v, '{"d":4}') AS merged; +SELECT JSON_EXTRACT(v, '$.a') AS extracted; +SELECT JSON_TYPE(v) AS what, JSON_DEPTH(v) AS deep, JSON_LENGTH(v) AS len; +SELECT JSON_VALID(v) AS valid, JSON_KEYS(v) AS keys_of; +END $$ +CALL p_read(); +held +{"a": 1, "b": 2} +edited +{"a": 1, "b": 2, "c": 3} +inserted +{"a": 1, "b": 2, "c": 3} +replaced +{"a": 9, "b": 2} +removed +{"b": 2} +appended +{"a": 1, "b": 2, "arr": [1, 2]} +merged +{"a": 1, "b": 2, "d": 4} +extracted +1 +what deep len +OBJECT 2 2 +valid keys_of +1 ["a", "b"] +DROP PROCEDURE p_read; +# +# 2. The same value spliced into a new document. A variable is not +# typed as a document, so it is quoted - which is what a released +# server does with it and what it goes on doing. +# +CREATE PROCEDURE p_splice() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE j JSON DEFAULT NULL; +SET v= JSON_SET('{"a":1}', '$.b', 2); +SET j= JSON_SET('{"a":1}', '$.b', 2); +SELECT JSON_ARRAY(v) AS text_var, JSON_ARRAY(j) AS json_var; +SELECT JSON_OBJECT('k', v) AS obj_text, JSON_OBJECT('k', j) AS obj_json; +SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; +END $$ +CALL p_splice(); +text_var json_var +["{\"a\": 1, \"b\": 2}"] ["{\"a\": 1, \"b\": 2}"] +obj_text obj_json +{"k": "{\"a\": 1, \"b\": 2}"} {"k": "{\"a\": 1, \"b\": 2}"} +still_valid +1 +DROP PROCEDURE p_splice; +# +# 3. One variable assigned from another. The assignment reaches the +# funnel as a read of the field behind the source, so what was said +# about the value travels with it and does not have to be said +# again. +# +CREATE PROCEDURE p_var_to_var() +BEGIN +DECLARE a LONGTEXT DEFAULT NULL; +DECLARE b LONGTEXT DEFAULT NULL; +SET a= JSON_SET('{"a":1}', '$.b', 2); +SET b= a; +SELECT JSON_SET(b, '$.c', 3) AS edited; +SET a= 'not a document at all'; +SELECT JSON_SET(b, '$.d', 4) AS still_edited; +SELECT b AS unchanged; +END $$ +CALL p_var_to_var(); +edited +{"a": 1, "b": 2, "c": 3} +still_edited +{"a": 1, "b": 2, "d": 4} +unchanged +{"a": 1, "b": 2} +DROP PROCEDURE p_var_to_var; +# +# 4. A variable that is both the source and the target. +# +CREATE PROCEDURE p_self() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_SET(v, '$.b', 2); +SET v= JSON_SET(v, '$.c', 3); +SELECT v AS built; +SET v= v; +SELECT JSON_SET(v, '$.d', 4) AS after_self_assign; +END $$ +CALL p_self(); +built +{"a": 1, "b": 2, "c": 3} +after_self_assign +{"a": 1, "b": 2, "c": 3, "d": 4} +DROP PROCEDURE p_self; +# +# 5. Assignments that put something else there. A literal is not +# attested by anybody, and a value the store had to change is +# not the value that was attested. +# +CREATE PROCEDURE p_untrusted() +BEGIN +DECLARE lit LONGTEXT DEFAULT NULL; +DECLARE short_v VARCHAR(8) DEFAULT NULL; +DECLARE num INT DEFAULT NULL; +SET lit= '{"a":1}'; +SELECT JSON_SET(lit, '$.b', 2) AS from_literal; +SELECT JSON_TYPE(lit) AS what; +SET num= JSON_LENGTH(JSON_SET('{"a":1}', '$.b', 2)); +SELECT JSON_SET(CONCAT('[', num, ']'), '$[1]', 9) AS from_number; +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET short_v= JSON_SET('{"a":1}', '$.b', 2); +END; +SELECT short_v AS truncated; +SELECT JSON_VALID(short_v) AS valid_after_truncation; +END $$ +CALL p_untrusted(); +from_literal +{"a": 1, "b": 2} +what +OBJECT +from_number +[2, 9] +truncated +NULL +valid_after_truncation +NULL +DROP PROCEDURE p_untrusted; +# +# 6. An assignment that does not happen. What the one before it +# left standing goes with it, the bytes under it being whatever the +# store got as far as putting down. +# +CREATE PROCEDURE p_failed() +BEGIN +DECLARE v VARCHAR(20) DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET v= JSON_SET('{"a":1}', '$.b', 2); +SELECT JSON_SET(v, '$.c', 3) AS edited; +SET v= JSON_SET('{"aaaaaaaaaa":1}', '$.bbbbbbbbbb', 2); +SELECT v AS after_failure; +SELECT JSON_VALID(v) AS valid_after_failure; +END $$ +CALL p_failed(); +edited +{"a": 1, "b": 2, "c": 3} +after_failure +NULL +valid_after_failure +NULL +DROP PROCEDURE p_failed; +# +# 7. Row variables. A row is written a member at a time whichever +# form the assignment takes, each member going through the same +# funnel a plain variable goes through, so each of them is answered +# for on its own and by whatever made the value that landed in it. +# +CREATE PROCEDURE p_row() +BEGIN +DECLARE r ROW(a LONGTEXT, b LONGTEXT); +SET r.a= JSON_SET('{"x":1}', '$.y', 2); +SET r.b= JSON_SET('[1]', '$[1]', 2); +SELECT JSON_SET(r.a, '$.z', 3) AS member_a; +SELECT JSON_ARRAY_APPEND(r.b, '$', 3) AS member_b; +SET r= ROW(JSON_OBJECT('p', 1), JSON_ARRAY(7, 8)); +SELECT JSON_SET(r.a, '$.q', 2) AS whole_row_a; +SELECT JSON_ARRAY_APPEND(r.b, '$', 9) AS whole_row_b; +SET r= ROW('{"lit":1}', '[0]'); +SELECT JSON_SET(r.a, '$.q', 2) AS literal_row_a; +END $$ +CALL p_row(); +member_a +{"x": 1, "y": 2, "z": 3} +member_b +[1, 2, 3] +whole_row_a +{"p": 1, "q": 2} +whole_row_b +[7, 8, 9] +literal_row_a +{"lit": 1, "q": 2} +DROP PROCEDURE p_row; +# +# 8. A row assignment that fails part of the way through. The +# members it did not reach are still holding what the assignment +# before this one left, so their answers go; the ones it wrote keep +# theirs, having been attested on the way in. +# +CREATE PROCEDURE p_row_partial() +BEGIN +DECLARE r ROW(a VARCHAR(20), b VARCHAR(8)); +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET r.a= JSON_SET('{"x":1}', '$.y', 2); +SET r.b= JSON_SET('[1]', '$[1]', 2); +SELECT JSON_SET(r.a, '$.z', 3) AS before_a; +SET r= ROW(JSON_OBJECT('p', 1), JSON_OBJECT('qqqqqqqqqq', 22222)); +SELECT r.a AS after_a, r.b AS after_b; +SELECT JSON_VALID(r.a) AS valid_a, JSON_VALID(r.b) AS valid_b; +END $$ +CALL p_row_partial(); +before_a +{"x": 1, "y": 2, "z": 3} +after_a after_b +{"p": 1} NULL +valid_a valid_b +1 NULL +DROP PROCEDURE p_row_partial; +# +# 9. A value that arrives from a table rather than from a function, +# by SELECT INTO and by FETCH. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); +CREATE PROCEDURE p_into() +BEGIN +DECLARE done INT DEFAULT 0; +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE c CURSOR FOR SELECT j FROM t1 ORDER BY id; +DECLARE CONTINUE HANDLER FOR NOT FOUND SET done= 1; +SELECT j INTO v FROM t1 WHERE id= 1; +SELECT JSON_SET(v, '$.b', 2) AS from_select_into; +SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 2; +SELECT JSON_ARRAY_APPEND(v, '$', 3) AS from_extract_into; +OPEN c; +read_loop: LOOP +FETCH c INTO v; +IF done THEN +LEAVE read_loop; +END IF; +SELECT JSON_TYPE(v) AS fetched_type, JSON_DEPTH(v) AS fetched_depth; +END LOOP; +CLOSE c; +END $$ +CALL p_into(); +from_select_into +{"a": 1, "b": 2} +from_extract_into +[1, 2, 3] +fetched_type fetched_depth +OBJECT 2 +fetched_type fetched_depth +ARRAY 2 +DROP PROCEDURE p_into; +# +# 10. Through a stored function, whose parameter is a variable of the +# frame the call makes and whose result is not one at all. +# +CREATE FUNCTION f_edit(doc LONGTEXT) RETURNS LONGTEXT +BEGIN +RETURN JSON_SET(doc, '$.f', 1); +END $$ +SELECT f_edit(JSON_OBJECT('a', 1)) AS called_with_document; +called_with_document +{"a": 1, "f": 1} +SELECT f_edit('{"a":1}') AS called_with_literal; +called_with_literal +{"a": 1, "f": 1} +SELECT JSON_SET(f_edit(JSON_OBJECT('a', 1)), '$.g', 2) AS over_the_result; +over_the_result +{"a": 1, "f": 1, "g": 2} +SELECT JSON_ARRAY(f_edit(JSON_OBJECT('a', 1))) AS spliced_result; +spliced_result +["{\"a\": 1, \"f\": 1}"] +SELECT f_edit(j) AS over_a_column FROM t1 ORDER BY id; +over_a_column +{"a": 1, "f": 1} +[1, 2] +DROP FUNCTION f_edit; +# +# 11. Values the declared type has to change on the way in: a +# character set that can hold every character of the document, and +# one that cannot. +# +CREATE PROCEDURE p_charset() +BEGIN +DECLARE l1 VARCHAR(64) CHARACTER SET latin1 DEFAULT NULL; +DECLARE u VARCHAR(64) CHARACTER SET utf8mb4 DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET u= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); +SELECT JSON_SET(u, '$.c', 'y') AS utf8_var; +SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); +SELECT JSON_SET(l1, '$.c', 'y') AS latin1_var; +SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xC3A4); +SELECT HEX(l1) AS latin1_converted; +SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xE4B8AD); +SELECT HEX(l1) AS latin1_lossy; +SELECT JSON_VALID(l1) AS valid_after_loss; +END $$ +CALL p_charset(); +utf8_var +{"a": "e", "b": "z", "c": "y"} +latin1_var +{"a": "e", "b": "z", "c": "y"} +latin1_converted +7B2261223A202265222C202262223A2022E4227D +latin1_lossy +NULL +valid_after_loss +NULL +DROP PROCEDURE p_charset; +# +# 12. A declared type that changes nothing on the way in and still +# leaves something else there. A store into the binary type keeps +# the bytes and calls them by another name, so a document written +# in a wide set arrives as bytes that read as no document at all. +# Once for a conversion written out in the assignment, and once for +# a variable already holding the wide encoding. +# +CREATE PROCEDURE p_binary() +BEGIN +DECLARE b BLOB DEFAULT NULL; +DECLARE w VARCHAR(64) CHARACTER SET ucs2 DEFAULT NULL; +SET b= CONVERT(JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z') +USING ucs2); +SELECT HEX(b) AS binary_converted, JSON_VALID(b) AS valid_converted; +SET w= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); +SELECT JSON_VALID(w) AS valid_wide; +SET b= w; +SELECT HEX(b) AS binary_from_wide, JSON_VALID(b) AS valid_from_wide; +END $$ +CALL p_binary(); +binary_converted valid_converted +007B002200610022003A0020002200650022002C0020002200620022003A00200022007A0022007D 0 +valid_wide +1 +binary_from_wide valid_from_wide +007B002200610022003A0020002200650022002C0020002200620022003A00200022007A0022007D 0 +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +DROP PROCEDURE p_binary; +DROP TABLE t1; +# +# 13. A row variable written from a list of values rather than from +# a row. Section 8 assigns a whole row at once; SELECT INTO and +# FETCH INTO write the members one at a time from a list and stop at +# the first that fails. What the variable holds afterwards is a +# mixture of three things: the members before the failure carry what +# this assignment wrote, the one it stopped at is left holding +# nothing, and the ones after it still carry what the assignment +# before left. Anything said about the last two belongs to an +# assignment that did not finish, and goes with it. +# +# A variable is never typed as a document, so a value read out of +# one is quoted into a new document rather than spliced into it, in +# both states - which is what the whole-assignment case is printed +# for. What the answers show is that reading the mixture says the +# same things about it as reading a variable nothing went wrong +# with. +# +SET @@sql_mode='STRICT_ALL_TABLES'; +CREATE PROCEDURE p_row_into() +BEGIN +DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION +SELECT r.a AS written_by_this_assignment, +r.b AS the_member_it_stopped_at, +r.c AS left_by_the_assignment_before, +JSON_VALID(r.c) AS still_a_document, +JSON_TYPE(r.c) AS ty, +JSON_ARRAY(r.c) AS quoted; +SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; +SELECT r.a AS a, r.b AS b, r.c AS c, +JSON_TYPE(r.c) AS ty, +JSON_ARRAY(r.c) AS quoted_after_a_whole_assignment; +SELECT JSON_OBJECT('second', 2), 'not a number', JSON_OBJECT('fourth', 4) +INTO r; +SELECT 'the handler ran and the call went on' AS resumed; +END $$ +CREATE PROCEDURE p_row_fetch() +BEGIN +DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); +DECLARE cur CURSOR FOR +SELECT JSON_OBJECT('fifth', 5) AS a, 'not a number' AS b, +JSON_OBJECT('sixth', 6) AS c; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION +SELECT r.a AS written_by_this_fetch, +r.b AS the_member_it_stopped_at, +r.c AS left_by_the_assignment_before, +JSON_VALID(r.c) AS still_a_document, +JSON_ARRAY(r.c) AS quoted; +SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; +OPEN cur; +FETCH cur INTO r; +CLOSE cur; +END $$ +CALL p_row_into(); +a b c ty quoted_after_a_whole_assignment +{"first": 1} 1 {"third": 3} OBJECT ["{\"third\": 3}"] +written_by_this_assignment the_member_it_stopped_at left_by_the_assignment_before still_a_document ty quoted +{"second": 2} NULL {"third": 3} 1 OBJECT ["{\"third\": 3}"] +resumed +the handler ran and the call went on +CALL p_row_fetch(); +written_by_this_fetch the_member_it_stopped_at left_by_the_assignment_before still_a_document quoted +{"fifth": 5} NULL {"third": 3} 1 ["{\"third\": 3}"] +DROP PROCEDURE p_row_into; +DROP PROCEDURE p_row_fetch; +SET @@sql_mode=DEFAULT; diff --git a/mysql-test/main/func_json_sp_trust.test b/mysql-test/main/func_json_sp_trust.test new file mode 100644 index 0000000000000..a2614b5d80845 --- /dev/null +++ b/mysql-test/main/func_json_sp_trust.test @@ -0,0 +1,366 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # What a JSON function reads out of a stored program's variable. +--echo # +--echo # A variable is written by one funnel and by nothing else, and every +--echo # assignment to one arrives there with the expression that made the +--echo # value still in hand. So what was put in a variable can be said at +--echo # the moment it is put there, and a function later handed that +--echo # variable as the document to work on can be spared finding out +--echo # again what is in it. +--echo # +--echo # A variable is not typed as a document however it was declared, so +--echo # a value out of one is still quoted into a document rather than +--echo # spliced into it. That is decided by the type and is left alone +--echo # here. The answers below are the same either way - the reading +--echo # that is left out could only have found out what was already +--echo # known - so this file is here to say that they stay the same. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A variable assigned by a function that attests to its result, +--echo # then read as a document. +--echo # +--delimiter $$ +CREATE PROCEDURE p_read() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_SET('{"a":1}', '$.b', 2); + SELECT v AS held; + SELECT JSON_SET(v, '$.c', 3) AS edited; + SELECT JSON_INSERT(v, '$.c', 3) AS inserted; + SELECT JSON_REPLACE(v, '$.a', 9) AS replaced; + SELECT JSON_REMOVE(v, '$.a') AS removed; + SELECT JSON_ARRAY_APPEND(JSON_SET(v, '$.arr', JSON_ARRAY(1)), '$.arr', 2) + AS appended; + SELECT JSON_MERGE_PATCH(v, '{"d":4}') AS merged; + SELECT JSON_EXTRACT(v, '$.a') AS extracted; + SELECT JSON_TYPE(v) AS what, JSON_DEPTH(v) AS deep, JSON_LENGTH(v) AS len; + SELECT JSON_VALID(v) AS valid, JSON_KEYS(v) AS keys_of; +END $$ +--delimiter ; +CALL p_read(); +DROP PROCEDURE p_read; + +--echo # +--echo # 2. The same value spliced into a new document. A variable is not +--echo # typed as a document, so it is quoted - which is what a released +--echo # server does with it and what it goes on doing. +--echo # +--delimiter $$ +CREATE PROCEDURE p_splice() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE j JSON DEFAULT NULL; + SET v= JSON_SET('{"a":1}', '$.b', 2); + SET j= JSON_SET('{"a":1}', '$.b', 2); + SELECT JSON_ARRAY(v) AS text_var, JSON_ARRAY(j) AS json_var; + SELECT JSON_OBJECT('k', v) AS obj_text, JSON_OBJECT('k', j) AS obj_json; + SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; +END $$ +--delimiter ; +CALL p_splice(); +DROP PROCEDURE p_splice; + +--echo # +--echo # 3. One variable assigned from another. The assignment reaches the +--echo # funnel as a read of the field behind the source, so what was said +--echo # about the value travels with it and does not have to be said +--echo # again. +--echo # +--delimiter $$ +CREATE PROCEDURE p_var_to_var() +BEGIN + DECLARE a LONGTEXT DEFAULT NULL; + DECLARE b LONGTEXT DEFAULT NULL; + SET a= JSON_SET('{"a":1}', '$.b', 2); + SET b= a; + SELECT JSON_SET(b, '$.c', 3) AS edited; + SET a= 'not a document at all'; + SELECT JSON_SET(b, '$.d', 4) AS still_edited; + SELECT b AS unchanged; +END $$ +--delimiter ; +CALL p_var_to_var(); +DROP PROCEDURE p_var_to_var; + +--echo # +--echo # 4. A variable that is both the source and the target. +--echo # +--delimiter $$ +CREATE PROCEDURE p_self() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_SET(v, '$.b', 2); + SET v= JSON_SET(v, '$.c', 3); + SELECT v AS built; + SET v= v; + SELECT JSON_SET(v, '$.d', 4) AS after_self_assign; +END $$ +--delimiter ; +CALL p_self(); +DROP PROCEDURE p_self; + +--echo # +--echo # 5. Assignments that put something else there. A literal is not +--echo # attested by anybody, and a value the store had to change is +--echo # not the value that was attested. +--echo # +--delimiter $$ +CREATE PROCEDURE p_untrusted() +BEGIN + DECLARE lit LONGTEXT DEFAULT NULL; + DECLARE short_v VARCHAR(8) DEFAULT NULL; + DECLARE num INT DEFAULT NULL; + SET lit= '{"a":1}'; + SELECT JSON_SET(lit, '$.b', 2) AS from_literal; + SELECT JSON_TYPE(lit) AS what; + SET num= JSON_LENGTH(JSON_SET('{"a":1}', '$.b', 2)); + SELECT JSON_SET(CONCAT('[', num, ']'), '$[1]', 9) AS from_number; + BEGIN + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET short_v= JSON_SET('{"a":1}', '$.b', 2); + END; + SELECT short_v AS truncated; + SELECT JSON_VALID(short_v) AS valid_after_truncation; +END $$ +--delimiter ; +CALL p_untrusted(); +DROP PROCEDURE p_untrusted; + +--echo # +--echo # 6. An assignment that does not happen. What the one before it +--echo # left standing goes with it, the bytes under it being whatever the +--echo # store got as far as putting down. +--echo # +--delimiter $$ +CREATE PROCEDURE p_failed() +BEGIN + DECLARE v VARCHAR(20) DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET v= JSON_SET('{"a":1}', '$.b', 2); + SELECT JSON_SET(v, '$.c', 3) AS edited; + SET v= JSON_SET('{"aaaaaaaaaa":1}', '$.bbbbbbbbbb', 2); + SELECT v AS after_failure; + SELECT JSON_VALID(v) AS valid_after_failure; +END $$ +--delimiter ; +CALL p_failed(); +DROP PROCEDURE p_failed; + +--echo # +--echo # 7. Row variables. A row is written a member at a time whichever +--echo # form the assignment takes, each member going through the same +--echo # funnel a plain variable goes through, so each of them is answered +--echo # for on its own and by whatever made the value that landed in it. +--echo # +--delimiter $$ +CREATE PROCEDURE p_row() +BEGIN + DECLARE r ROW(a LONGTEXT, b LONGTEXT); + SET r.a= JSON_SET('{"x":1}', '$.y', 2); + SET r.b= JSON_SET('[1]', '$[1]', 2); + SELECT JSON_SET(r.a, '$.z', 3) AS member_a; + SELECT JSON_ARRAY_APPEND(r.b, '$', 3) AS member_b; + SET r= ROW(JSON_OBJECT('p', 1), JSON_ARRAY(7, 8)); + SELECT JSON_SET(r.a, '$.q', 2) AS whole_row_a; + SELECT JSON_ARRAY_APPEND(r.b, '$', 9) AS whole_row_b; + SET r= ROW('{"lit":1}', '[0]'); + SELECT JSON_SET(r.a, '$.q', 2) AS literal_row_a; +END $$ +--delimiter ; +CALL p_row(); +DROP PROCEDURE p_row; + +--echo # +--echo # 8. A row assignment that fails part of the way through. The +--echo # members it did not reach are still holding what the assignment +--echo # before this one left, so their answers go; the ones it wrote keep +--echo # theirs, having been attested on the way in. +--echo # +--delimiter $$ +CREATE PROCEDURE p_row_partial() +BEGIN + DECLARE r ROW(a VARCHAR(20), b VARCHAR(8)); + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET r.a= JSON_SET('{"x":1}', '$.y', 2); + SET r.b= JSON_SET('[1]', '$[1]', 2); + SELECT JSON_SET(r.a, '$.z', 3) AS before_a; + SET r= ROW(JSON_OBJECT('p', 1), JSON_OBJECT('qqqqqqqqqq', 22222)); + SELECT r.a AS after_a, r.b AS after_b; + SELECT JSON_VALID(r.a) AS valid_a, JSON_VALID(r.b) AS valid_b; +END $$ +--delimiter ; +CALL p_row_partial(); +DROP PROCEDURE p_row_partial; + +--echo # +--echo # 9. A value that arrives from a table rather than from a function, +--echo # by SELECT INTO and by FETCH. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); + +--delimiter $$ +CREATE PROCEDURE p_into() +BEGIN + DECLARE done INT DEFAULT 0; + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE c CURSOR FOR SELECT j FROM t1 ORDER BY id; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done= 1; + SELECT j INTO v FROM t1 WHERE id= 1; + SELECT JSON_SET(v, '$.b', 2) AS from_select_into; + SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 2; + SELECT JSON_ARRAY_APPEND(v, '$', 3) AS from_extract_into; + OPEN c; + read_loop: LOOP + FETCH c INTO v; + IF done THEN + LEAVE read_loop; + END IF; + SELECT JSON_TYPE(v) AS fetched_type, JSON_DEPTH(v) AS fetched_depth; + END LOOP; + CLOSE c; +END $$ +--delimiter ; +CALL p_into(); +DROP PROCEDURE p_into; + +--echo # +--echo # 10. Through a stored function, whose parameter is a variable of the +--echo # frame the call makes and whose result is not one at all. +--echo # +--delimiter $$ +CREATE FUNCTION f_edit(doc LONGTEXT) RETURNS LONGTEXT +BEGIN + RETURN JSON_SET(doc, '$.f', 1); +END $$ +--delimiter ; +SELECT f_edit(JSON_OBJECT('a', 1)) AS called_with_document; +SELECT f_edit('{"a":1}') AS called_with_literal; +SELECT JSON_SET(f_edit(JSON_OBJECT('a', 1)), '$.g', 2) AS over_the_result; +SELECT JSON_ARRAY(f_edit(JSON_OBJECT('a', 1))) AS spliced_result; +SELECT f_edit(j) AS over_a_column FROM t1 ORDER BY id; +DROP FUNCTION f_edit; + +--echo # +--echo # 11. Values the declared type has to change on the way in: a +--echo # character set that can hold every character of the document, and +--echo # one that cannot. +--echo # +--delimiter $$ +CREATE PROCEDURE p_charset() +BEGIN + DECLARE l1 VARCHAR(64) CHARACTER SET latin1 DEFAULT NULL; + DECLARE u VARCHAR(64) CHARACTER SET utf8mb4 DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET u= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); + SELECT JSON_SET(u, '$.c', 'y') AS utf8_var; + SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); + SELECT JSON_SET(l1, '$.c', 'y') AS latin1_var; + SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xC3A4); + SELECT HEX(l1) AS latin1_converted; + SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xE4B8AD); + SELECT HEX(l1) AS latin1_lossy; + SELECT JSON_VALID(l1) AS valid_after_loss; +END $$ +--delimiter ; +CALL p_charset(); +DROP PROCEDURE p_charset; + +--echo # +--echo # 12. A declared type that changes nothing on the way in and still +--echo # leaves something else there. A store into the binary type keeps +--echo # the bytes and calls them by another name, so a document written +--echo # in a wide set arrives as bytes that read as no document at all. +--echo # Once for a conversion written out in the assignment, and once for +--echo # a variable already holding the wide encoding. +--echo # +--delimiter $$ +CREATE PROCEDURE p_binary() +BEGIN + DECLARE b BLOB DEFAULT NULL; + DECLARE w VARCHAR(64) CHARACTER SET ucs2 DEFAULT NULL; + SET b= CONVERT(JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z') + USING ucs2); + SELECT HEX(b) AS binary_converted, JSON_VALID(b) AS valid_converted; + SET w= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); + SELECT JSON_VALID(w) AS valid_wide; + SET b= w; + SELECT HEX(b) AS binary_from_wide, JSON_VALID(b) AS valid_from_wide; +END $$ +--delimiter ; +CALL p_binary(); +DROP PROCEDURE p_binary; + +DROP TABLE t1; + +--echo # +--echo # 13. A row variable written from a list of values rather than from +--echo # a row. Section 8 assigns a whole row at once; SELECT INTO and +--echo # FETCH INTO write the members one at a time from a list and stop at +--echo # the first that fails. What the variable holds afterwards is a +--echo # mixture of three things: the members before the failure carry what +--echo # this assignment wrote, the one it stopped at is left holding +--echo # nothing, and the ones after it still carry what the assignment +--echo # before left. Anything said about the last two belongs to an +--echo # assignment that did not finish, and goes with it. +--echo # +--echo # A variable is never typed as a document, so a value read out of +--echo # one is quoted into a new document rather than spliced into it, in +--echo # both states - which is what the whole-assignment case is printed +--echo # for. What the answers show is that reading the mixture says the +--echo # same things about it as reading a variable nothing went wrong +--echo # with. +--echo # +SET @@sql_mode='STRICT_ALL_TABLES'; + +--delimiter $$ +CREATE PROCEDURE p_row_into() +BEGIN + DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION + SELECT r.a AS written_by_this_assignment, + r.b AS the_member_it_stopped_at, + r.c AS left_by_the_assignment_before, + JSON_VALID(r.c) AS still_a_document, + JSON_TYPE(r.c) AS ty, + JSON_ARRAY(r.c) AS quoted; + SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; + SELECT r.a AS a, r.b AS b, r.c AS c, + JSON_TYPE(r.c) AS ty, + JSON_ARRAY(r.c) AS quoted_after_a_whole_assignment; + SELECT JSON_OBJECT('second', 2), 'not a number', JSON_OBJECT('fourth', 4) + INTO r; + SELECT 'the handler ran and the call went on' AS resumed; +END $$ + +CREATE PROCEDURE p_row_fetch() +BEGIN + DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); + DECLARE cur CURSOR FOR + SELECT JSON_OBJECT('fifth', 5) AS a, 'not a number' AS b, + JSON_OBJECT('sixth', 6) AS c; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION + SELECT r.a AS written_by_this_fetch, + r.b AS the_member_it_stopped_at, + r.c AS left_by_the_assignment_before, + JSON_VALID(r.c) AS still_a_document, + JSON_ARRAY(r.c) AS quoted; + SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; + OPEN cur; + FETCH cur INTO r; + CLOSE cur; +END $$ +--delimiter ; + +CALL p_row_into(); +CALL p_row_fetch(); + +DROP PROCEDURE p_row_into; +DROP PROCEDURE p_row_fetch; +SET @@sql_mode=DEFAULT; diff --git a/sql/field.cc b/sql/field.cc index 1878d22c1acfd..a497b15c02e0f 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1510,6 +1510,7 @@ bool Field::sp_prepare_and_store_item(THD *thd, Item **value) DBUG_ASSERT(value); Item *expr_item; + int store_rc; if (!(expr_item= thd->sp_fix_func_item_for_assignment(this, value))) goto error; @@ -1519,10 +1520,18 @@ bool Field::sp_prepare_and_store_item(THD *thd, Item **value) /* Save the value in the field. Convert the value if needed. */ - expr_item->save_in_field(this, 0); + store_rc= expr_item->save_in_field(this, 0); if (likely(!thd->is_error())) + { + /* + The one place a stored program's variable is written, and so the + one place there is anything to say about what is in it - see + Field::set_json_held_marks(). + */ + set_json_held_marks(expr_item, store_rc); DBUG_RETURN(false); + } error: /* @@ -1532,6 +1541,13 @@ bool Field::sp_prepare_and_store_item(THD *thd, Item **value) set x = x + 1; */ set_null(); + /* + The assignment failed, so nothing attests to what this field holds: + the bytes are whatever the store put down before the error, under a + NULL that was set without clearing them. The marks the previous + assignment left go with them. + */ + clear_json_held_marks(); DBUG_ASSERT(thd->is_error()); DBUG_RETURN(true); } @@ -3058,6 +3074,106 @@ void Field::confirm_is_valid_json_static_from(Field *from) } +/* + What this field is holding, where it is a stored program's variable and + something was said about the value put here - see TABLE::json_held_marks. + + A field with nowhere to keep an answer answers the same as one that was + never granted anything, which is the answer everything but a stored + program's variable gives and the answer those give until an assignment + says otherwise. +*/ + +bool Field::is_valid_json_held() const +{ + return table->json_held_marks && + table->json_held_marks[field_index].valid(); +} + + +bool Field::is_nice_json_held() const +{ + return table->json_held_marks && + table->json_held_marks[field_index].nice(); +} + + +uint Field::json_held_depth() const +{ + return table->json_held_marks ? table->json_held_marks[field_index].depth() + : JSON_DEPTH_UNKNOWN; +} + + +void Field::clear_json_held_marks() +{ + if (table->json_held_marks) + table->json_held_marks[field_index].clear(); +} + + +/* + Said once per assignment, with the item that was assigned in hand and + the store it just did already done. + + The rule is the one every store site applies - the item said it was a + document, the store kept every character of it, and nothing went wrong + while it happened - and it is applied here for the same reason it is + applied there. What is different is what the answer is about: a + column is attested once and read over many rows, while a variable + is attested as often as it is written and read only until it is + written again. So the formatting and the depth are taken from the item + as well, exactly rather than as a bound: there is one value here, the + item that made it is right here, and it is the only value the answer + has to cover. + + The store having returned 0 is asked about and not merely whether + anything went wrong, because in this direction the two are not the + same: a store that had to drop characters or write them another way + answers 2, and answers it while succeeding. Assigning a variable does + raise an error where a column store would not, but that is a property + of this path rather than of the rule, and the rule is the conservative + one either way. + + Nothing is cleared on the way in. Every way out of the funnel this is + called from ends either here or at the clear beside it, so a value + that never arrived leaves no answer standing over the bytes of the one + before it. Not clearing first is also what lets SET x = x keep an + answer: the item is this same field, and what it says is read after + the store rather than wiped before it. +*/ + +void Field::set_json_held_marks(const Item *item, int store_rc) +{ + if (!table->json_held_marks) + return; + Json_result_marks &marks= table->json_held_marks[field_index]; + /* + The same rule a column's store is attested by - see + Field::is_attestation_preserved() - and a NULL besides, which is + not read as a document and is not attested as one. + */ + if (!is_attestation_preserved(item, store_rc) || is_null()) + { + marks.clear(); + return; + } + { + /* + The value is read back out of the field rather than taken from the + item, so that what the answer is checked against is what a reader + of this field will find. Only the check reads it - see + Json_result_marks::set(), which uses the value for nothing else - + so the reading is the debug build's work, the same as at the + confirm above. + */ + IF_DBUG(StringBuffer kept;,) + marks.set(IF_DBUG(val_str(&kept), NULL), true, item->is_nice_json(), + item->last_depth()); + } +} + + int Field::set_default() { if (default_value) diff --git a/sql/field.h b/sql/field.h index 862334c7003b1..71ae3d707e700 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1309,34 +1309,63 @@ class Field: public Value_source void confirm_is_valid_json_static_from(Field *from); void confirm_json_static_value(bool is_nice, uint depth); + /* + Attest to the one value this field is holding, where the field is a + stored program's variable - see TABLE::json_held_marks. Out of line + for the same reason the depth above is: what the answers are kept in + is declared where a field header cannot see it. + + Read without asking first whether there is anything here to read: a + field with nowhere to keep an answer gives the same answer as a + field that has one and was not granted it, and no caller has ever + had a reason to tell those two apart. + */ + bool is_valid_json_held() const; + bool is_nice_json_held() const; + uint json_held_depth() const; + void set_json_held_marks(const Item *item, int store_rc); + void clear_json_held_marks(); /* - What this field can say about the bytes it is holding. The long - form of why it can be believed is at Item_field::is_valid_json(). + What this field can say about the bytes it is holding, which is + whichever of the two channels above has anything to say. Only one + of them is ever there to be asked: a table cannot both be one the + server built for a query and be where a stored program keeps its + variables, so the two never disagree. The long form of why either + of them can be believed is at Item_field::is_valid_json(). Asked by everything that reads a value through a field rather than through the item that made it - Item_field for the field it names, and the Item_ref family for the result field it reads a copy out of. - A row holding no value at all is not attested, and that is asked - here rather than left to the reader. The grant is about the column: - it is given once, where the table is built, and what it says is that - every value the column comes to hold arrived from something that - attests to the values it makes. That is as true of a row that came - out SQL NULL as of any other, there being no value in it to be wrong - about - so the grant goes on standing over such a row while there - are no characters under it to have been attested. A caller that - reads it there puts the answer down somewhere the NULL itself cannot - go, the column taking whatever it takes in place of one, and the - answer is then about bytes nothing ever saw. + A row holding no value at all is attested by neither, and that + is asked here rather than left to the reader. The standing grant is + about the column: it is given once, where the table is built, and + what it says is that every value the column comes to hold arrived + from something that attests to the values it makes. That is as + true of a row that came out SQL NULL as of any other, there being no + value in it to be wrong about - so the grant goes on standing over + such a row while there are no characters under it to have been + attested. A caller that reads it there puts the answer down + somewhere the NULL itself cannot go, the column taking whatever it + takes in place of one, and the answer is then about bytes nothing + ever saw. The held answers are put down one value at a time and are + already cleared for a NULL where they are set - see + Field::set_json_held_marks() - so the question is asked of both only + so that a reader never has to know which of the two it is reading. */ bool attests_is_valid_json() const - { return !is_null() && is_valid_json_static(); } + { return !is_null() && (is_valid_json_static() || is_valid_json_held()); } bool attests_is_nice_json() const - { return !is_null() && is_nice_json_static(); } + { return !is_null() && (is_nice_json_static() || is_nice_json_held()); } + /* + The smaller of the two, which is the one that was answered: the + other is JSON_DEPTH_UNKNOWN, that being the largest there is and + what a field with nothing to say about a depth says. + */ uint attested_json_depth() const - { return json_static_depth(); } + { return MY_MIN(json_static_depth(), json_held_depth()); } virtual my_time_t get_timestamp(const uchar *pos, ulong *sec_part) const { DBUG_ASSERT(0); return 0; } diff --git a/sql/item.cc b/sql/item.cc index abd9697cf2ed1..cece67b607b30 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -7193,6 +7193,10 @@ void Item_field::make_send_field(THD *thd, Send_field *tmp_field) */ static bool field_reads_back_as_document(Field *f) { + /* + The two channels read the way is_valid_json() reads them, and not by + calling it: what it does before answering is ask for this. + */ if (!f->attests_is_valid_json()) return true; { diff --git a/sql/item.h b/sql/item.h index 3ad5ab8b4b8e5..539062f8a69e6 100644 --- a/sql/item.h +++ b/sql/item.h @@ -3397,6 +3397,37 @@ class Item_sp_variable :public Item_fixed_hybrid bool is_null() override; public: + /* + What the variable this stands for is holding, asked of the field the + value is in - see TABLE::json_held_marks. + + Asked through this_item(), which finds the running frame the way + every other read of the variable does and never remembers what it + found. The same item is called upon twice over, from both sides: + reading the variable comes through here, and assigning it to another + one arrives at the store as that same field, dereferenced on the way + by THD::sp_fix_func_item(). So one answer serves both, and a value + moved from variable to variable carries what was said about it + without anything having to move it. + + A variable of a stored program is not typed as a document however it + was declared, so a value out of one is quoted into a document rather + than spliced into it, and these three answers change nothing about + that. What they are read for is the other thing attestation saves: + a function given this value as the document to work on need not + parse it again. + + The standing question is not answered here and cannot be. A + variable holds what the last assignment put in it and the next one + will put something else there, so nothing about the value in it now + holds for every evaluation of it. + */ + bool is_valid_json() const override + { return this_item()->is_valid_json(); } + bool is_nice_json() const override + { return this_item()->is_nice_json(); } + uint last_depth() const override + { return this_item()->last_depth(); } void make_send_field(THD *thd, Send_field *field) override; bool const_item() const override { return true; } Field *create_tmp_field_ex(MEM_ROOT *root, @@ -4155,12 +4186,31 @@ class Item_field :public Item_ident, is an upper bound on what it says by the time the row arrives, and the later column asks it again at every fill. See Field::confirm_is_valid_json_static_from(). + + The other exception is a field a stored program keeps a variable in, + which was told what was put there by the assignment that put it - + see TABLE::json_held_marks. That is a different promise about a + different thing and it is asked here for one reason: an assignment + of one variable to another arrives at the store as a read of the + field behind the source, this item and no other, so a variable + attesting to its value at all means this item attesting to it. + + Only one of the two is ever there to be asked. A table cannot both + be built by the server for a query and be where a stored program + keeps its variables, so the two never disagree and nothing has to + decide which of them wins. */ bool is_valid_json() const override { DBUG_ASSERT(reads_back_as_document()); return field->attests_is_valid_json(); } + /* + The standing question, which only the first of the two answers. A + variable is written as many times as somebody assigns it, so what is + in it now says nothing about what will be in it when a column built + out of it comes to be filled. + */ bool is_valid_json_static() const override { return field->is_valid_json_static(); } /* @@ -4193,6 +4243,11 @@ class Item_field :public Item_ident, uint last_depth() const override { DBUG_ASSERT(reads_back_no_deeper_than_claimed()); + /* + Where a variable is what is being read the figure is the depth of + the one value in it rather than the deepest of a column's rows, + and so is exact. + */ return field->attested_json_depth(); } /* diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 056d299f0affb..114ebf9112d13 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -3566,10 +3566,17 @@ static int append_json_value(String *str, Item *item, String *tmp_val, /* Being a document is what gets a value spliced rather than quoted, - so an item attesting to one it is not going to be asked about has - lost track of which of the two it returns. + and an item can attest to a value that is still quoted. A stored + program's variable is the case: it is never typed as a document, + however it was declared, and it attests to its value all the same + - see Item_sp_variable::is_valid_json(), which is answered so that + a function handed the value as its DOCUMENT can stop reading what + it already knows, and not so that this one splices it. + + Nothing here reads the answer, and quoting a value that is a + document is what quoting any other value is: the characters are + written out escaped and the quotes go round them. */ - DBUG_ASSERT(!item->is_valid_json()); rc= append_escaped_value(str, sv, item->result_type() == STRING_RESULT, fname, n_param); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 3f68ce970ad59..f4428ca10fcc9 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -21954,6 +21954,37 @@ bool Virtual_tmp_table::init(uint field_count) }; +/* + Where a stored program's variables keep what was said about the values + in them, and the only place one is left - see TABLE::json_held_marks. + + Given to the tables that hold such variables and to no others. Three + callers build a table through the init() above, and only one of them + builds one out of a stored program's variable definitions - the other + two are the table SUM(DISTINCT) counts in and the one row-based + replication converts through, neither of which holds anything anybody + assigns to. Both ways a stored program gets one come through that + caller, the row a cursor is read into among them, so there is no table + of this kind that has to be given room later. + + Said to be nothing before any of it is read, this being room off a + memory root and not a fresh page. +*/ + +bool Virtual_tmp_table::init_json_held_marks(uint field_count) +{ + DBUG_ENTER("Virtual_tmp_table::init_json_held_marks"); + if (!(json_held_marks= (Json_result_marks *) + alloc_root(in_use->mem_root, + field_count * + sizeof(Json_result_marks)))) + DBUG_RETURN(true); + for (uint i= 0; i < field_count; i++) + json_held_marks[i].clear(); + DBUG_RETURN(false); +} + + bool Virtual_tmp_table::add(List &field_list) { /* Create all fields and calculate the total length of record */ @@ -22076,7 +22107,19 @@ bool Virtual_tmp_table::sp_set_all_fields_from_item_list(THD *thd, for (uint i= 0 ; (item= it++) ; i++) { if (field[i]->sp_prepare_and_store_item(thd, &item)) + { + /* + The members after the one that failed were never reached, so what + they hold is what the assignment before this one left and what + was said about it then - an answer about to be read beside + members this assignment did write. Those go with the assignment + that did not finish. The ones already written keep theirs: each + of them went through the funnel and was attested to there, a + member being attested to one at a time. + */ + clear_json_held_marks_from(i); return true; + } } return false; } @@ -22089,7 +22132,11 @@ bool Virtual_tmp_table::sp_set_all_fields_from_item(THD *thd, Item *value) for (uint i= 0; i < value->cols(); i++) { if (field[i]->sp_prepare_and_store_item(thd, value->addr(i))) + { + /* A half-written row, as above. */ + clear_json_held_marks_from(i); return true; + } } return false; } diff --git a/sql/sql_select.h b/sql/sql_select.h index 1c607883be91e..317bef682553b 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2275,6 +2275,17 @@ class Virtual_tmp_table: public TABLE */ bool init(uint field_count); + /** + Give the table room to keep what was said about the value in each + of its members - see TABLE::json_held_marks. Only for a table that + holds a stored program's variables; the others init() builds hold + nothing anybody assigns to. + @param field_count - The number of fields we plan to add to the table. + @returns false - on success. + @returns true - on error. + */ + bool init_json_held_marks(uint field_count); + /** Add one Field to the end of the field array, update members: s->reclength, s->fields, s->blob_fields, s->null_fuelds. @@ -2318,10 +2329,25 @@ class Virtual_tmp_table: public TABLE */ bool open(); + /* + What was said about the value in each member, given up together. + Whatever writes a whole row at once writes it outside the funnel a + member is otherwise written through, and a member nobody wrote this + time is still holding what the last assignment left - so an answer + that was about that is taken back here rather than left standing + over bytes a later reader will pair it with. + */ + void clear_json_held_marks_from(uint first) + { + for (uint i= first; i < s->fields; i++) + field[i]->clear_json_held_marks(); + } + void clear_all_json_held_marks() { clear_json_held_marks_from(0); } void set_all_fields_to_null() { for (uint i= 0; i < s->fields; i++) field[i]->set_null(); + clear_all_json_held_marks(); } /** Set all fields from a compatible item list. @@ -2400,6 +2426,7 @@ create_virtual_tmp_table(THD *thd, List &field_list) DBUG_SET("+d,simulate_out_of_memory");); if (table->init(field_list.elements) || + table->init_json_held_marks(field_list.elements) || table->add(field_list) || table->open()) { diff --git a/sql/table.h b/sql/table.h index 4e58dd77eb078..90afa16549bf9 100644 --- a/sql/table.h +++ b/sql/table.h @@ -76,6 +76,7 @@ class Table_triggers_list; class TMP_TABLE_PARAM; class SEQUENCE; class Range_rowid_filter_cost_info; +class Json_result_marks; class derived_handler; class Pushdown_derived; struct Name_resolution_context; @@ -1455,6 +1456,35 @@ struct TABLE documents at all. */ uint *json_static_depth; + /* + What was said about the value each field of this table is holding + right now, where the table is one a stored program keeps its + variables in. + + A variable is not a column and cannot be attested to the way the + three above attest to one. A column of a table the server built + for itself has one producer, settled while the column is being made, + so what will be written there is known before a row exists. A + variable has as many producers as there are assignments to it, each + one arriving at a different moment and saying something different, + so there is nothing to say in advance and everything to say each + time one lands. + + So these are written where a value is written, by the one funnel + every assignment goes through, and each of them is about the bytes + the field holds until the next assignment replaces both together. + That is a narrower promise than the standing one and a stronger one: + it is about a value that exists, and it can carry the formatting and + the depth exactly rather than as a bound over rows. + + Only Virtual_tmp_table leaves an array here, and it is the whole of + the discriminator, exactly as the pointer above it is: a base table + and a table the user asked for have nowhere to keep an answer and so + give none. A table a stored program keeps its variables in cannot + be named from SQL and is written by nothing but that one funnel, + which is what makes an answer about it worth keeping at all. + */ + Json_result_marks *json_held_marks; /* The ID of the query that opened and is using this table. Has different From 1dd9b5e85175387114edd1c1adda7dafb9cc01d2 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Sat, 8 Aug 2026 18:14:41 -0400 Subject: [PATCH 21/30] MDEV-40609 JSON_KEYS gives an array, so put an array in The keys of an object are returned as a JSON array, and the documentation says as much, but the function was declared `Item_str_func` rather than `Item_json_func`. So `is_json_type()` answered false for it, and every function given it in value position escaped the array into a string: JSON_ARRAY(JSON_KEYS('{"a":1,"b":2}')) -> ["[\"a\", \"b\"]"] where the members of the same family that ARE typed put the same shape of value in as a document: JSON_ARRAY(JSON_EXTRACT('{"a":[1,2]}','$.a')) -> [[1, 2]] It is the only function in this file documented to return a document and left out of `Item_json_func`. Deriving from it is the whole of the change; the value it writes is untouched. The array itself was never in doubt. Read as a document it always was one - `JSON_VALID` 1, `JSON_LENGTH` 2, `JSON_TYPE` ARRAY - and the workaround for the mismatch said so: `JSON_EXTRACT(JSON_KEYS(...), '$')` parsed the array back out of itself before it could be put in. Document position is unchanged and so is that workaround, which now reaches the same document by the longer route. ## The answer is attested, and by the writing not by measuring Typing it is only half of what a splice needs. A caller splicing a document reads it again unless whatever wrote it says what it is, so the type alone would have left every splice of an array of key names parsing it back out of itself - which is the reading the type exists to make unnecessary. All three things there are to say about the answer are settled by how it was written: A name goes in as it was written in the document it came from, escapes and all - a key written `a\"b` there is written `a\"b` here - out of an object that was read to its end, so whatever needed escaping in it is escaped already and a name that would not go into an array would not have come out of an object. Read to its end is the whole of what is needed, and it is not the same as the argument having parsed: this reads one object and stops where that object closes, so text standing beyond it is never met, and JSON_KEYS('{"a":1} rubbish') ["a"] JSON_KEYS('{"a":1}{"b":2}') ["a"] JSON_KEYS('{"a":1}}') ["a"] JSON_KEYS('{"a":{"z":1} "b":2}', '$.a') ["z"] all answer, while a break inside the object refuses the answer whole and says where it was: JSON_KEYS('{"a":1 "b":2}') NULL, position 8 JSON_KEYS('{"a":{"z":1 "y":2}}', '$.a') NULL, position 13 Between two names goes the pair of characters the loose formatting puts there, written from the constant that formatting is defined by rather than from a copy of it, and nothing goes after the opening bracket or before the closing one, so the answer is formatted the way reading it back would write it. Nothing is wrong until the two disagree, so it is measured that way: with the constant temporarily made comma-tab, `JSON_KEYS('{"a":1,"b":2}')` answers its two keys separated by a comma and a tab, where a hand-written separator stops a debug build on the niceness half of the self-check. And an array of strings is one level deep, a key name being a string whatever it holds. The character set is the argument's, `fix_length_and_dec()` taking the collation from it and the copy being same-set, so the one thing left to ask is whether a document can be written in that set at all. That guard is written as `JSON_SEARCH`'s is, and it is a live one. A set with no bracket to write has no braces either and never parses a document to take keys off, but that is not the question the guard asks: `is_json_compatible_charset()` asks whether the set is ASCII-compatible, and a set can decode every ASCII byte to itself and still be flagged as not. `sjis` is one. It parses a document, the answer is a real array, and the mark is withheld from it: SELECT JSON_KEYS(j) FROM ts; -- sjis ["a", "b"] -- JSON_VALID says 1 SELECT JSON_ARRAY(JSON_KEYS(j)) FROM ts; Json_scans 2 SELECT JSON_ARRAY(JSON_KEYS(j)) FROM tu; Json_scans 1 so a splice there reads again exactly the array the mark exists to stop it reading. Nothing computed is wrong - an underclaimed mark costs a reading and never an answer. Those two figures are `func_json_keys_typed_scans`, which sources `have_debug.inc`: `Json_scans` is a debug build's own bookkeeping and is registered nowhere else, so a file that reads it runs wherever it is built and answers only where the counter is there to answer. `is_valid_json_static()` is granted for the same reason. Every row is an array of names or it is NULL, neither of which has to be read, and that is what a temporary table asks as it is built - which is before a row of it exists. ## What does not move `MAX()` over it still quotes, being typed by nothing it is taken over. `MAX(JSON_EXTRACT(...))` does the same and always has; it is not this. `JSON_QUOTE` is left as it is. Its documentation calls the result a string, so implementation and documentation already agree, and typing it would make the quoting it exists to do a no-op in the one position it was meant for. New test `func_json_keys_typed`, which records value position in the constructors and the mutators, document position beside it, the keys that need an escape, the two-argument form, an aggregate and a temporary table, and the result metadata that decides all of it. Its section 12 puts both character sets side by side, the one that writes a document and the one that parses a document and is flagged all the same; its section 13 records the boundary of the reading from both sides, the four shapes that answer and the two that refuse. What is attested shows in no result at all, only in readings not made, so it is counted rather than printed. `func_json_scan_count` gains four rows whose key names are long enough that the length says more levels than a document is allowed, leaving the depth the only thing keeping the answer from being read again; two of the four are the answers that are not arrays of names, an object with no keys and a document that is not an object. Both forms of the call go from seven readings to four, the four being the function's own. `func_json_tmp_scan_count` puts the same names through a temporary table, where the standing promise is what answers, and four readings become two - beside a control running the same query with the store into that column arranged to come up short, which reads four whether the grant is there or not, so the difference between the two is the column and nothing else. --- mysql-test/main/func_json_charset.result | 2 +- mysql-test/main/func_json_keys_typed.result | 327 ++++++++++++++++++ mysql-test/main/func_json_keys_typed.test | 205 +++++++++++ .../main/func_json_keys_typed_scans.result | 32 ++ .../main/func_json_keys_typed_scans.test | 37 ++ mysql-test/main/func_json_scan_count.result | 36 ++ mysql-test/main/func_json_scan_count.test | 39 +++ .../main/func_json_tmp_scan_count.result | 59 +++- mysql-test/main/func_json_tmp_scan_count.test | 62 +++- sql/item_jsonfunc.cc | 37 +- sql/item_jsonfunc.h | 20 +- 11 files changed, 828 insertions(+), 28 deletions(-) create mode 100644 mysql-test/main/func_json_keys_typed.result create mode 100644 mysql-test/main/func_json_keys_typed.test create mode 100644 mysql-test/main/func_json_keys_typed_scans.result create mode 100644 mysql-test/main/func_json_keys_typed_scans.test diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result index aa56e5763a119..a0c1a10d649bb 100644 --- a/mysql-test/main/func_json_charset.result +++ b/mysql-test/main/func_json_charset.result @@ -500,7 +500,7 @@ v "a" SELECT JSON_KEYS('{"a":1}') AS v; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def v 253 28 5 Y 0 39 45 +def v 253 (format=json) 28 5 Y 0 39 45 v ["a"] SELECT JSON_COMPACT('{}') AS v; diff --git a/mysql-test/main/func_json_keys_typed.result b/mysql-test/main/func_json_keys_typed.result new file mode 100644 index 0000000000000..d422b0c26e77a --- /dev/null +++ b/mysql-test/main/func_json_keys_typed.result @@ -0,0 +1,327 @@ +# +# JSON_KEYS returns a JSON array, so a function given it in value +# position puts an array in rather than a string that reads as one. +# +SET NAMES utf8mb4; +# +# 1. Value position, in the functions that build a document. +# +SELECT JSON_ARRAY(JSON_KEYS('{"a":1,"b":2}')) AS v; +v +[["a", "b"]] +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"k": ["a", "b"]} +SELECT JSON_ARRAY(JSON_KEYS('{"a":1}'), JSON_KEYS('{"b":2,"c":3}')) AS v; +v +[["a"], ["b", "c"]] +SELECT JSON_OBJECT('one', JSON_KEYS('{"a":1}'), +'two', JSON_KEYS('{"b":2,"c":3}')) AS v; +v +{"one": ["a"], "two": ["b", "c"]} +# +# 2. Value position, in the functions that edit one. +# +SELECT JSON_SET('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"x": 1, "y": ["a", "b"]} +SELECT JSON_INSERT('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"x": 1, "y": ["a", "b"]} +SELECT JSON_REPLACE('{"x":1}', '$.x', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"x": ["a", "b"]} +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +[1, ["a", "b"]] +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +[["a", "b"], 1] +# +# 3. The same shape from the functions that were already typed as +# returning a document, for comparison. +# +SELECT JSON_ARRAY(JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; +v +[[1, 2]] +SELECT JSON_ARRAY(JSON_QUERY('{"a":[1,2]}','$.a')) AS v; +v +[[1,2]] +SELECT JSON_SET('{"x":1}', '$.y', JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; +v +{"x": 1, "y": [1, 2]} +# +# 4. Document position is unchanged: the value was always a document +# when it was read as one. +# +SELECT JSON_VALID(JSON_KEYS('{"a":1,"b":2}')) AS valid, +JSON_LENGTH(JSON_KEYS('{"a":1,"b":2}')) AS len, +JSON_TYPE(JSON_KEYS('{"a":1,"b":2}')) AS type; +valid len type +1 2 ARRAY +SELECT JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$[0]') AS v; +v +"a" +SELECT JSON_DEPTH(JSON_KEYS('{"a":{"b":1},"c":2}')) AS v; +v +2 +# the workaround that used to be needed still gives the same document +SELECT JSON_ARRAY(JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$')) AS v; +v +[["a", "b"]] +# +# 5. Keys that have to be written with an escape. The key goes in as +# it was written in the document it came from, so the array is +# still one however the key is written. +# +SELECT JSON_KEYS('{"a\\"b":1,"c\\\\d":2}') AS v; +v +["a\"b", "c\\d"] +SELECT JSON_VALID(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS ok; +ok +1 +SELECT JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS v; +v +[["a\"b", "c\\d"]] +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}'))) AS ok; +ok +1 +# an escaped control character and an escaped code point +SELECT JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}')) AS v; +v +[["a\nb", "c\u00e4d"]] +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}'))) AS ok; +ok +1 +# a key written in characters of its own rather than in escapes +SELECT HEX(JSON_ARRAY(JSON_KEYS(_utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) +AS h; +h +5B5B22C3A4222C2022C3B6225D5D +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS( +_utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) AS ok; +ok +1 +# an empty key +SELECT JSON_ARRAY(JSON_KEYS('{"":1,"b":2}')) AS v; +v +[["", "b"]] +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"":1,"b":2}'))) AS ok; +ok +1 +# +# 6. The answers that are not arrays. An object with no keys is an +# empty array; anything that is not an object is NULL, and a NULL +# goes in as a null. +# +SELECT JSON_ARRAY(JSON_KEYS('{}')) AS v; +v +[[]] +SELECT JSON_ARRAY(JSON_KEYS('[1,2]')) AS v; +v +[null] +SELECT JSON_ARRAY(JSON_KEYS('7')) AS v; +v +[null] +SELECT JSON_ARRAY(JSON_KEYS(NULL)) AS v; +v +[null] +SELECT JSON_OBJECT('k', JSON_KEYS('[1,2]')) AS v; +v +{"k": null} +# input that does not read as a document at all +SELECT JSON_ARRAY(JSON_KEYS('{"a":')) AS v; +v +[null] +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +# +# 7. The two-argument form, which reaches an object further in. +# +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +v +[["b", "c"]] +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +v +{"k": ["b", "c"]} +# a path that reaches nothing +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1}}', '$.zz')) AS v; +v +[null] +# +# 8. Nested one inside the other. +# +SELECT JSON_ARRAY(JSON_KEYS(JSON_OBJECT('a', 1, 'b', 2))) AS v; +v +[["a", "b"]] +SELECT JSON_KEYS(JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}'))) AS v; +v +["k"] +# +# 9. Over rows, including through an aggregate and through a +# temporary table. +# +CREATE TABLE t1 (id INT, j VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2,"c":3}'); +SELECT id, JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY id; +id v +1 [["a"]] +2 [["b", "c"]] +SELECT JSON_ARRAYAGG(JSON_KEYS(j)) AS v FROM t1; +v +[["a"],["b", "c"]] +SELECT JSON_OBJECTAGG(id, JSON_KEYS(j)) AS v FROM t1; +v +{"1":["a"], "2":["b", "c"]} +# materialised into a temporary table on the way out +SELECT JSON_ARRAY(k) AS v FROM (SELECT JSON_KEYS(j) AS k FROM t1) d ORDER BY v; +v +[["a"]] +[["b", "c"]] +# MAX() is not typed by what it is taken over, so its result is +# quoted here as it is for any other document producing function +SELECT JSON_ARRAY(MAX(JSON_KEYS(j))) AS v FROM t1 GROUP BY id ORDER BY v; +v +["[\"a\"]"] +["[\"b\", \"c\"]"] +SELECT JSON_ARRAY(MAX(JSON_EXTRACT(j, '$'))) AS v FROM t1 GROUP BY id ORDER BY v; +v +["{\"a\": 1}"] +["{\"b\": 2, \"c\": 3}"] +SELECT DISTINCT JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY v; +v +[["a"]] +[["b", "c"]] +# the column a result set of it makes, and the column the already +# typed function makes beside it +CREATE TABLE t2 AS SELECT JSON_KEYS(j) AS k FROM t1; +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `k` varchar(64) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT JSON_ARRAY(k) AS v FROM t2 ORDER BY v; +v +["[\"a\"]"] +["[\"b\", \"c\"]"] +DROP TABLE t2; +CREATE TABLE t3 AS SELECT JSON_EXTRACT(j, '$') AS k FROM t1; +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `k` varchar(128) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE t3; +DROP TABLE t1; +# +# 10. The result metadata, which is what decides all of the above. +# +SELECT JSON_KEYS('{"a":1,"b":2}') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 52 10 Y 0 39 45 +v +["a", "b"] +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 88 6 Y 0 39 45 +v +[1, 2] +SELECT JSON_QUOTE('a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 32 3 Y 128 39 45 +v +"a" +# +# 11. JSON_QUOTE is documented to return a string and still does. +# +SELECT JSON_ARRAY(JSON_QUOTE('a')) AS v; +v +["\"a\""] +SELECT JSON_OBJECT('k', JSON_QUOTE('a')) AS v; +v +{"k": "\"a\""} +# +# 12. The array is attested as well as typed, so a splice of it +# goes in without being read again - which no result here can +# show and Json_scans can, and does in func_json_scan_count. +# +# What stands in the way of that is a character set this +# cannot attest to, and there are two kinds of those. +# +# A set that cannot encode a bracket cannot encode a brace +# either, so no array is ever answered in it and the question +# never comes up - swe7, below. +# +# But the question actually asked is whether the set is +# ASCII-compatible, and a set can decode every ASCII byte to +# itself and still be flagged as not. sjis is one: it parses +# a document, answers a real array, and has the mark withheld +# from it all the same, so a splice reads that array again. +# The array is a perfectly good document either way; what is +# lost is the reading it was meant to save. +# +# The counts that say which of the two happened are in +# func_json_keys_typed_scans, which a debug build runs. +# +SELECT JSON_KEYS(CONVERT('{"a":1}' USING swe7)) AS no_answer; +no_answer +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 1 +SELECT JSON_VALID(CONVERT('{"a":1}' USING swe7)) AS nor_a_document; +nor_a_document +0 +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# sjis: a real answer, and one the mark is withheld from +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +SELECT JSON_KEYS(j) AS answered, JSON_VALID(JSON_KEYS(j)) AS ok FROM ts; +answered ok +["a", "b"] 1 +DROP TABLE ts; +# +# 13. What the answer is attested does not say the argument was +# a document all the way through. This reads an object and +# stops at its end, so a break past that end is never met - and +# one it does meet refuses the answer whole. +# +# never met: the break is past the object, or past the sub-object +# the path chose +SELECT JSON_KEYS('{"a":1} rubbish') AS trailing_text; +trailing_text +["a"] +SELECT JSON_KEYS('{"a":1}{"b":2}') AS second_document; +second_document +["a"] +SELECT JSON_KEYS('{"a":1}}') AS trailing_brace; +trailing_brace +["a"] +SELECT JSON_KEYS('{"a":{"z":1} "b":2}', '$.a') AS break_after_the_step; +break_after_the_step +["z"] +# and the answers are documents, which is what the mark says +SELECT JSON_VALID(JSON_KEYS('{"a":1} rubbish')) AS ok; +ok +1 +SELECT JSON_ARRAY(JSON_KEYS('{"a":1} rubbish')) AS spliced; +spliced +[["a"]] +# met: refused whole, and reported where it happened +SELECT JSON_KEYS('{"a":1 "b":2}') AS break_inside; +break_inside +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 8 +SELECT JSON_KEYS('{"a":') AS ends_early; +ends_early +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +SELECT JSON_KEYS('{"a":{"z":1 "y":2}}', '$.a') AS break_inside_the_step; +break_inside_the_step +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 13 diff --git a/mysql-test/main/func_json_keys_typed.test b/mysql-test/main/func_json_keys_typed.test new file mode 100644 index 0000000000000..1a2363903db1a --- /dev/null +++ b/mysql-test/main/func_json_keys_typed.test @@ -0,0 +1,205 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # JSON_KEYS returns a JSON array, so a function given it in value +--echo # position puts an array in rather than a string that reads as one. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Value position, in the functions that build a document. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_ARRAY(JSON_KEYS('{"a":1}'), JSON_KEYS('{"b":2,"c":3}')) AS v; +SELECT JSON_OBJECT('one', JSON_KEYS('{"a":1}'), + 'two', JSON_KEYS('{"b":2,"c":3}')) AS v; + +--echo # +--echo # 2. Value position, in the functions that edit one. +--echo # + +SELECT JSON_SET('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_INSERT('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_REPLACE('{"x":1}', '$.x', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_KEYS('{"a":1,"b":2}')) AS v; + +--echo # +--echo # 3. The same shape from the functions that were already typed as +--echo # returning a document, for comparison. +--echo # + +SELECT JSON_ARRAY(JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; +SELECT JSON_ARRAY(JSON_QUERY('{"a":[1,2]}','$.a')) AS v; +SELECT JSON_SET('{"x":1}', '$.y', JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; + +--echo # +--echo # 4. Document position is unchanged: the value was always a document +--echo # when it was read as one. +--echo # + +SELECT JSON_VALID(JSON_KEYS('{"a":1,"b":2}')) AS valid, + JSON_LENGTH(JSON_KEYS('{"a":1,"b":2}')) AS len, + JSON_TYPE(JSON_KEYS('{"a":1,"b":2}')) AS type; +SELECT JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$[0]') AS v; +SELECT JSON_DEPTH(JSON_KEYS('{"a":{"b":1},"c":2}')) AS v; +--echo # the workaround that used to be needed still gives the same document +SELECT JSON_ARRAY(JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$')) AS v; + +--echo # +--echo # 5. Keys that have to be written with an escape. The key goes in as +--echo # it was written in the document it came from, so the array is +--echo # still one however the key is written. +--echo # + +SELECT JSON_KEYS('{"a\\"b":1,"c\\\\d":2}') AS v; +SELECT JSON_VALID(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS ok; +SELECT JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS v; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}'))) AS ok; +--echo # an escaped control character and an escaped code point +SELECT JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}')) AS v; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}'))) AS ok; +--echo # a key written in characters of its own rather than in escapes +SELECT HEX(JSON_ARRAY(JSON_KEYS(_utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) + AS h; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS( + _utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) AS ok; +--echo # an empty key +SELECT JSON_ARRAY(JSON_KEYS('{"":1,"b":2}')) AS v; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"":1,"b":2}'))) AS ok; + +--echo # +--echo # 6. The answers that are not arrays. An object with no keys is an +--echo # empty array; anything that is not an object is NULL, and a NULL +--echo # goes in as a null. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS('{}')) AS v; +SELECT JSON_ARRAY(JSON_KEYS('[1,2]')) AS v; +SELECT JSON_ARRAY(JSON_KEYS('7')) AS v; +SELECT JSON_ARRAY(JSON_KEYS(NULL)) AS v; +SELECT JSON_OBJECT('k', JSON_KEYS('[1,2]')) AS v; +--echo # input that does not read as a document at all +SELECT JSON_ARRAY(JSON_KEYS('{"a":')) AS v; + +--echo # +--echo # 7. The two-argument form, which reaches an object further in. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +--echo # a path that reaches nothing +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1}}', '$.zz')) AS v; + +--echo # +--echo # 8. Nested one inside the other. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS(JSON_OBJECT('a', 1, 'b', 2))) AS v; +SELECT JSON_KEYS(JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}'))) AS v; + +--echo # +--echo # 9. Over rows, including through an aggregate and through a +--echo # temporary table. +--echo # + +CREATE TABLE t1 (id INT, j VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2,"c":3}'); + +SELECT id, JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY id; +SELECT JSON_ARRAYAGG(JSON_KEYS(j)) AS v FROM t1; +SELECT JSON_OBJECTAGG(id, JSON_KEYS(j)) AS v FROM t1; +--echo # materialised into a temporary table on the way out +SELECT JSON_ARRAY(k) AS v FROM (SELECT JSON_KEYS(j) AS k FROM t1) d ORDER BY v; +--echo # MAX() is not typed by what it is taken over, so its result is +--echo # quoted here as it is for any other document producing function +SELECT JSON_ARRAY(MAX(JSON_KEYS(j))) AS v FROM t1 GROUP BY id ORDER BY v; +SELECT JSON_ARRAY(MAX(JSON_EXTRACT(j, '$'))) AS v FROM t1 GROUP BY id ORDER BY v; +SELECT DISTINCT JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY v; + +--echo # the column a result set of it makes, and the column the already +--echo # typed function makes beside it +CREATE TABLE t2 AS SELECT JSON_KEYS(j) AS k FROM t1; +SHOW CREATE TABLE t2; +SELECT JSON_ARRAY(k) AS v FROM t2 ORDER BY v; +DROP TABLE t2; +CREATE TABLE t3 AS SELECT JSON_EXTRACT(j, '$') AS k FROM t1; +SHOW CREATE TABLE t3; +DROP TABLE t3; + +DROP TABLE t1; + +--echo # +--echo # 10. The result metadata, which is what decides all of the above. +--echo # + +--enable_metadata +SELECT JSON_KEYS('{"a":1,"b":2}') AS v; +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +SELECT JSON_QUOTE('a') AS v; +--disable_metadata + +--echo # +--echo # 11. JSON_QUOTE is documented to return a string and still does. +--echo # + +SELECT JSON_ARRAY(JSON_QUOTE('a')) AS v; +SELECT JSON_OBJECT('k', JSON_QUOTE('a')) AS v; + +--echo # +--echo # 12. The array is attested as well as typed, so a splice of it +--echo # goes in without being read again - which no result here can +--echo # show and Json_scans can, and does in func_json_scan_count. +--echo # +--echo # What stands in the way of that is a character set this +--echo # cannot attest to, and there are two kinds of those. +--echo # +--echo # A set that cannot encode a bracket cannot encode a brace +--echo # either, so no array is ever answered in it and the question +--echo # never comes up - swe7, below. +--echo # +--echo # But the question actually asked is whether the set is +--echo # ASCII-compatible, and a set can decode every ASCII byte to +--echo # itself and still be flagged as not. sjis is one: it parses +--echo # a document, answers a real array, and has the mark withheld +--echo # from it all the same, so a splice reads that array again. +--echo # The array is a perfectly good document either way; what is +--echo # lost is the reading it was meant to save. +--echo # +--echo # The counts that say which of the two happened are in +--echo # func_json_keys_typed_scans, which a debug build runs. +--echo # + +SELECT JSON_KEYS(CONVERT('{"a":1}' USING swe7)) AS no_answer; +SELECT JSON_VALID(CONVERT('{"a":1}' USING swe7)) AS nor_a_document; + +--echo # sjis: a real answer, and one the mark is withheld from +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +SELECT JSON_KEYS(j) AS answered, JSON_VALID(JSON_KEYS(j)) AS ok FROM ts; +DROP TABLE ts; + +--echo # +--echo # 13. What the answer is attested does not say the argument was +--echo # a document all the way through. This reads an object and +--echo # stops at its end, so a break past that end is never met - and +--echo # one it does meet refuses the answer whole. +--echo # + +--echo # never met: the break is past the object, or past the sub-object +--echo # the path chose +SELECT JSON_KEYS('{"a":1} rubbish') AS trailing_text; +SELECT JSON_KEYS('{"a":1}{"b":2}') AS second_document; +SELECT JSON_KEYS('{"a":1}}') AS trailing_brace; +SELECT JSON_KEYS('{"a":{"z":1} "b":2}', '$.a') AS break_after_the_step; +--echo # and the answers are documents, which is what the mark says +SELECT JSON_VALID(JSON_KEYS('{"a":1} rubbish')) AS ok; +SELECT JSON_ARRAY(JSON_KEYS('{"a":1} rubbish')) AS spliced; + +--echo # met: refused whole, and reported where it happened +SELECT JSON_KEYS('{"a":1 "b":2}') AS break_inside; +SELECT JSON_KEYS('{"a":') AS ends_early; +SELECT JSON_KEYS('{"a":{"z":1 "y":2}}', '$.a') AS break_inside_the_step; diff --git a/mysql-test/main/func_json_keys_typed_scans.result b/mysql-test/main/func_json_keys_typed_scans.result new file mode 100644 index 0000000000000..af53917a13269 --- /dev/null +++ b/mysql-test/main/func_json_keys_typed_scans.result @@ -0,0 +1,32 @@ +# +# The reading a JSON_KEYS array saves where it is spliced, and the +# reading it does not save where the set it is written in cannot +# be attested. Nothing but the count shows either of them. +# +# sjis decodes every ASCII byte to itself and is flagged as not +# ASCII-compatible all the same, so the array it answers is a +# perfectly good document with no mark on it, and the splice reads +# it again. utf8mb4 stands beside it, over the same document, +# where the mark is given and the splice reads nothing - which is +# what makes the count a fact about the set rather than about the +# document or the splice. +# +# What the answers themselves look like is in +# func_json_keys_typed. +# +SET NAMES utf8mb4; +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +CREATE TABLE tu (j TEXT) CHARACTER SET utf8mb4; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +INSERT INTO tu VALUES ('{"a":1,"b":2}'); +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM ts; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM tu; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +DROP TABLE ts, tu; diff --git a/mysql-test/main/func_json_keys_typed_scans.test b/mysql-test/main/func_json_keys_typed_scans.test new file mode 100644 index 0000000000000..904c09fc04426 --- /dev/null +++ b/mysql-test/main/func_json_keys_typed_scans.test @@ -0,0 +1,37 @@ +--source include/have_debug.inc +--source include/have_utf8mb4.inc + +--echo # +--echo # The reading a JSON_KEYS array saves where it is spliced, and the +--echo # reading it does not save where the set it is written in cannot +--echo # be attested. Nothing but the count shows either of them. +--echo # +--echo # sjis decodes every ASCII byte to itself and is flagged as not +--echo # ASCII-compatible all the same, so the array it answers is a +--echo # perfectly good document with no mark on it, and the splice reads +--echo # it again. utf8mb4 stands beside it, over the same document, +--echo # where the mark is given and the splice reads nothing - which is +--echo # what makes the count a fact about the set rather than about the +--echo # document or the splice. +--echo # +--echo # What the answers themselves look like is in +--echo # func_json_keys_typed. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +CREATE TABLE tu (j TEXT) CHARACTER SET utf8mb4; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +INSERT INTO tu VALUES ('{"a":1,"b":2}'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM ts; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM tu; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE ts, tu; diff --git a/mysql-test/main/func_json_scan_count.result b/mysql-test/main/func_json_scan_count.result index 3921057d883f5..9a4e431fd61b5 100644 --- a/mysql-test/main/func_json_scan_count.result +++ b/mysql-test/main/func_json_scan_count.result @@ -373,6 +373,42 @@ Variable_name Value Json_scans 2 DROP TABLE t4; # +# Keys are returned as an array of the names, written a name at +# a time out of the document they were read from. What that makes, +# how it is formatted and how deep it goes are all settled by the +# writing rather than measured afterwards, so a splice of the +# answer reads nothing. The names here are long, so the length +# says more levels than a document is allowed and the depth is the +# only thing keeping the answer from being read again. +# +# Four rows, the last two being the answers that are not arrays of +# names: an object with no keys at all, and a document that is not +# an object and returns nothing. +# +CREATE TABLE t5 (id INT, js TEXT, jn TEXT); +INSERT INTO t5 VALUES +(1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}'), +CONCAT('{"o":{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}}')), +(2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}'), +CONCAT('{"o":{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}}')), +(3, '{}', '{"o":{}}'), +(4, '[1,2]', '[3,4]'); +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(js)) FROM t5; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# The two argument form, which finds the object further in and then +# writes the same answer in the same way. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(jn, '$.o')) FROM t5; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +DROP TABLE t5; +# # Asking whether a value is a document reads it once. There is # nothing here left to take away. # diff --git a/mysql-test/main/func_json_scan_count.test b/mysql-test/main/func_json_scan_count.test index d90f845da529d..1d7d15afe9c97 100644 --- a/mysql-test/main/func_json_scan_count.test +++ b/mysql-test/main/func_json_scan_count.test @@ -440,6 +440,45 @@ SHOW STATUS LIKE 'Json_scans'; DROP TABLE t4; +--echo # +--echo # Keys are returned as an array of the names, written a name at +--echo # a time out of the document they were read from. What that makes, +--echo # how it is formatted and how deep it goes are all settled by the +--echo # writing rather than measured afterwards, so a splice of the +--echo # answer reads nothing. The names here are long, so the length +--echo # says more levels than a document is allowed and the depth is the +--echo # only thing keeping the answer from being read again. +--echo # +--echo # Four rows, the last two being the answers that are not arrays of +--echo # names: an object with no keys at all, and a document that is not +--echo # an object and returns nothing. +--echo # +CREATE TABLE t5 (id INT, js TEXT, jn TEXT); +INSERT INTO t5 VALUES + (1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}'), + CONCAT('{"o":{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}}')), + (2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}'), + CONCAT('{"o":{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}}')), + (3, '{}', '{"o":{}}'), + (4, '[1,2]', '[3,4]'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(js)) FROM t5; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The two argument form, which finds the object further in and then +--echo # writes the same answer in the same way. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(jn, '$.o')) FROM t5; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t5; + --echo # --echo # Asking whether a value is a document reads it once. There is --echo # nothing here left to take away. diff --git a/mysql-test/main/func_json_tmp_scan_count.result b/mysql-test/main/func_json_tmp_scan_count.result index 5a847106e5ada..7fcda73629aef 100644 --- a/mysql-test/main/func_json_tmp_scan_count.result +++ b/mysql-test/main/func_json_tmp_scan_count.result @@ -630,7 +630,42 @@ Variable_name Value Json_scans 8 DROP TABLE t31; # -# 37. Branches written in different character sets. A column takes +# 37. A column filled with the names of a document's keys. Every +# row of one of those is an array of strings whatever the document +# held, and that is so before a row of it exists - so the column is +# built saying it, and the splice on the way out reads nothing. +# +# The names are long here for the same reason as above: the length +# says more levels than a document is allowed, so the depth the +# column keeps is what the splice has to go on. +# +CREATE TABLE t37 (id INT, j TEXT) CHARSET utf8mb4; +INSERT INTO t37 VALUES +(1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}')), +(2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}')); +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 38. The same query with the store into that column coming up +# short, which is how 8 takes 3's column away. The count is what +# 37 would have come to with nothing said, so the difference +# between the two is the column and nothing else. +# +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +DROP TABLE t37; +# +# 39. Branches written in different character sets. A column takes # one set for all of them, and a branch in another set is stored # into it converting - except into the binary set, which converts # nothing at all and keeps every byte under the other set's name. @@ -638,33 +673,33 @@ DROP TABLE t31; # written, so the column takes its answer back and every row is # read on the way out. # -CREATE TABLE t37 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; -INSERT INTO t37 VALUES ('{"a":1}'), ('[1,2]'); -CREATE TABLE t37b (b BLOB); -INSERT INTO t37b VALUES ('{"b":2}'), ('[3,4]'); +CREATE TABLE t39 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t39 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t39b (b BLOB); +INSERT INTO t39b VALUES ('{"b":2}'), ('[3,4]'); FLUSH STATUS; SELECT JSON_ARRAY(x) AS r FROM -(SELECT JSON_QUERY(j, '$') AS x FROM t37 +(SELECT JSON_QUERY(j, '$') AS x FROM t39 UNION ALL -SELECT JSON_QUERY(b, '$') FROM t37b) d; +SELECT JSON_QUERY(b, '$') FROM t39b) d; SHOW STATUS LIKE 'Json_scans'; Variable_name Value Json_scans 8 # -# 38. The same two branches with the wide one alone, so that the +# 40. The same two branches with the wide one alone, so that the # column takes that same set and every character arrives as the one # written. Nothing is taken back, and the count is the branches' -# own readings and nothing else - which is what 37 would have come +# own readings and nothing else - which is what 39 would have come # to had the characters come across. # FLUSH STATUS; SELECT JSON_ARRAY(x) AS r FROM -(SELECT JSON_QUERY(j, '$') AS x FROM t37 +(SELECT JSON_QUERY(j, '$') AS x FROM t39 UNION ALL -SELECT JSON_QUERY(j, '$') FROM t37) d; +SELECT JSON_QUERY(j, '$') FROM t39) d; SHOW STATUS LIKE 'Json_scans'; Variable_name Value Json_scans 4 -DROP TABLE t37, t37b; +DROP TABLE t39, t39b; DROP TABLE t1; DROP TABLE t13, t13m; diff --git a/mysql-test/main/func_json_tmp_scan_count.test b/mysql-test/main/func_json_tmp_scan_count.test index f82b778c415e7..e58d571d70bfc 100644 --- a/mysql-test/main/func_json_tmp_scan_count.test +++ b/mysql-test/main/func_json_tmp_scan_count.test @@ -508,7 +508,45 @@ SHOW STATUS LIKE 'Json_scans'; DROP TABLE t31; --echo # ---echo # 37. Branches written in different character sets. A column takes +--echo # 37. A column filled with the names of a document's keys. Every +--echo # row of one of those is an array of strings whatever the document +--echo # held, and that is so before a row of it exists - so the column is +--echo # built saying it, and the splice on the way out reads nothing. +--echo # +--echo # The names are long here for the same reason as above: the length +--echo # says more levels than a document is allowed, so the depth the +--echo # column keeps is what the splice has to go on. +--echo # +CREATE TABLE t37 (id INT, j TEXT) CHARSET utf8mb4; +INSERT INTO t37 VALUES + (1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}')), + (2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}')); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 38. The same query with the store into that column coming up +--echo # short, which is how 8 takes 3's column away. The count is what +--echo # 37 would have come to with nothing said, so the difference +--echo # between the two is the column and nothing else. +--echo # +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +DROP TABLE t37; + +--echo # +--echo # 39. Branches written in different character sets. A column takes --echo # one set for all of them, and a branch in another set is stored --echo # into it converting - except into the binary set, which converts --echo # nothing at all and keeps every byte under the other set's name. @@ -516,36 +554,36 @@ DROP TABLE t31; --echo # written, so the column takes its answer back and every row is --echo # read on the way out. --echo # -CREATE TABLE t37 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; -INSERT INTO t37 VALUES ('{"a":1}'), ('[1,2]'); -CREATE TABLE t37b (b BLOB); -INSERT INTO t37b VALUES ('{"b":2}'), ('[3,4]'); +CREATE TABLE t39 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t39 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t39b (b BLOB); +INSERT INTO t39b VALUES ('{"b":2}'), ('[3,4]'); FLUSH STATUS; --disable_result_log SELECT JSON_ARRAY(x) AS r FROM - (SELECT JSON_QUERY(j, '$') AS x FROM t37 + (SELECT JSON_QUERY(j, '$') AS x FROM t39 UNION ALL - SELECT JSON_QUERY(b, '$') FROM t37b) d; + SELECT JSON_QUERY(b, '$') FROM t39b) d; --enable_result_log SHOW STATUS LIKE 'Json_scans'; --echo # ---echo # 38. The same two branches with the wide one alone, so that the +--echo # 40. The same two branches with the wide one alone, so that the --echo # column takes that same set and every character arrives as the one --echo # written. Nothing is taken back, and the count is the branches' ---echo # own readings and nothing else - which is what 37 would have come +--echo # own readings and nothing else - which is what 39 would have come --echo # to had the characters come across. --echo # FLUSH STATUS; --disable_result_log SELECT JSON_ARRAY(x) AS r FROM - (SELECT JSON_QUERY(j, '$') AS x FROM t37 + (SELECT JSON_QUERY(j, '$') AS x FROM t39 UNION ALL - SELECT JSON_QUERY(j, '$') FROM t37) d; + SELECT JSON_QUERY(j, '$') FROM t39) d; --enable_result_log SHOW STATUS LIKE 'Json_scans'; -DROP TABLE t37, t37b; +DROP TABLE t39, t39b; DROP TABLE t1; DROP TABLE t13, t13m; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 114ebf9112d13..6e539a407b38b 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -6625,6 +6625,8 @@ String *Item_func_json_keys::val_str(String *str) int array_counters[JSON_DEPTH_LIMIT]= {0}; THD *thd; + m_marks.clear(); + if ((args[0]->null_value)) goto null_return; @@ -6700,7 +6702,7 @@ String *Item_func_json_keys::val_str(String *str) if (!check_key_in_list(str, key_start, key_len)) { - if ((n_keys > 0 && str->append(", ", 2)) || + if ((n_keys > 0 && str->append(STRING_WITH_LEN(json_loose_comma))) || str->append('"') || append_simple(str, key_start, key_len) || str->append('"')) @@ -6721,6 +6723,39 @@ String *Item_func_json_keys::val_str(String *str) if (unlikely(je.s.error || str->append(']'))) goto err_return; + /* + What was just written is an array of the object's key names, and + all three things there are to say about it are settled by how it + was written rather than by anything that would have to be measured. + + It READS AS A DOCUMENT. A key goes in between quotes with its bytes + copied across as they stood, and they stood inside a document that + parsed - so whatever needed escaping in them is already escaped, and + a key that would not go into an array would not have come out of an + object. Nothing else can get in: every key written was copied out + of an object this walk read to its end, and a break the walk reaches + refuses the answer whole at the test just above. What it does not + reach it does not read - text standing after the object, or a second + document behind it - so the argument need not be a document all the + way through for this to hold, and it is not asked to be. + + It is FORMATTED THE PLAIN WAY. The one thing that stands between two + of anything here is the pair of characters the plain formatting puts + there, and nothing goes after the opening bracket or before the + closing one. + + It is ONE LEVEL DEEP. An array of strings, and a key name is a + string whatever it holds. + + All of it in the character set the argument was read in - the copy + is same-set, so the bytes mean here what they meant there - which + leaves only whether a document can be written in that set at all. + */ + { + bool is_json_compatible= is_json_compatible_charset(str->charset()); + + m_marks.set(str, is_json_compatible, is_json_compatible, 1); + } null_value= 0; return str; diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 54ecf2ddc8ede..b58a2ae5bd7d9 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -827,7 +827,17 @@ class Item_func_json_remove: public Item_json_str_multipath }; -class Item_func_json_keys: public Item_str_func +/* + Typed as returning a document like the rest of the family: the keys + are returned as an array, and a function given that array in value + position is meant to embed an array. + + The array is attested as well. It is written one key at a time out of + a document that parsed, and being an array of strings fixes its + validity, its formatting and its depth without measuring any of the + three - see where the result is returned. +*/ +class Item_func_json_keys: public Item_json_func { protected: json_path_with_flags path; @@ -835,7 +845,7 @@ class Item_func_json_keys: public Item_str_func public: Item_func_json_keys(THD *thd, List &list): - Item_str_func(thd, list) {} + Item_json_func(thd, list) {} LEX_CSTRING func_name_cstring() const override { static LEX_CSTRING name= {STRING_WITH_LEN("json_keys") }; @@ -843,6 +853,12 @@ class Item_func_json_keys: public Item_str_func } bool fix_length_and_dec(THD *thd) override; String *val_str(String *) override; + /* + A document or nothing, whichever row it is asked about: an object + gives back an array of its key names and anything else gives back + NULL, so a column of these can be built saying so. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override From bac5c7be5f4a3fb80b369e0e4c1679abeb9d933b Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Sat, 8 Aug 2026 18:15:13 -0400 Subject: [PATCH 22/30] MDEV-40677 Cut a JSON group back to a whole element Neither JSON aggregate cut its group to a document. They failed in opposite directions and one fix answers both: a group that has to be cut is cut at a boundary between elements, and what is left still reads as JSON. ## JSON_ARRAYAGG cut at a byte offset It derives from `Item_func_group_concat` and so inherited the check `dump_leaf_key()` makes once per row, which cuts at the byte the limit falls on because for GROUP_CONCAT the result has no structure to respect. The override it carried repaired one landing position out of several - a cut immediately after a quote, backed off one byte with a quote written back - and left every other one as GROUP_CONCAT made it: cap 7 ["aaaa""] not JSON cap 8 ["aaaa","] not JSON cap 9 ["aaaa",""] JSON, and the second element is invented The last of those is the serious one. It is valid, so nothing downstream rejects it, and it holds an empty string that no row of the group ever had: the cut landed just past a separator and an opening quote, and backing off one byte to write a quote closed a string with nothing in it. Warning 1260 says a row was cut, not that one was manufactured. `old_length`, which `dump_leaf_key()` hands the override already, is the length the result had before the separator and this element were written - the end of the last whole element, and the one place a cut leaves something the brackets can go round. So the override is now that cut and nothing else. ## JSON_OBJECTAGG did not cut at all It derives from `Item_sum`, keeps its own `String`, and its `add()` had no length test on any path, so the object grew with the data and stopped only where the allocator refused it. `group_concat_max_len` appeared nowhere in the file. A group of 2000 rows of 200 bytes came to 422893 bytes against a cap of 1024, and lowering the cap to 4 changed nothing. `LENGTH()` sends one integer to the client, so the whole buffer is built server side and `max_allowed_packet` is never consulted - as it is not for an assignment to a routine variable, an INSERT ... SELECT or a materialised derived table. `max_session_mem_used` does not apply either: the buffer is not `MY_THREAD_SPECIFIC`, so it is not counted against the session. `add()` now remembers where each pair began and takes the pair back off whole where it overran, closing the group there and saying so with the same warning 1260. Nothing goes in after that: a later pair small enough to fit in what is left would go in behind the pair that did not, putting the object out of the order the group was read in. The session the cut is measured against is taken once, where everything else the group starts with is set, rather than looked up again for every row of it: a group runs on one connection, so the lookup answers the same thing every time it is made. What is read THROUGH it is read per row still - the limit is the session's setting as it stands while the group is being built, which is the whole reason it is asked for there rather than taken at the width the item was fixed for. Nothing goes in, but the rows still go by. An argument is an expression, and the object works one out for every row of the group whether it has room for the answer or not; a group that stopped reading would stop whatever the expression does, which is the caller's to decide and not this function's to drop. So a row past the cut is read exactly as a row before it is - the key, and the value where the key makes a pair - and only the writing stops. `Item_func_group_concat::add()` divides the work the same way, its own cut reaching no further than the append `dump_leaf_key()` makes. `report_cut_value_error()` stops being static so the object aggregate can raise the warning the same way, having no GROUP_CONCAT machinery under it to raise it. ## The brace is one of the bytes returned Both aggregates are documented the same way - *"the maximum returned length in bytes is determined by the `group_concat_max_len` server system variable"* - and `GROUP_CONCAT`, which shares that sentence, ends on it exactly. The brace that closes the object is written when the group is asked for, which is after every pair has been through the length test in `add()`, so a group whose pairs ended exactly on the limit would come back at the limit plus one: cap 14 {"aaaa":"aaaa"} len 15 The room for it is kept back from the limit, and the pair that would need that room does not go in. The brace is not one byte wide in a set that writes no character in one byte, so the width is asked for rather than assumed - at two bytes to the character the same group would come back two over. Where the room cannot be spared the answer is `{}`, which is what a cap too small for a whole pair means here anyway, and the warning names the row that would not fit rather than the one after it. A pair that fits with its brace fits as it always did. ## What the cut costs and what it counts A cap that does not reach the end of the first element leaves an empty container rather than half of one - `[]` and `{}` - which is what a whole-element cut means when no whole element fits. The row the warning names is counted per group for the object, over the rows that have a key to make a pair of. The array's counter is GROUP_CONCAT's, which runs on across the groups of a statement; that is left as it is. The mark block over `Item_func_json_arrayagg::val_str()` says what the cut now leaves, and so does the test beside it: a group cut to fit the length limit is cut back to the last whole element. New test `func_json_agg_cut`, which walks the cut across every offset within an element for both aggregates, over strings, numbers and documents, and covers the empty container, DISTINCT and ORDER BY, characters wider than a byte, and a NULL key. It counts what an argument does over the same rows under a cap that cuts and under one that does not, on either side of the pair and with the array beside it, and the counts match. `func_json_agg_limits` records the old shapes; its queries are unchanged and its result is re-recorded, with the prose that described the old cut corrected. A pair that could not be written at all is a different thing from one that would not fit, and `func_json_agg_pair_oom` is where the difference is recorded. A pair taken off for the limit leaves an object the brace can still go round; a pair the buffer would not take is written in part, and what is left is an object no brace can close, so the group is refused whole. Nothing reaches that second case by asking an ordinary question - the room a pair takes is asked for before it is written - so each of the three writes a pair is made of names a debug keyword and is failed on its own, with a single row beside them where there is no separator to write at all. --- mysql-test/main/func_json.result | 2 +- mysql-test/main/func_json_agg_cut.result | 992 ++++++++++++++++++ mysql-test/main/func_json_agg_cut.test | 279 +++++ mysql-test/main/func_json_agg_limits.result | 106 +- mysql-test/main/func_json_agg_limits.test | 41 +- mysql-test/main/func_json_agg_pair_oom.result | 74 ++ mysql-test/main/func_json_agg_pair_oom.test | 73 ++ mysql-test/main/func_json_marks.result | 14 +- mysql-test/main/func_json_marks.test | 12 +- sql/item_jsonfunc.cc | 124 ++- sql/item_jsonfunc.h | 30 +- sql/item_sum.cc | 2 +- sql/item_sum.h | 13 + 13 files changed, 1640 insertions(+), 122 deletions(-) create mode 100644 mysql-test/main/func_json_agg_cut.result create mode 100644 mysql-test/main/func_json_agg_cut.test create mode 100644 mysql-test/main/func_json_agg_pair_oom.result create mode 100644 mysql-test/main/func_json_agg_pair_oom.test diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index 80f7f753ac0c9..d895884f3b258 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -1617,7 +1617,7 @@ insert into t1 values (concat('x64-', repeat('b', 60))); insert into t1 values (concat('x64-', repeat('c', 60))); select json_arrayagg(a) from t1; json_arrayagg(a) -["x64-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] +[] Warnings: Warning 1260 Row 1 was cut by JSON_ARRAYAGG() drop table t1; diff --git a/mysql-test/main/func_json_agg_cut.result b/mysql-test/main/func_json_agg_cut.result new file mode 100644 index 0000000000000..4d07a94a6d168 --- /dev/null +++ b/mysql-test/main/func_json_agg_cut.result @@ -0,0 +1,992 @@ +# +# A group too long for group_concat_max_len is cut back to the last +# whole element or pair, so what is returned is still a document. +# Both aggregates honour the setting and both say so with a warning. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES +(1, 'aaaa'), (1, 'bbbb'), (1, 'cccc'), (1, 'dddd'), +(2, 'ee'), (2, 'ff'); +# +# 1. The cut walked across every offset within an element. The uncut +# group 1 is ["aaaa","bbbb","cccc","dddd"], 29 bytes. +# +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 1; +v +["aaaa","bbbb","cccc","dddd"] +SET SESSION group_concat_max_len = 3; +Warnings: +Warning 1292 Truncated incorrect group_concat_max_len value: '3' +SELECT 3 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +3 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +4 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +5 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +6 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +7 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +8 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +9 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +10 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +11 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +12 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +13 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +14 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +15 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +16 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +17 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +18 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +19 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 2. The same walk for the object, which used to ignore the setting +# altogether. The uncut group 1 is +# {"aaaa":"aaaa", "bbbb":"bbbb", "cccc":"cccc", "dddd":"dddd"}. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 1; +v +{"aaaa":"aaaa", "bbbb":"bbbb", "cccc":"cccc", "dddd":"dddd"} +SET SESSION group_concat_max_len = 3; +Warnings: +Warning 1292 Truncated incorrect group_concat_max_len value: '3' +SELECT 3 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +3 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +4 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +5 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +6 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +7 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +8 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +9 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +10 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +11 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +12 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +13 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +14 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +15 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +16 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +17 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +18 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +19 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 3. A cap smaller than the first element leaves an empty container +# rather than half of one. +# +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +v ok +[] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SELECT JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +v ok +{} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 4. Elements that are not quoted strings. +# +CREATE TABLE t2 (n INT, j VARCHAR(20)); +INSERT INTO t2 VALUES (1111, '{"k":1}'), (2222, '{"k":2}'), (3333, '{"k":3}'); +# numbers, where the old cut left a trailing separator +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +4 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +5 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +6 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +7 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +8 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +9 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +10 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +11 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +12 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +13 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +14 [1111,2222,3333] 1 +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +15 [1111,2222,3333] 1 +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +16 [1111,2222,3333] 1 +SET SESSION group_concat_max_len = DEFAULT; +# documents, where the old cut landed inside an object +SET SESSION group_concat_max_len = 12; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +v ok +[{"k":1}] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +v ok +[{"k":1},{"k":2}] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT JSON_OBJECTAGG(n, JSON_COMPACT(j)) AS v, +JSON_VALID(JSON_OBJECTAGG(n, JSON_COMPACT(j))) AS ok FROM t2; +v ok +{"1111":{"k":1}} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; +# +# 5. A cut group and an uncut group in the same statement, and the +# row the warning names. +# +SET SESSION group_concat_max_len = 14; +SELECT g, JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ["aaaa","bbbb"] +2 ["ee","ff"] +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 {} +2 {"ee":"ee"} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 6. DISTINCT and ORDER BY inside the array, which reach the cut by +# a different route. +# +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(DISTINCT a ORDER BY a) AS v FROM t1 WHERE g = 1; +v +["aaaa","bbbb"] +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +v +["dddd","cccc"] +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 7. A cut result read back by another JSON function. +# +SET SESSION group_concat_max_len = 14; +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a ORDER BY a), '$[0]') AS v FROM t1 WHERE g = 1; +v +"aaaa" +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SELECT JSON_LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +v +2 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SELECT JSON_ARRAY(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +v +[["aaaa","bbbb"]] +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SELECT JSON_LENGTH(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +v +0 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SELECT JSON_ARRAY(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +v +[{}] +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 8. Characters wider than a byte. The cap counts bytes, and an +# element boundary is a boundary whatever the characters cost. +# +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t3 VALUES (_utf8mb4 X'C3A4C3A4'), (_utf8mb4 X'C3B6C3B6'); +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +h ok +5B22C3A4C3A4225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +h ok +5B22C3A4C3A4225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_OBJECTAGG(a, 1)) AS h, +JSON_VALID(JSON_OBJECTAGG(a, 1)) AS ok FROM t3; +h ok +7B7D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; +# +# 9. Nothing to cut: a group that fits raises no warning, and an +# empty group is unaffected. +# +SET SESSION group_concat_max_len = 1024; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 2; +v +["ee","ff"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +v +{"ee":"ee", "ff":"ff"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +v +NULL +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +v +NULL +SET SESSION group_concat_max_len = DEFAULT; +# +# 10. A NULL key is not a pair of the object, so it is not the row +# the warning names either. +# +CREATE TABLE t4 (k VARCHAR(20), v INT); +INSERT INTO t4 VALUES (NULL, 1), ('kkkk', 2), ('llll', 3), ('mmmm', 4); +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, v) AS v FROM t4; +v +{"kkkk":2} +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; +DROP TABLE t1; +# +# 11. A cut group is still read to the end. An argument is an +# expression, and the rows past the cut work it out exactly as +# the rows before it do - only the writing stops. Each shape +# below is run twice over the same rows, once under a cap that +# cuts and once under one that does not, and counts what the +# argument did. The two counts are the same because the +# reading is. +# +CREATE TABLE t5 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t5 VALUES +('kkkk', 'aaaa'), ('llll', 'bbbb'), ('mmmm', 'cccc'), ('nnnn', 'dddd'); +CREATE FUNCTION seen_once(x VARCHAR(20)) RETURNS VARCHAR(20) NO SQL +BEGIN +SET @calls= @calls + 1; +RETURN x; +END// +# the array, which reads every row of the group already +SET @calls= 0; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +v +["aaaa","bbbb","cccc","dddd"] +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +v +["aaaa","bbbb"] +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +4 4 +# the object, on the value of the pair +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +v +{"kkkk":"aaaa", "llll":"bbbb", "mmmm":"cccc", "nnnn":"dddd"} +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +v +{} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +4 4 +# the object, on the key. The object also asks the key whether it +# is NULL, and asking that of a function is running it again, so +# the count here is two a row - under either cap. +SET @calls= 0; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +v +{"kkkk":"aaaa", "llll":"bbbb", "mmmm":"cccc", "nnnn":"dddd"} +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +v +{} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +8 8 +# a row whose key is NULL makes no pair, so its value is not read +# for it - past the cut no differently from before it +CREATE TABLE t6 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t6 VALUES +('kkkk', 'aaaa'), ('llll', 'bbbb'), (NULL, 'cccc'), ('nnnn', 'dddd'); +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +v +{"kkkk":"aaaa", "llll":"bbbb", "nnnn":"dddd"} +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +v +{} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +3 3 +DROP TABLE t6; +DROP FUNCTION seen_once; +DROP TABLE t5; +# +# 12. The limit is on the bytes returned, and the brace that +# closes the object is one of them. It goes on after the +# length has been tested, so the room for it has to be kept +# back when the test is made. +# +CREATE TABLE t7 (a VARCHAR(20)); +INSERT INTO t7 VALUES ('aaaa'), ('bbbb'), ('cccc'); +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 4 AS within FROM t7; +cap v len within +4 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 5 AS within FROM t7; +cap v len within +5 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 6 AS within FROM t7; +cap v len within +6 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 7 AS within FROM t7; +cap v len within +7 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 8 AS within FROM t7; +cap v len within +8 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 9 AS within FROM t7; +cap v len within +9 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 10 AS within FROM t7; +cap v len within +10 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 11 AS within FROM t7; +cap v len within +11 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 12 AS within FROM t7; +cap v len within +12 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 13 AS within FROM t7; +cap v len within +13 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 14 AS within FROM t7; +cap v len within +14 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 15 AS within FROM t7; +cap v len within +15 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 16 AS within FROM t7; +cap v len within +16 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 17 AS within FROM t7; +cap v len within +17 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 18 AS within FROM t7; +cap v len within +18 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 19 AS within FROM t7; +cap v len within +19 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 20 AS within FROM t7; +cap v len within +20 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 21 AS within FROM t7; +cap v len within +21 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 22 AS within FROM t7; +cap v len within +22 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 23 AS within FROM t7; +cap v len within +23 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t7; +# A set that writes no character in one byte: the brace costs two +# bytes there, so two is what has to be kept back. +CREATE TABLE t8 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t8 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 12 AS within FROM t8; +cap h len within +12 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 13 AS within FROM t8; +cap h len within +13 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 14 AS within FROM t8; +cap h len within +14 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 15 AS within FROM t8; +cap h len within +15 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 16 AS within FROM t8; +cap h len within +16 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 17 AS within FROM t8; +cap h len within +17 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 18 AS within FROM t8; +cap h len within +18 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 19 AS within FROM t8; +cap h len within +19 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 20 AS within FROM t8; +cap h len within +20 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 21 AS within FROM t8; +cap h len within +21 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t8; diff --git a/mysql-test/main/func_json_agg_cut.test b/mysql-test/main/func_json_agg_cut.test new file mode 100644 index 0000000000000..274c707f62468 --- /dev/null +++ b/mysql-test/main/func_json_agg_cut.test @@ -0,0 +1,279 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # A group too long for group_concat_max_len is cut back to the last +--echo # whole element or pair, so what is returned is still a document. +--echo # Both aggregates honour the setting and both say so with a warning. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES + (1, 'aaaa'), (1, 'bbbb'), (1, 'cccc'), (1, 'dddd'), + (2, 'ee'), (2, 'ff'); + +--echo # +--echo # 1. The cut walked across every offset within an element. The uncut +--echo # group 1 is ["aaaa","bbbb","cccc","dddd"], 29 bytes. +--echo # + +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 1; + +--let $cap= 3 +while ($cap < 20) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 2. The same walk for the object, which used to ignore the setting +--echo # altogether. The uncut group 1 is +--echo # {"aaaa":"aaaa", "bbbb":"bbbb", "cccc":"cccc", "dddd":"dddd"}. +--echo # + +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 1; + +--let $cap= 3 +while ($cap < 20) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_OBJECTAGG(a, a) AS v, + JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 3. A cap smaller than the first element leaves an empty container +--echo # rather than half of one. +--echo # + +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +SELECT JSON_OBJECTAGG(a, a) AS v, + JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 4. Elements that are not quoted strings. +--echo # + +CREATE TABLE t2 (n INT, j VARCHAR(20)); +INSERT INTO t2 VALUES (1111, '{"k":1}'), (2222, '{"k":2}'), (3333, '{"k":3}'); + +--echo # numbers, where the old cut left a trailing separator +--let $cap= 4 +while ($cap < 17) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, + JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # documents, where the old cut landed inside an object +SET SESSION group_concat_max_len = 12; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, + JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +SET SESSION group_concat_max_len = 17; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, + JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +SET SESSION group_concat_max_len = 20; +SELECT JSON_OBJECTAGG(n, JSON_COMPACT(j)) AS v, + JSON_VALID(JSON_OBJECTAGG(n, JSON_COMPACT(j))) AS ok FROM t2; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 5. A cut group and an uncut group in the same statement, and the +--echo # row the warning names. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT g, JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 GROUP BY g ORDER BY g; +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 6. DISTINCT and ORDER BY inside the array, which reach the cut by +--echo # a different route. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(DISTINCT a ORDER BY a) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 7. A cut result read back by another JSON function. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a ORDER BY a), '$[0]') AS v FROM t1 WHERE g = 1; +SELECT JSON_LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_LENGTH(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAY(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 8. Characters wider than a byte. The cap counts bytes, and an +--echo # element boundary is a boundary whatever the characters cost. +--echo # + +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t3 VALUES (_utf8mb4 X'C3A4C3A4'), (_utf8mb4 X'C3B6C3B6'); +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_OBJECTAGG(a, 1)) AS h, + JSON_VALID(JSON_OBJECTAGG(a, 1)) AS ok FROM t3; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; + +--echo # +--echo # 9. Nothing to cut: a group that fits raises no warning, and an +--echo # empty group is unaffected. +--echo # + +SET SESSION group_concat_max_len = 1024; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 2; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 10. A NULL key is not a pair of the object, so it is not the row +--echo # the warning names either. +--echo # + +CREATE TABLE t4 (k VARCHAR(20), v INT); +INSERT INTO t4 VALUES (NULL, 1), ('kkkk', 2), ('llll', 3), ('mmmm', 4); +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, v) AS v FROM t4; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; + +DROP TABLE t1; + +--echo # +--echo # 11. A cut group is still read to the end. An argument is an +--echo # expression, and the rows past the cut work it out exactly as +--echo # the rows before it do - only the writing stops. Each shape +--echo # below is run twice over the same rows, once under a cap that +--echo # cuts and once under one that does not, and counts what the +--echo # argument did. The two counts are the same because the +--echo # reading is. +--echo # + +CREATE TABLE t5 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t5 VALUES + ('kkkk', 'aaaa'), ('llll', 'bbbb'), ('mmmm', 'cccc'), ('nnnn', 'dddd'); + +DELIMITER //; +CREATE FUNCTION seen_once(x VARCHAR(20)) RETURNS VARCHAR(20) NO SQL +BEGIN + SET @calls= @calls + 1; + RETURN x; +END// +DELIMITER ;// + +--echo # the array, which reads every row of the group already +SET @calls= 0; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; + +--echo # the object, on the value of the pair +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; + +--echo # the object, on the key. The object also asks the key whether it +--echo # is NULL, and asking that of a function is running it again, so +--echo # the count here is two a row - under either cap. +SET @calls= 0; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; + +--echo # a row whose key is NULL makes no pair, so its value is not read +--echo # for it - past the cut no differently from before it +CREATE TABLE t6 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t6 VALUES + ('kkkk', 'aaaa'), ('llll', 'bbbb'), (NULL, 'cccc'), ('nnnn', 'dddd'); +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +DROP TABLE t6; + +DROP FUNCTION seen_once; +DROP TABLE t5; + +--echo # +--echo # 12. The limit is on the bytes returned, and the brace that +--echo # closes the object is one of them. It goes on after the +--echo # length has been tested, so the room for it has to be kept +--echo # back when the test is made. +--echo # + +CREATE TABLE t7 (a VARCHAR(20)); +INSERT INTO t7 VALUES ('aaaa'), ('bbbb'), ('cccc'); + +--let $cap= 4 +while ($cap < 24) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_OBJECTAGG(a, a) AS v, + LENGTH(JSON_OBJECTAGG(a, a)) AS len, + LENGTH(JSON_OBJECTAGG(a, a)) <= $cap AS within FROM t7; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t7; + +--echo # A set that writes no character in one byte: the brace costs two +--echo # bytes there, so two is what has to be kept back. +CREATE TABLE t8 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t8 VALUES ('aa'), ('bb'); + +--let $cap= 12 +while ($cap < 22) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, + LENGTH(JSON_OBJECTAGG(a, a)) AS len, + LENGTH(JSON_OBJECTAGG(a, a)) <= $cap AS within FROM t8; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t8; diff --git a/mysql-test/main/func_json_agg_limits.result b/mysql-test/main/func_json_agg_limits.result index d81e928259324..6daf1b53b44eb 100644 --- a/mysql-test/main/func_json_agg_limits.result +++ b/mysql-test/main/func_json_agg_limits.result @@ -1,13 +1,11 @@ # # Behavioral baseline: the JSON aggregate functions. # -# JSON_ARRAYAGG inherits the GROUP_CONCAT length cap, so its result is -# chopped at group_concat_max_len bytes. Depending on where the cut -# lands, the value it returns is either still parseable or not, and -# the only signal is a warning. This test walks the cut across every -# interesting byte offset and records the exact result each time. -# JSON_OBJECTAGG is included next to it, including the fact that it -# does not honour the same cap. +# Both aggregates are held to group_concat_max_len, and a group too +# long for it is cut back to the last whole element or pair, so what +# comes back is still a document wherever the cap falls. This test +# walks the cap across every interesting byte offset and records the +# exact result each time. # SET NAMES utf8mb4; CREATE TABLE t1 (g INT, a VARCHAR(20)); @@ -84,7 +82,7 @@ FROM t1 WHERE g = 2; v [{"k":"dd"},{"k":"ee"}] # -# 2. The cut, walked byte by byte. The uncut result for group 1 is +# 2. The cap, walked byte by byte. The uncut result for group 1 is # ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"], 40 bytes. # SET SESSION group_concat_max_len = 40; @@ -101,7 +99,7 @@ SET SESSION group_concat_max_len = 30; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B2261616161616161616161222C2262626262626262626262222C226363225D 1 +5B2261616161616161616161222C2262626262626262626262225D 1 Warnings: Warning 1260 Row 3 was cut by JSON_ARRAYAGG() Warning 1260 Row 3 was cut by JSON_ARRAYAGG() @@ -109,40 +107,38 @@ SET SESSION group_concat_max_len = 20; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B2261616161616161616161222C226262626262225D 1 +5B2261616161616161616161225D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -# the cut lands just after the separator, leaving an empty member +# the cap falls just after the separator SET SESSION group_concat_max_len = 15; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B2261616161616161616161222C22225D 1 +5B2261616161616161616161225D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -# the cut lands on the opening quote of the next member +# the cap falls on the opening quote of the next member SET SESSION group_concat_max_len = 14; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B2261616161616161616161222C225D 0 +5B2261616161616161616161225D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' -# the cut lands on the separator itself +# the cap falls on the separator itself SET SESSION group_concat_max_len = 13; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B226161616161616161616122225D 0 +5B2261616161616161616161225D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 -# the cut lands inside the first member +# the cap falls inside the first member SET SESSION group_concat_max_len = 12; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; @@ -155,7 +151,7 @@ SET SESSION group_concat_max_len = 5; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B22616161225D 1 +5B5D 1 Warnings: Warning 1260 Row 1 was cut by JSON_ARRAYAGG() Warning 1260 Row 1 was cut by JSON_ARRAYAGG() @@ -163,7 +159,7 @@ SET SESSION group_concat_max_len = 4; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B226161225D 1 +5B5D 1 Warnings: Warning 1260 Row 1 was cut by JSON_ARRAYAGG() Warning 1260 Row 1 was cut by JSON_ARRAYAGG() @@ -174,27 +170,25 @@ h 5B226464225D SET SESSION group_concat_max_len = DEFAULT; # -# 3. A cut that falls inside an escape sequence, where the trailing -# bytes are not a complete character. +# 3. A cap that falls inside an escape sequence, where the bytes +# after it would not be a complete character. # CREATE TABLE t2 (a VARCHAR(20)); INSERT INTO t2 VALUES ('xx"yy'), ('zzzz'); SET SESSION group_concat_max_len = 10; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok -5B2278785C227979222C225D 0 +5B2278785C227979225D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' SET SESSION group_concat_max_len = 9; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok -5B2278785C22797922225D 0 +5B2278785C227979225D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 SET SESSION group_concat_max_len = 8; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok @@ -205,32 +199,31 @@ Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 7; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok -5B2278785C2279225D 1 +5B5D 1 Warnings: Warning 1260 Row 1 was cut by JSON_ARRAYAGG() Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 6; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok -5B2278785C22225D 1 +5B5D 1 Warnings: Warning 1260 Row 1 was cut by JSON_ARRAYAGG() Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = DEFAULT; DROP TABLE t2; # -# 3a. A cut over members that are not quoted strings. The re-close -# path only fixes a split trailing quote, so a cut that ends on a -# digit or inside an object takes the other branch entirely. +# 3a. A cap over members that are not quoted strings, where there is +# no quote for a cut to land after and nothing to re-close. # CREATE TABLE t3 (n INT, j JSON); INSERT INTO t3 VALUES (111, '{"k":1}'), (222, '{"k":2}'), (333, '{"k":3}'); -# numeric members: the cut can leave a shorter but still parseable array +# numeric members SET SESSION group_concat_max_len = 9; SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; h ok -5B3131312C3232322C335D 1 +5B3131312C3232325D 1 Warnings: Warning 1260 Row 3 was cut by JSON_ARRAYAGG() Warning 1260 Row 3 was cut by JSON_ARRAYAGG() @@ -238,102 +231,95 @@ SET SESSION group_concat_max_len = 5; SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; h ok -5B3131312C325D 1 +5B3131315D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -# document members: the cut lands inside an object instead +# document members, where the cap falls inside an object SET SESSION group_concat_max_len = 12; SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; h ok -5B7B226B223A317D2C7B226B225D 0 +5B7B226B223A317D5D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 13 SET SESSION group_concat_max_len = 17; SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; h ok -5B7B226B223A317D2C7B226B223A327D2C7B5D 0 +5B7B226B223A317D2C7B226B223A327D5D 1 Warnings: Warning 1260 Row 3 was cut by JSON_ARRAYAGG() Warning 1260 Row 3 was cut by JSON_ARRAYAGG() -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 19 # members built by a constructor rather than read from a column SET SESSION group_concat_max_len = 12; SELECT HEX(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS ok FROM t3; h ok -5B7B226B223A203131317D2C7B5D 0 +5B7B226B223A203131317D5D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 SET SESSION group_concat_max_len = DEFAULT; DROP TABLE t3; # -# 4. A cut result consumed by another JSON function. Today the cut -# value is handed on as a document argument like any other. +# 4. A cut result consumed by another JSON function. The cut value +# is passed on as a document argument like any other. # SET SESSION group_concat_max_len = 14; SELECT JSON_SET(JSON_ARRAYAGG(a), '$[0]', 1) AS v FROM t1 WHERE g = 1; v -NULL +[1] Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' SELECT JSON_EXTRACT(JSON_ARRAYAGG(a), '$[0]') AS v FROM t1 WHERE g = 1; v -NULL +"aaaaaaaaaa" Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; v -[["aaaaaaaaaa","]] +[["aaaaaaaaaa"]] Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_DEPTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; v -NULL +2 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_depth' SELECT JSON_LENGTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; v -NULL +1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_length' SELECT JSON_COMPACT(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; v -NULL +["aaaaaaaaaa"] Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_compact' # an uncut group and a cut group in the same statement SET SESSION group_concat_max_len = 14; SELECT g, HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 GROUP BY g ORDER BY g; g h ok -1 5B2261616161616161616161222C225D 0 +1 5B2261616161616161616161225D 1 2 5B226464222C226565225D 1 Warnings: Warning 1260 Row 2 was cut by JSON_ARRAYAGG() Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' SET SESSION group_concat_max_len = DEFAULT; # -# 5. JSON_OBJECTAGG does not honour group_concat_max_len. +# 5. JSON_OBJECTAGG under the same cap. # SET SESSION group_concat_max_len = 5; SELECT HEX(JSON_OBJECTAGG(a, a)) AS h, JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; h ok -7B2261616161616161616161223A2261616161616161616161222C202262626262626262626262223A2262626262626262626262222C202263636363636363636363223A2263636363636363636363227D 1 +7B7D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() SET SESSION group_concat_max_len = DEFAULT; # duplicate keys are kept, not merged SELECT JSON_OBJECTAGG(g, a) AS v FROM t1 WHERE g = 1; diff --git a/mysql-test/main/func_json_agg_limits.test b/mysql-test/main/func_json_agg_limits.test index 6c08d153754d6..da5a58ea01e23 100644 --- a/mysql-test/main/func_json_agg_limits.test +++ b/mysql-test/main/func_json_agg_limits.test @@ -3,13 +3,11 @@ --echo # --echo # Behavioral baseline: the JSON aggregate functions. --echo # ---echo # JSON_ARRAYAGG inherits the GROUP_CONCAT length cap, so its result is ---echo # chopped at group_concat_max_len bytes. Depending on where the cut ---echo # lands, the value it returns is either still parseable or not, and ---echo # the only signal is a warning. This test walks the cut across every ---echo # interesting byte offset and records the exact result each time. ---echo # JSON_OBJECTAGG is included next to it, including the fact that it ---echo # does not honour the same cap. +--echo # Both aggregates are held to group_concat_max_len, and a group too +--echo # long for it is cut back to the last whole element or pair, so what +--echo # comes back is still a document wherever the cap falls. This test +--echo # walks the cap across every interesting byte offset and records the +--echo # exact result each time. --echo # SET NAMES utf8mb4; @@ -50,7 +48,7 @@ SELECT JSON_ARRAYAGG(JSON_COMPACT(CONCAT('{"k":"', a, '"}'))) AS v FROM t1 WHERE g = 2; --echo # ---echo # 2. The cut, walked byte by byte. The uncut result for group 1 is +--echo # 2. The cap, walked byte by byte. The uncut result for group 1 is --echo # ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"], 40 bytes. --echo # @@ -66,19 +64,19 @@ SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok SET SESSION group_concat_max_len = 20; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; ---echo # the cut lands just after the separator, leaving an empty member +--echo # the cap falls just after the separator SET SESSION group_concat_max_len = 15; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; ---echo # the cut lands on the opening quote of the next member +--echo # the cap falls on the opening quote of the next member SET SESSION group_concat_max_len = 14; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; ---echo # the cut lands on the separator itself +--echo # the cap falls on the separator itself SET SESSION group_concat_max_len = 13; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; ---echo # the cut lands inside the first member +--echo # the cap falls inside the first member SET SESSION group_concat_max_len = 12; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; @@ -94,8 +92,8 @@ SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1 WHERE a = 'dd'; SET SESSION group_concat_max_len = DEFAULT; --echo # ---echo # 3. A cut that falls inside an escape sequence, where the trailing ---echo # bytes are not a complete character. +--echo # 3. A cap that falls inside an escape sequence, where the bytes +--echo # after it would not be a complete character. --echo # CREATE TABLE t2 (a VARCHAR(20)); @@ -114,21 +112,20 @@ SET SESSION group_concat_max_len = DEFAULT; DROP TABLE t2; --echo # ---echo # 3a. A cut over members that are not quoted strings. The re-close ---echo # path only fixes a split trailing quote, so a cut that ends on a ---echo # digit or inside an object takes the other branch entirely. +--echo # 3a. A cap over members that are not quoted strings, where there is +--echo # no quote for a cut to land after and nothing to re-close. --echo # CREATE TABLE t3 (n INT, j JSON); INSERT INTO t3 VALUES (111, '{"k":1}'), (222, '{"k":2}'), (333, '{"k":3}'); ---echo # numeric members: the cut can leave a shorter but still parseable array +--echo # numeric members SET SESSION group_concat_max_len = 9; SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; SET SESSION group_concat_max_len = 5; SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; ---echo # document members: the cut lands inside an object instead +--echo # document members, where the cap falls inside an object SET SESSION group_concat_max_len = 12; SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; @@ -143,8 +140,8 @@ SET SESSION group_concat_max_len = DEFAULT; DROP TABLE t3; --echo # ---echo # 4. A cut result consumed by another JSON function. Today the cut ---echo # value is handed on as a document argument like any other. +--echo # 4. A cut result consumed by another JSON function. The cut value +--echo # is passed on as a document argument like any other. --echo # SET SESSION group_concat_max_len = 14; @@ -161,7 +158,7 @@ SELECT g, HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok SET SESSION group_concat_max_len = DEFAULT; --echo # ---echo # 5. JSON_OBJECTAGG does not honour group_concat_max_len. +--echo # 5. JSON_OBJECTAGG under the same cap. --echo # SET SESSION group_concat_max_len = 5; diff --git a/mysql-test/main/func_json_agg_pair_oom.result b/mysql-test/main/func_json_agg_pair_oom.result new file mode 100644 index 0000000000000..3d5187befd3fc --- /dev/null +++ b/mysql-test/main/func_json_agg_pair_oom.result @@ -0,0 +1,74 @@ +# +# A pair that could not be written at all, which is a different +# thing from one that would not fit. +# +# A pair taken off for the length limit leaves an object the brace +# can still go round, and the group is returned short of it. A +# pair the buffer would not take is written in part, and what is +# left is an object no brace can close, so the group is refused +# whole instead. +# +# Nothing reaches that second outcome by asking the server an +# ordinary question: the room a pair takes is asked for before it +# is written, so the writes never run short. Each of the three a +# pair is made of is therefore failed on its own here - the +# separator in front of it, the quote that opens its key, and the +# colon that closes it. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (k VARCHAR(16), v INT); +INSERT INTO t1 VALUES ('a', 1), ('b', 2), ('c', 3); +# +# 1. What the group answers with nothing wrong. More than one +# row, so the separator has a pair in front of it to be +# written after. +# +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; +whole +{"a":1, "b":2, "c":3} +# +# 2. The separator in front of a pair +# +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_separator FROM t1; +no_separator +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 3. The quote that opens the key +# +SET SESSION debug_dbug = '+d,json_objectagg_key_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_key_quote FROM t1; +no_key_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 4. The colon that closes it +# +SET SESSION debug_dbug = '+d,json_objectagg_colon_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_colon FROM t1; +no_colon +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 5. One row, where there is no separator to write, so that the +# first of the three is shown not to fire where the group has +# nothing in front of the pair +# +CREATE TABLE t2 (k VARCHAR(16), v INT); +INSERT INTO t2 VALUES ('a', 1); +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS one_pair FROM t2; +one_pair +{"a":1} +SET SESSION debug_dbug = DEFAULT; +DROP TABLE t2; +# +# 6. The group once the room is there again, which is what says +# the refusals above are about the writing rather than about +# the rows +# +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; +whole +{"a":1, "b":2, "c":3} +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_pair_oom.test b/mysql-test/main/func_json_agg_pair_oom.test new file mode 100644 index 0000000000000..8cb6396a507a8 --- /dev/null +++ b/mysql-test/main/func_json_agg_pair_oom.test @@ -0,0 +1,73 @@ +--source include/have_debug.inc + +--echo # +--echo # A pair that could not be written at all, which is a different +--echo # thing from one that would not fit. +--echo # +--echo # A pair taken off for the length limit leaves an object the brace +--echo # can still go round, and the group is returned short of it. A +--echo # pair the buffer would not take is written in part, and what is +--echo # left is an object no brace can close, so the group is refused +--echo # whole instead. +--echo # +--echo # Nothing reaches that second outcome by asking the server an +--echo # ordinary question: the room a pair takes is asked for before it +--echo # is written, so the writes never run short. Each of the three a +--echo # pair is made of is therefore failed on its own here - the +--echo # separator in front of it, the quote that opens its key, and the +--echo # colon that closes it. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE t1 (k VARCHAR(16), v INT); +INSERT INTO t1 VALUES ('a', 1), ('b', 2), ('c', 3); + +--echo # +--echo # 1. What the group answers with nothing wrong. More than one +--echo # row, so the separator has a pair in front of it to be +--echo # written after. +--echo # +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; + +--echo # +--echo # 2. The separator in front of a pair +--echo # +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_separator FROM t1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 3. The quote that opens the key +--echo # +SET SESSION debug_dbug = '+d,json_objectagg_key_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_key_quote FROM t1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 4. The colon that closes it +--echo # +SET SESSION debug_dbug = '+d,json_objectagg_colon_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_colon FROM t1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 5. One row, where there is no separator to write, so that the +--echo # first of the three is shown not to fire where the group has +--echo # nothing in front of the pair +--echo # +CREATE TABLE t2 (k VARCHAR(16), v INT); +INSERT INTO t2 VALUES ('a', 1); +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS one_pair FROM t2; +SET SESSION debug_dbug = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 6. The group once the room is there again, which is what says +--echo # the refusals above are about the writing rather than about +--echo # the rows +--echo # +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_marks.result b/mysql-test/main/func_json_marks.result index a645c3b5e517a..af7c2669dedf2 100644 --- a/mysql-test/main/func_json_marks.result +++ b/mysql-test/main/func_json_marks.result @@ -150,18 +150,18 @@ g arr 2 [3,4] NULL [1,2,3,4] # -# A group cut to fit group_concat_max_len is cut at the byte -# the limit falls on. Here that lands just past an opening -# quote, and the quote written back to close the string leaves -# an element no row of the group held - a whole document, and -# one row of it invented. A group built out of a broken column -# is complete too, and is not a document at all. +# A group cut to fit group_concat_max_len is cut back to the +# last whole element, so every element in it is one a row of +# the group held and the row it stopped at is reported. What +# comes out is a document, but not the value the group has. +# A group built out of a broken column is complete too, and is +# not a document at all. # SET @save_len = @@group_concat_max_len; SET SESSION group_concat_max_len = 10; SELECT JSON_ARRAYAGG(s) AS cut FROM tg; cut -["a","b",""] +["a","b"] Warnings: Warning 1260 Row 3 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = @save_len; diff --git a/mysql-test/main/func_json_marks.test b/mysql-test/main/func_json_marks.test index b4398c8fdc6d4..7d972737832df 100644 --- a/mysql-test/main/func_json_marks.test +++ b/mysql-test/main/func_json_marks.test @@ -101,12 +101,12 @@ SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS agg_then_built FROM tg; SELECT g, JSON_ARRAYAGG(v) AS arr FROM tg GROUP BY g WITH ROLLUP; --echo # ---echo # A group cut to fit group_concat_max_len is cut at the byte ---echo # the limit falls on. Here that lands just past an opening ---echo # quote, and the quote written back to close the string leaves ---echo # an element no row of the group held - a whole document, and ---echo # one row of it invented. A group built out of a broken column ---echo # is complete too, and is not a document at all. +--echo # A group cut to fit group_concat_max_len is cut back to the +--echo # last whole element, so every element in it is one a row of +--echo # the group held and the row it stopped at is reported. What +--echo # comes out is a document, but not the value the group has. +--echo # A group built out of a broken column is complete too, and is +--echo # not a document at all. --echo # SET @save_len = @@group_concat_max_len; SET SESSION group_concat_max_len = 10; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 6e539a407b38b..7c8bb73781f7d 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -7394,20 +7394,24 @@ String *Item_func_json_arrayagg::get_str_from_field(Item *i,Field *f, } +/* + Cut back to where the element that overran began, rather than to the + byte the limit falls on. What GROUP_CONCAT passes as old_length + is the length the result had before the separator and this element + were written, so it is the end of the last whole element - the one + place a cut leaves something the brackets can go round. + + A cut anywhere else leaves half an element behind, and the three ways + it can land are each their own kind of wrong. On a quote it leaves a + string opened twice; on a separator it leaves one with nothing after + it; and just past an opening quote it leaves an empty string that no + row of the group ever held - which reads back as an element and is + not one, so nothing downstream has any way to know it is not data. +*/ void Item_func_json_arrayagg::cut_max_length(String *result, - uint old_length, uint max_length) const + uint old_length, uint max_length __attribute__((unused))) const { - if (result->length() == 0) - return; - - if (result->end()[-1] != '"' || old_length == max_length) - { - Item_func_group_concat::cut_max_length(result, old_length, max_length); - return; - } - - Item_func_group_concat::cut_max_length(result, old_length, max_length-1); - result->append('"'); + result->length(old_length); } @@ -7475,14 +7479,12 @@ String* Item_func_json_arrayagg::val_str(String *str) } /* - A group that was cut to fit the length limit is cut at the byte - the limit falls on, so what stands between the brackets can be - half an element - or, where the cut lands just past an opening - quote and that quote is written back to close the string, an - element no row of the group ever held. One that carried - something not readable as JSON is a group with that in it still. - Either way the brackets are around something this cannot answer - for. + A group that was cut to fit the length limit is cut back to the + last whole element, so what stands between the brackets reads - + and it is a shorter answer than the group has, with the row it + stopped at reported. One that carried something not readable as + JSON is a group with that in it still. Either way the brackets + are around something this cannot attest to. Never said to be nicely written: what goes between the elements is the separator this function inherits from GROUP_CONCAT, a comma @@ -7505,7 +7507,8 @@ String* Item_func_json_arrayagg::val_str(String *str) Item_func_json_objectagg:: Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item) :Item_sum(thd, item), m_bad_pair(false), m_closed(false), - m_pairs_valid(true), m_pairs_depth(1) + m_pairs_valid(true), m_pairs_depth(1), m_cut(false), m_row_count(0), + m_thd(NULL) { quick_group= FALSE; result.set_charset(collation.collation); @@ -7591,11 +7594,24 @@ void Item_func_json_objectagg::clear() m_closed= false; m_pairs_valid= true; m_pairs_depth= 1; + m_cut= false; + m_row_count= 0; + m_thd= current_thd; if (result.append('{')) m_bad_pair= true; } +/* + The mirror of json_arrayagg_name above, and the other way round: the + message the cut warning goes into supplies its own ')', so the name + written into it is the one with the opening bracket already on it. + func_name() is the bare name here, being written into notes about the + data as well. +*/ +static const char json_objectagg_cut_name[]= "json_objectagg("; + + bool Item_func_json_objectagg::add() { StringBuffer buf; @@ -7606,22 +7622,50 @@ bool Item_func_json_objectagg::add() Json_splice_marks marks(1); String *key; int rc; + /* + Where this pair begins, so that a pair taking the object past the + length limit can be taken back off whole - see the cut at the end + of this function. + */ + uint32 old_length= result.length(); key= args[0]->val_str(&buf); if (args[0]->is_null()) return 0; + m_row_count++; + + /* + Nothing more goes into a group that has been cut, but the rows of it + are still read to the end. An argument is an expression, and working + one out once for every row of the group is what the object has always + done - a row that stops being read stops whatever the expression does, + which is not the caller's answer to give away. So only the writing + stops here; see the cut at the end of this function. + + The value is worked out and dropped rather than written, and it is + asked for the way append_json_value() would ask a value of its type. + The key was worked out above, where every row needs it: whether a row + makes a pair of it is what its being NULL decides, cut or not. + */ + if (m_cut) + { + args[1]->update_null_value(); + return 0; + } + /* Whether this pair needs a separator in front of it is whether a pair has been written already, which is what null_value says until this row changes it. Comparing the buffer against the opening brace would be comparing it against a width that is not always one byte. */ - if (!null_value && result.append(STRING_WITH_LEN(", "))) + if ((!null_value && result.append(STRING_WITH_LEN(", "))) || + (!null_value && DBUG_IF("json_objectagg_separator_out_of_memory"))) goto bad_pair; null_value= 0; - if (result.append('"')) + if (result.append('"') || DBUG_IF("json_objectagg_key_out_of_memory")) goto bad_pair; /* @@ -7642,7 +7686,8 @@ bool Item_func_json_objectagg::add() m_pairs_valid= false; } - if (result.append(STRING_WITH_LEN("\":"))) + if (result.append(STRING_WITH_LEN("\":")) || + DBUG_IF("json_objectagg_colon_out_of_memory")) goto bad_pair; buf.length(0); @@ -7654,6 +7699,37 @@ bool Item_func_json_objectagg::add() if (rc == JSON_APPEND_OOM) goto bad_pair; + /* + The same length limit the sister aggregate is held to, read where it + is read there - the session's setting as it stands while the group is + being built, rather than the width this item was fixed for. + + Cut back to where this pair began rather than to the byte the limit + falls on, for the reason given at + Item_func_json_arrayagg::cut_max_length(): pairs are what the braces + can go round, and half of one is not. + + The limit is on the bytes returned, and the brace that closes the + object is one of them; it is written when the group is asked for, + which is after every pair has been through here. So the room for it + is kept back from the limit rather than left to be discovered once + there is nothing to be done about it. The opening brace is in the + buffer already and is counted where it stands. Neither of them is + one byte wide in a set that writes no character in one byte, which + is why the width is asked for rather than assumed. + */ + { + uint32 close_len= collation.collation->mbminlen; + + DBUG_ASSERT(m_thd); + if (result.length() + close_len > m_thd->gconcat_max_len()) + { + result.length(old_length); + m_cut= true; + report_cut_value_error(m_thd, m_row_count, json_objectagg_cut_name); + } + } + return 0; bad_pair: diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index b58a2ae5bd7d9..0021b7c708139 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -1072,6 +1072,33 @@ class Item_func_json_objectagg : public Item_sum m_elements_depth, for the same reason. */ uint m_pairs_depth; + /* + The group has been cut back to fit group_concat_max_len and nothing + further goes into it. A later pair small enough to fit in what is + left would go in behind the pair that did not fit, putting the + object out of the order the group was read in. Reset for each group + by clear(). + */ + bool m_cut; + /* + Which row of this group is being added, counted over the rows that + have a key to make a pair of - a row whose key is NULL is not one of + them and is not counted. Only the cut warning reads it, to say + where the group stopped. + + Counted per group, where the sister aggregate's row_count runs on + across the groups of a statement. Nothing here needs it to run on, + and a number naming a row of the group is what the warning reads as. + */ + uint m_row_count; + /* + The session this group is being built for, taken once by clear() + rather than looked up again for every row of it. A group runs on + one connection, so the lookup answers the same thing every time it + is made; what is read THROUGH it is read per row still, the length + limit being the session's setting as it stands. + */ + THD *m_thd; Json_result_marks m_marks; public: /* @@ -1083,7 +1110,8 @@ class Item_func_json_objectagg : public Item_sum */ Item_func_json_objectagg(THD *thd, Item *key, Item *value) : Item_sum(thd, key, value), m_bad_pair(false), m_closed(false), - m_pairs_valid(true), m_pairs_depth(1) + m_pairs_valid(true), m_pairs_depth(1), m_cut(false), m_row_count(0), + m_thd(NULL) { quick_group= FALSE; } diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 1c2789f86b737..bb3b2319eeef6 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -3786,7 +3786,7 @@ int group_concat_key_cmp_with_order_with_nulls(void *arg, } -static void report_cut_value_error(THD *thd, uint row_count, const char *fname) +void report_cut_value_error(THD *thd, uint row_count, const char *fname) { size_t fn_len= strlen(fname); char *fname_upper= (char *) my_alloca(fn_len + 1); diff --git a/sql/item_sum.h b/sql/item_sum.h index 77e02cac30f1c..354861ab6c5db 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -1986,6 +1986,19 @@ int dump_leaf_key(void* key_arg, void* item_arg); C_MODE_END +/* + Says that a group was cut to fit group_concat_max_len, naming the row + the cut fell on. 'fname' is written into the message ahead of a ')' + the message supplies itself, so it is the function's name with its + opening bracket already on it. + + Shared with the JSON object aggregate, which cuts its own group and + has to say so the same way, having no GROUP_CONCAT machinery under it + to say it. +*/ +void report_cut_value_error(THD *thd, uint row_count, const char *fname); + + class Item_func_group_concat : public Item_sum { protected: From 71c3c4baca502cbfa13ce9ff497582df14544d58 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 10 Aug 2026 12:36:41 -0400 Subject: [PATCH 23/30] MDEV-40692 Let a walk that wrote no row still finish the group `Item_func_group_concat::val_str()` walks the tree only while `result_finalized` is false, and `dump_leaf_key()` raises that flag for the first row it writes. A row that falls inside the offset is skipped by an earlier return, which decrements the offset counter and leaves the flag alone, so a walk in which every row was skipped writes nothing and raises nothing. The next caller therefore walks again, and by then the counter has been spent, so the rows skipped the first time are written - into a result buffer that belongs to the item and has already been handed out once. Nothing says how often a statement asks for the result of a group. `HAVING` on the alias is the shortest one that asks twice, and the answer must not depend on how often it was asked: SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) FROM t1; '' SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 HAVING v LIKE '%'; 'a,b' The second is wrong on its own terms as well: the group has four rows and the offset is four, so there is nothing left for the limit to take. `DISTINCT` reaches the same walk by the other route and does the same. The walk is what produces the result, so the walk is what finishes it, whether or not it had anything to write. `JSON_ARRAYAGG` is built on this class and puts its brackets round what the parent returns, so the second walk appends the group's elements after the closing bracket the first one put on and the value stops being a document: SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 HAVING v LIKE '%'; [[]"a","b"] `func_gconcat` gains the case: asked once, twice and three times; the `DISTINCT` route; a group per row of output, where one group spends its offset and the other does not; offsets that stop inside the group and no limit at all, none of which move; and the same expression through the aggregate built on this one, with `JSON_VALID` over it. --- mysql-test/main/func_gconcat.result | 65 +++++++++++++++++++++++++++++ mysql-test/main/func_gconcat.test | 43 +++++++++++++++++++ sql/item_sum.cc | 10 +++++ 3 files changed, 118 insertions(+) diff --git a/mysql-test/main/func_gconcat.result b/mysql-test/main/func_gconcat.result index 6c50416ee2717..a727895576599 100644 --- a/mysql-test/main/func_gconcat.result +++ b/mysql-test/main/func_gconcat.result @@ -1556,3 +1556,68 @@ Warning 1260 Row 1 was cut by GROUP_CONCAT() disconnect u; connection default; # End of 10.6 tests +# +# A group with an OFFSET past its last row, asked for more than once +# +# Nothing says how often a statement asks for the result of a group. +# HAVING on the alias is the shortest one that asks twice, and the +# answer must not depend on how often it was asked. +# +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f'); +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +v + +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v + +# two conditions, so it is asked a third time +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%' AND v NOT LIKE 'a%'; +v + +# DISTINCT reaches the same walk by the other route +SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v + +# a group per row of output, so more than one group is replayed +SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 +GROUP BY g HAVING v LIKE '%' ORDER BY g; +g v +1 +2 +# an offset that stops inside the group, which must not move +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +b,c +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 3) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +d +# and no LIMIT at all +SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1 HAVING v LIKE '%'; +v +a,b,c,d +# the same through the aggregate that is built on this one, where an +# element written a second time lands outside the brackets already +# put round the group +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +v +[] +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +[] +SELECT JSON_VALID(JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4)) AS v +FROM t1 WHERE g = 1 HAVING v LIKE '%'; +v +1 +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +["b","c"] +DROP TABLE t1; +# End of 10.11 tests diff --git a/mysql-test/main/func_gconcat.test b/mysql-test/main/func_gconcat.test index 90bfc9188e7dd..650d438710fb8 100644 --- a/mysql-test/main/func_gconcat.test +++ b/mysql-test/main/func_gconcat.test @@ -1137,3 +1137,46 @@ disconnect u; connection default; --echo # End of 10.6 tests + +--echo # +--echo # A group with an OFFSET past its last row, asked for more than once +--echo # +--echo # Nothing says how often a statement asks for the result of a group. +--echo # HAVING on the alias is the shortest one that asks twice, and the +--echo # answer must not depend on how often it was asked. +--echo # +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f'); +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +--echo # two conditions, so it is asked a third time +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%' AND v NOT LIKE 'a%'; +--echo # DISTINCT reaches the same walk by the other route +SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +--echo # a group per row of output, so more than one group is replayed +SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 + GROUP BY g HAVING v LIKE '%' ORDER BY g; +--echo # an offset that stops inside the group, which must not move +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 3) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +--echo # and no LIMIT at all +SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1 HAVING v LIKE '%'; + +--echo # the same through the aggregate that is built on this one, where an +--echo # element written a second time lands outside the brackets already +--echo # put round the group +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +SELECT JSON_VALID(JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4)) AS v + FROM t1 WHERE g = 1 HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +DROP TABLE t1; + +--echo # End of 10.11 tests diff --git a/sql/item_sum.cc b/sql/item_sum.cc index bb3b2319eeef6..c5d4e1902a656 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -4498,6 +4498,16 @@ String* Item_func_group_concat::val_str(String* str) return &result; else DBUG_ASSERT(false); // Can't happen + + /* + The walk is what produces the result, and it has now happened. + dump_leaf_key() raises this itself for the first row it writes, + but a walk in which every row fell inside OFFSET writes none and + raises nothing, so the next caller would walk again - with the + offset already spent, and the rows it skipped the first time + appended to a result that has already been handed out once. + */ + result_finalized= true; } if (table && table->blob_storage && From 7712ac8644156bdd3d05bfe9299026878ca459ef Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 10 Aug 2026 15:08:36 -0400 Subject: [PATCH 24/30] MDEV-40694 Refuse a merge argument that never completed a value `JSON_MERGE` and `JSON_MERGE_PATCH` walk an object argument's keys with while (json_scan_next(je) == 0 && je->state != JST_OBJ_END) which ends on a refusal exactly as it ends on the object's end, the condition asking only whether the scanner moved. Each function has two such loops, one per argument, `copy_value_patch()` has a fifth, and not one of them looks at `je->s.error` after the loop. So a document that broke before its first value was complete is merged from as far as it got: SELECT JSON_MERGE('{}', '{"a":1 "b":2}'); {"a": 1} `{"a":1 "b":2}` has no comma between its members and is not a document at all. What comes back is an object no argument held - composed, well formed, and silently missing whatever stood past the break. Nothing downstream has any reason to reject it, which is what makes it the quiet kind of wrong: `JSON_VALID` answers 1 for it and `JSON_LENGTH` answers 1. The documentation says this does not happen. Of `JSON_MERGE`: An error occurs if any of the arguments are not valid JSON documents. and the function already does exactly that whenever the same breakage is met anywhere else - one loop further in, or in an argument that is not an object. It is the same characters every other JSON function refuses, `JSON_EXTRACT` naming position 8 for the document above. Only the ends of these five loops were letting it through, and which side of the inconsistency an argument falls on is decided by something that has nothing to do with it - whether the *other* argument has any keys, because that is what decides whether an inner loop with a check in it ever runs: SELECT JSON_MERGE('{}', '{"a":1 "b":2}'); -- {"a": 1} SELECT JSON_MERGE('{"b":9}', '{"a":1 "b":2}'); -- NULL, warning Both, in one statement, over one column: SELECT id, JSON_MERGE(a, b) FROM t ORDER BY id; 1 {"a": 1} 2 NULL The fifth loop is the same inconsistency one step further out. `JSON_MERGE_PATCH` walks two objects together only when both arguments are objects; where argument 1 is not one there is nothing to merge with, so argument 2 is copied whole by `copy_value_patch()`, which walks an object's keys the same way and whose caller propagates whatever it returns. So the outcome turned on whether argument 1 was an object at all: id v -- argument 2 is '{"a":1 "b":2}' throughout 1 {"a": 1} -- argument 1 is [] 3 NULL, warning -- argument 1 is {} That loop is reached for every object nested inside the value as well as for the value itself, so a break below the top goes through it too: `JSON_MERGE_PATCH('[]', '{"a":{"p":1 "q":2}}')` composes `{"a": {"p": 1}}`. Each loop end now refuses what its scanner refused. Every case above becomes NULL with the warning the scanner raised, at the position it raised it - position 13 for the nested one - which is what the other side of the inconsistency has always answered. The whole documents beside them do not move. A value read to its end keeps everything it had. Text standing after a complete value is not this and does not move: it is still answered, and still noted rather than warned about, for the reasons given over `report_json_trailing_note()`. `func_json_merge_trailing` said every value in it kept what it always had, which four of its cases no longer do, and it now says which cases the claim is about. `func_json_merge_broken` carries the shapes: both functions, both argument positions, a break inside, a document that stops after a comma, one that stops at the opening brace, a nested break, three arguments, a merge onto SQL NULL, out of a table so one row does not settle another, and `JSON_VALID`/`JSON_TYPE`/`JSON_LENGTH` over the result. Arrays, scalars, whole documents and trailing text are the controls and none of them move. The defect is older than this branch: a released 10.11 answers `{"a": 1}` here too. Both servers compose the same answer, so no comparison between them could have shown it; it comes of reading the functions against their documented contracts instead. --- mysql-test/main/func_json_merge_broken.result | 286 ++++++++++++++++++ mysql-test/main/func_json_merge_broken.test | 133 ++++++++ .../main/func_json_merge_trailing.result | 21 +- mysql-test/main/func_json_merge_trailing.test | 15 +- sql/item_jsonfunc.cc | 46 +++ 5 files changed, 487 insertions(+), 14 deletions(-) create mode 100644 mysql-test/main/func_json_merge_broken.result create mode 100644 mysql-test/main/func_json_merge_broken.test diff --git a/mysql-test/main/func_json_merge_broken.result b/mysql-test/main/func_json_merge_broken.result new file mode 100644 index 0000000000000..d33dca675bba2 --- /dev/null +++ b/mysql-test/main/func_json_merge_broken.result @@ -0,0 +1,286 @@ +# --------------------------------------------------------------- +# what every other JSON function says about these documents +# --------------------------------------------------------------- +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS no_comma; +no_comma +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +SELECT JSON_EXTRACT('{"a":1,', '$') AS stops_after_comma; +stops_after_comma +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{', '$') AS stops_at_once; +stops_at_once +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS no_comma_nested; +no_comma_nested +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +# --------------------------------------------------------------- +# JSON_MERGE - an argument that never completes a value +# --------------------------------------------------------------- +# argument 2 breaks, argument 1 being an empty object. Its keys +# are what is walked first, and an empty object has none, so the +# walk that meets the breakage is the one that was reached second +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS empty_lhs; +empty_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{}', '{"a":1,') AS empty_lhs_stops; +empty_lhs_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('{}', '{') AS empty_lhs_at_once; +empty_lhs_at_once +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS empty_lhs_nested; +empty_lhs_nested +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 13 +# argument 1 breaks, argument 2 being an empty object +SELECT JSON_MERGE('{"a":1 "b":2}', '{}') AS empty_rhs; +empty_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{"a":1,', '{}') AS empty_rhs_stops; +empty_rhs_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +# the same breakage where the other argument has keys of its own, +# which is the case that has always been refused +SELECT JSON_MERGE('{"z":0}', '{"a":1 "b":2}') AS keyed_lhs; +keyed_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{"a":1 "b":2}', '{"z":0}') AS keyed_rhs; +keyed_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 8 +# three arguments, so a composed document is on the left by the +# time the broken one is reached +SELECT JSON_MERGE('{}', '{"a":1 "b":2}', '{"c":3}') AS three_args; +three_args +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{}', '{"c":3}', '{"a":1 "b":2}') AS three_args_last; +three_args_last +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 3 to function 'json_merge_preserve' at position 8 +# --------------------------------------------------------------- +# JSON_MERGE_PATCH, which walks the same two loops +# --------------------------------------------------------------- +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_empty_lhs; +patch_empty_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('{}', '{"a":1,') AS patch_empty_lhs_stops; +patch_empty_lhs_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +SELECT JSON_MERGE_PATCH('{"a":1 "b":2}', '{}') AS patch_empty_rhs; +patch_empty_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('{"z":0}', '{"a":1 "b":2}') AS patch_keyed_lhs; +patch_keyed_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}', '{"c":3}') AS patch_three; +patch_three +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +# merging onto SQL NULL, where the argument is taken over whole +SELECT JSON_MERGE_PATCH(NULL, '{"a":1 "b":2}', '[3,4]') AS patch_null_obj; +patch_null_obj +[3, 4] +SELECT JSON_MERGE_PATCH(NULL, '[1,2 3]', '[3,4]') AS patch_null_arr; +patch_null_arr +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 6 +# --------------------------------------------------------------- +# arrays and scalars, which have always refused this +# --------------------------------------------------------------- +SELECT JSON_MERGE('[]', '[1,2') AS arr_empty_lhs; +arr_empty_lhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('[1]', '[2,3') AS arr_keyed_lhs; +arr_keyed_lhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('[]', '[1,2 3]') AS arr_no_comma; +arr_no_comma +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 6 +SELECT JSON_MERGE_PATCH('[]', '[1,2') AS patch_arr; +patch_arr +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +# --------------------------------------------------------------- +# controls - a complete value with text after it is not this, and +# must keep the answer it has always had +# --------------------------------------------------------------- +SELECT JSON_MERGE('{}', '{"a":1} rubbish') AS trailing_rhs; +trailing_rhs +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2}') AS trailing_lhs; +trailing_lhs +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE_PATCH('{}', '{"a":1} rubbish') AS patch_trailing; +patch_trailing +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE('{}', '{"a":1} {"zzz":9}') AS trailing_whole_doc; +trailing_whole_doc +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +# and documents that are simply documents +SELECT JSON_MERGE('{}', '{"a":1, "b":2}') AS whole; +whole +{"a": 1, "b": 2} +SELECT JSON_MERGE('{"z":0}', '{"a":1, "b":2}') AS whole_keyed; +whole_keyed +{"z": 0, "a": 1, "b": 2} +SELECT JSON_MERGE_PATCH('{}', '{"a":1, "b":2}') AS patch_whole; +patch_whole +{"a": 1, "b": 2} +# --------------------------------------------------------------- +# out of a table, so one row does not settle what another says +# --------------------------------------------------------------- +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES +(1, '{}', '{"a":1 "b":2}'), +(2, '{"z":0}', '{"a":1 "b":2}'), +(3, '{}', '{"a":1, "b":2}'), +(4, '{}', '{"a":1} rubbish'); +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +id v +1 NULL +2 NULL +3 {"a": 1, "b": 2} +4 {"a": 1} +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +id v +1 NULL +2 NULL +3 {"a": 1, "b": 2} +4 {"a": 1} +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +DROP TABLE t; +# --------------------------------------------------------------- +# nothing downstream is handed a value the merging could not read +# --------------------------------------------------------------- +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1 "b":2}')) AS valid_broken; +valid_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_TYPE(JSON_MERGE('{}', '{"a":1 "b":2}')) AS type_broken; +type_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1 "b":2}')) AS length_broken; +length_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1, "b":2}')) AS valid_whole; +valid_whole +1 +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1, "b":2}')) AS length_whole; +length_whole +2 +# --------------------------------------------------------------- +# JSON_MERGE_PATCH where argument 1 is not an object +# --------------------------------------------------------------- +# Merging onto something that is not an object does not walk the +# two objects together; it takes argument 2 whole. That is a +# different walk over the same shape, and the argument has to be +# a document there too. +SELECT JSON_MERGE_PATCH('[]', '{"a":1 "b":2}') AS array_lhs; +array_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('1', '{"a":1 "b":2}') AS scalar_lhs; +scalar_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('"s"', '{"a":1,') AS string_lhs; +string_lhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +SELECT JSON_MERGE_PATCH('true', '{') AS opening_brace_only; +opening_brace_only +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +# the break is inside a member value rather than at the top +SELECT JSON_MERGE_PATCH('[]', '{"a":{"p":1 "q":2}}') AS nested_break; +nested_break +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 13 +# and out of a table, so one row does not settle another +CREATE TABLE tm (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO tm VALUES (1, '[]', '{"a":1 "b":2}'), +(2, '[]', '{"a":1, "b":2}'), +(3, '{}', '{"a":1 "b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM tm ORDER BY id; +id v +1 NULL +2 {"a": 1, "b": 2} +3 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +DROP TABLE tm; +# a whole document down that same walk keeps everything it had +SELECT JSON_MERGE_PATCH('[]', '{"a":1, "b":2}') AS array_lhs_whole; +array_lhs_whole +{"a": 1, "b": 2} +SELECT JSON_MERGE_PATCH('1', '{"a":{"p":1, "q":2}}') AS scalar_lhs_whole; +scalar_lhs_whole +{"a": {"p": 1, "q": 2}} diff --git a/mysql-test/main/func_json_merge_broken.test b/mysql-test/main/func_json_merge_broken.test new file mode 100644 index 0000000000000..e15d571c8bff5 --- /dev/null +++ b/mysql-test/main/func_json_merge_broken.test @@ -0,0 +1,133 @@ +# +# JSON_MERGE and JSON_MERGE_PATCH are documented to merge JSON documents, +# and to raise an error when an argument is not one. An argument that +# breaks before its first value is complete is not one. +# +# That is a different thing from text standing after a value that IS +# complete, which func_json_merge_trailing covers: there the value the +# merging works from is whole, and what stands after it has never reached +# the answer. Here there is no whole value to work from, and what the +# merging composes is part of an object nobody wrote. +# +# The refusal is not new either. It is what these functions already do +# for the same breakage met one loop further in, and what every other +# JSON function does for the same characters. What was escaping it was +# every loop that walks an object's keys and asks only whether the +# scanner moved - one per argument in each function, and one more where +# an argument is copied whole instead of merged with anything. +# + +--echo # --------------------------------------------------------------- +--echo # what every other JSON function says about these documents +--echo # --------------------------------------------------------------- +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS no_comma; +SELECT JSON_EXTRACT('{"a":1,', '$') AS stops_after_comma; +SELECT JSON_EXTRACT('{', '$') AS stops_at_once; +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS no_comma_nested; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE - an argument that never completes a value +--echo # --------------------------------------------------------------- + +--echo # argument 2 breaks, argument 1 being an empty object. Its keys +--echo # are what is walked first, and an empty object has none, so the +--echo # walk that meets the breakage is the one that was reached second +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS empty_lhs; +SELECT JSON_MERGE('{}', '{"a":1,') AS empty_lhs_stops; +SELECT JSON_MERGE('{}', '{') AS empty_lhs_at_once; +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS empty_lhs_nested; + +--echo # argument 1 breaks, argument 2 being an empty object +SELECT JSON_MERGE('{"a":1 "b":2}', '{}') AS empty_rhs; +SELECT JSON_MERGE('{"a":1,', '{}') AS empty_rhs_stops; + +--echo # the same breakage where the other argument has keys of its own, +--echo # which is the case that has always been refused +SELECT JSON_MERGE('{"z":0}', '{"a":1 "b":2}') AS keyed_lhs; +SELECT JSON_MERGE('{"a":1 "b":2}', '{"z":0}') AS keyed_rhs; + +--echo # three arguments, so a composed document is on the left by the +--echo # time the broken one is reached +SELECT JSON_MERGE('{}', '{"a":1 "b":2}', '{"c":3}') AS three_args; +SELECT JSON_MERGE('{}', '{"c":3}', '{"a":1 "b":2}') AS three_args_last; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH, which walks the same two loops +--echo # --------------------------------------------------------------- +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_empty_lhs; +SELECT JSON_MERGE_PATCH('{}', '{"a":1,') AS patch_empty_lhs_stops; +SELECT JSON_MERGE_PATCH('{"a":1 "b":2}', '{}') AS patch_empty_rhs; +SELECT JSON_MERGE_PATCH('{"z":0}', '{"a":1 "b":2}') AS patch_keyed_lhs; +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}', '{"c":3}') AS patch_three; + +--echo # merging onto SQL NULL, where the argument is taken over whole +SELECT JSON_MERGE_PATCH(NULL, '{"a":1 "b":2}', '[3,4]') AS patch_null_obj; +SELECT JSON_MERGE_PATCH(NULL, '[1,2 3]', '[3,4]') AS patch_null_arr; + +--echo # --------------------------------------------------------------- +--echo # arrays and scalars, which have always refused this +--echo # --------------------------------------------------------------- +SELECT JSON_MERGE('[]', '[1,2') AS arr_empty_lhs; +SELECT JSON_MERGE('[1]', '[2,3') AS arr_keyed_lhs; +SELECT JSON_MERGE('[]', '[1,2 3]') AS arr_no_comma; +SELECT JSON_MERGE_PATCH('[]', '[1,2') AS patch_arr; + +--echo # --------------------------------------------------------------- +--echo # controls - a complete value with text after it is not this, and +--echo # must keep the answer it has always had +--echo # --------------------------------------------------------------- +SELECT JSON_MERGE('{}', '{"a":1} rubbish') AS trailing_rhs; +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2}') AS trailing_lhs; +SELECT JSON_MERGE_PATCH('{}', '{"a":1} rubbish') AS patch_trailing; +SELECT JSON_MERGE('{}', '{"a":1} {"zzz":9}') AS trailing_whole_doc; + +--echo # and documents that are simply documents +SELECT JSON_MERGE('{}', '{"a":1, "b":2}') AS whole; +SELECT JSON_MERGE('{"z":0}', '{"a":1, "b":2}') AS whole_keyed; +SELECT JSON_MERGE_PATCH('{}', '{"a":1, "b":2}') AS patch_whole; + +--echo # --------------------------------------------------------------- +--echo # out of a table, so one row does not settle what another says +--echo # --------------------------------------------------------------- +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES + (1, '{}', '{"a":1 "b":2}'), + (2, '{"z":0}', '{"a":1 "b":2}'), + (3, '{}', '{"a":1, "b":2}'), + (4, '{}', '{"a":1} rubbish'); +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +DROP TABLE t; + +--echo # --------------------------------------------------------------- +--echo # nothing downstream is handed a value the merging could not read +--echo # --------------------------------------------------------------- +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1 "b":2}')) AS valid_broken; +SELECT JSON_TYPE(JSON_MERGE('{}', '{"a":1 "b":2}')) AS type_broken; +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1 "b":2}')) AS length_broken; +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1, "b":2}')) AS valid_whole; +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1, "b":2}')) AS length_whole; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH where argument 1 is not an object +--echo # --------------------------------------------------------------- +--echo # Merging onto something that is not an object does not walk the +--echo # two objects together; it takes argument 2 whole. That is a +--echo # different walk over the same shape, and the argument has to be +--echo # a document there too. +SELECT JSON_MERGE_PATCH('[]', '{"a":1 "b":2}') AS array_lhs; +SELECT JSON_MERGE_PATCH('1', '{"a":1 "b":2}') AS scalar_lhs; +SELECT JSON_MERGE_PATCH('"s"', '{"a":1,') AS string_lhs; +SELECT JSON_MERGE_PATCH('true', '{') AS opening_brace_only; +--echo # the break is inside a member value rather than at the top +SELECT JSON_MERGE_PATCH('[]', '{"a":{"p":1 "q":2}}') AS nested_break; +--echo # and out of a table, so one row does not settle another +CREATE TABLE tm (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO tm VALUES (1, '[]', '{"a":1 "b":2}'), + (2, '[]', '{"a":1, "b":2}'), + (3, '{}', '{"a":1 "b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM tm ORDER BY id; +DROP TABLE tm; +--echo # a whole document down that same walk keeps everything it had +SELECT JSON_MERGE_PATCH('[]', '{"a":1, "b":2}') AS array_lhs_whole; +SELECT JSON_MERGE_PATCH('1', '{"a":{"p":1, "q":2}}') AS scalar_lhs_whole; diff --git a/mysql-test/main/func_json_merge_trailing.result b/mysql-test/main/func_json_merge_trailing.result index 9a4aceca009cd..841d75713b2ad 100644 --- a/mysql-test/main/func_json_merge_trailing.result +++ b/mysql-test/main/func_json_merge_trailing.result @@ -216,8 +216,9 @@ DROP TABLE t; # where the trouble is said to be, which has to be where every # other JSON function reading the same bytes says it is # --------------------------------------------------------------- -# broken inside the value rather than after it, so the merging -# stops in the middle and composes out of what it had +# broken inside the value rather than after it, so there is no +# whole value to compose from and the merging refuses it - what is +# being compared here is where each of them says the trouble is SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS extract_says; extract_says NULL @@ -225,14 +226,14 @@ Warnings: Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS merge_says; merge_says -{"a": 1} +NULL Warnings: -Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_says; patch_says -{"a": 1} +NULL Warnings: -Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 # deeper in, so the position is further along SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS extract_deep; extract_deep @@ -252,14 +253,14 @@ Warnings: Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' SELECT JSON_MERGE('{"a":1,', '{}') AS merge_stops_lhs; merge_stops_lhs -{"a": 1} +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' SELECT JSON_MERGE('{}', '{"a":1,') AS merge_stops_rhs; merge_stops_rhs -{"a": 1} +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' # --------------------------------------------------------------- # a note does not become an error where a warning would # --------------------------------------------------------------- diff --git a/mysql-test/main/func_json_merge_trailing.test b/mysql-test/main/func_json_merge_trailing.test index 8167faa5e2dcb..0d9d5de5d4225 100644 --- a/mysql-test/main/func_json_merge_trailing.test +++ b/mysql-test/main/func_json_merge_trailing.test @@ -9,8 +9,14 @@ # not a warning, because a warning becomes an error under a strict mode # and would take away an answer that has always been given. # -# So every case below has to keep the value it has always had. The note -# is the only thing that is new. +# So a case whose value the merging read to its end has to keep the value +# it has always had, and the note is the only thing that is new about it. +# +# A document that breaks before its first value is complete is a +# different thing, having no whole value for any of the above to be true +# of, and it is refused rather than noted. func_json_merge_broken +# carries those; the section below reaches far enough into them to show +# that both say where the trouble is in the same place. # --echo # --------------------------------------------------------------- @@ -122,8 +128,9 @@ DROP TABLE t; --echo # other JSON function reading the same bytes says it is --echo # --------------------------------------------------------------- ---echo # broken inside the value rather than after it, so the merging ---echo # stops in the middle and composes out of what it had +--echo # broken inside the value rather than after it, so there is no +--echo # whole value to compose from and the merging refuses it - what is +--echo # being compared here is where each of them says the trouble is SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS extract_says; SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS merge_says; SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_says; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 7c8bb73781f7d..c4b3eb7546916 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -4626,6 +4626,26 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2, continue; } + /* + The loop above ends on a refusal exactly as it ends on the object's + end, its condition asking only whether the scanner moved. An + argument that broke before its first value was complete would + otherwise be merged from as far as it got, and what came out would + be an object no argument held: composed, well formed, and missing + whatever stood past the break. Nothing downstream has any reason + to reject such a value, which is what makes it the quiet kind of + wrong. + + Refusing it is what these functions already do for the same + breakage met one loop further in, and what every other JSON + function does with the same characters. Every loop of this shape + was letting it through, here and in the two functions' other + argument, and once more where an argument is copied whole instead + of being merged with anything. + */ + if (unlikely(je1->s.error)) + return 1; + *je2= sav_je2; /* Now loop through the Json_2 keys. @@ -4681,6 +4701,10 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2, continue; } + /* The other loop, ending the same way - see above. */ + if (unlikely(je2->s.error)) + return 1; + if (str->append('}')) return 3; } @@ -5063,6 +5087,17 @@ static int copy_value_patch(String *str, json_engine_t *je) copy_value_patch(str, je)) return 1; } + + /* + The same loop end again, in the arm taken when there is nothing to + merge with and the argument is copied whole. Everything said over + the walk that merges two objects together holds here word for word, + and this loop is reached for every object nested inside the value as + well as for the value itself. + */ + if (unlikely(je->s.error)) + return 1; + if (str->append('}')) return 1; @@ -5176,6 +5211,13 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, continue; } + /* + Ends on a refusal as well as on the object's end, exactly as the + loops in do_merge() do, and refused here for the same reason. + */ + if (unlikely(je1->s.error)) + return 1; + *je2= sav_je2; /* Now loop through the Json_2 keys. @@ -5243,6 +5285,10 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, continue; } + /* The other loop, ending the same way - see above. */ + if (unlikely(je2->s.error)) + return 1; + if (str->append('}')) return 3; } From 358b9732ae93d6748be07168f2150877b9c75807 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Mon, 10 Aug 2026 17:24:15 -0400 Subject: [PATCH 25/30] MDEV-40696 Count a JSON aggregate's brackets against its length limit `JSON_ARRAYAGG` is documented in the same words as `GROUP_CONCAT`, which it derives from: *"the maximum returned length in bytes is determined by the `group_concat_max_len` server system variable."* `GROUP_CONCAT` ends on that limit exactly. This ended two bytes past it. SET SESSION group_concat_max_len = 25; SELECT LENGTH(JSON_ARRAYAGG(s)) FROM t1; 27 `dump_leaf_key()` tests what it has accumulated once per row. For `GROUP_CONCAT` that is the whole answer, but this function's brackets are written later, in `val_str()`, when the group is asked for. So the row test was made against a body that was not yet the answer, and the answer came back longer than the body it was tested by. Where the elements ended exactly on the limit there was nothing to cut and no warning was raised, so the answer went over in silence - which is the shape above, `["aaaaaaaaaa","bbbbbbbbbb"]` at a cap of 25. `Item_func_group_concat` now says how much of the limit is spoken for by something not in the buffer yet. It answers nothing, so `GROUP_CONCAT` is not moved by any of this; the array aggregate answers with its two brackets, which go in through `String::append()` and are as wide as the narrowest character of the result's set. `JSON_OBJECTAGG` builds its opening brace into the buffer as the group begins and already kept back the room for the closing one, so nothing about the answer it returns moves either. The cost is that the last element goes rather than the last two bytes. A whole-element cut has nothing smaller to give back, so `["ab","cd"]` at a cap of 9 becomes `["ab"]`, and a cap that no longer reaches the end of the first element leaves `[]` - which is what a cap too small for any element already meant here. Nothing stops reading as JSON, and the warning now names the row that would not fit rather than the one after it. ## One limit that cannot be kept `group_concat_max_len` may be set as low as 4, and an empty array is as short as this function goes. In a set of four bytes to the character that is eight bytes, so a cap of 4 is answered by `[]` and the answer is over the cap with nothing left to cut. That is why `Item_func_json_arrayagg::fix_fields()` carries room for the brackets on top of the cap in the width it declares. It was written for the overshoot this commit removes, and it is still needed for this: the most this function returns is the cap or the two brackets, whichever is longer. That width asks for the room the brackets take rather than working it out a second time: it is the same figure the limit keeps back, and a figure written down twice is one that can be changed in one place and not the other - which is how the two came to disagree here in the first place. ## The same floor, and the same width, for the object `JSON_OBJECTAGG` goes no shorter than `{}` and writes its braces the same way, and declared a width worked out from the cap and nothing else. A cap too small for them left a column narrower than the shortest answer the function gives, and what did not fit was cut into it as the column was filled. At a cap of 4 in utf32 the column is one character wide, holds `{`, and no longer reads as JSON: obj_h obj_ok 0000007B 0 -- the answer was {}, of which this is half Nothing said so. The note raised alongside is about a row of the group being cut, which is a different thing and true anyway. It carries the same allowance now, and how wide a brace is is said in one place, which the row test below reads as well rather than formatting out its own. The width is worked out in a type wide enough to hold it. `max_length` is 32 bits and the cap alone can fill it: a gigabyte, the most `max_allowed_packet` allows, in a set of four bytes to the character. An addition made in that width carries round to nothing, and what comes out is a column too narrow to hold any answer at all - which the array aggregate's allowance, added in that width, did. The width the client is told about moves with it, in `type_json` and in two tests here that record the two aggregates one after the other. Each had the object short of the array by the room for the punctuation; the two now agree, which they should, having the same floor and writing the same two characters round the group. New test `func_json_agg_cap`, which walks the cap across a whole element and past the end of the group, over one-byte and two-byte sets, asserts every answer against the cap it was given, pins the empty-array floor, and stands `GROUP_CONCAT` beside the two aggregates under one cap. Two further sections are about the width rather than the answer: a column made out of both aggregates under a cap too small for their punctuation, read back as bytes and asked whether it is still JSON, and one made under a cap that fills the width on its own. --- mysql-test/main/func_json.result | 4 +- mysql-test/main/func_json_agg_cap.result | 760 ++++++++++++++++++++ mysql-test/main/func_json_agg_cap.test | 167 +++++ mysql-test/main/func_json_agg_cut.result | 90 +-- mysql-test/main/func_json_agg_limits.result | 35 +- mysql-test/main/func_json_charset.result | 2 +- mysql-test/main/type_json.result | 4 +- sql/item_jsonfunc.cc | 29 +- sql/item_jsonfunc.h | 15 + sql/item_sum.cc | 15 +- sql/item_sum.h | 8 + 11 files changed, 1062 insertions(+), 67 deletions(-) create mode 100644 mysql-test/main/func_json_agg_cap.result create mode 100644 mysql-test/main/func_json_agg_cap.test diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index d895884f3b258..d0e7b870547e0 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -1832,9 +1832,9 @@ create temporary table t (c varchar(20) character set latin1); insert into t values ('ab'), ('cd'), ('ef'); select json_arrayagg(c order by c) from t; json_arrayagg(c order by c) -["ab","cd"] +["ab"] Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() drop temporary table t; # End of 10.6 tests SELECT json_extract(t.j, '$') diff --git a/mysql-test/main/func_json_agg_cap.result b/mysql-test/main/func_json_agg_cap.result new file mode 100644 index 0000000000000..c6eb191f20e5d --- /dev/null +++ b/mysql-test/main/func_json_agg_cap.result @@ -0,0 +1,760 @@ +# +# The most a JSON aggregate returns is group_concat_max_len +# bytes, the same limit and the same wording GROUP_CONCAT is held +# to. The brackets that go round an aggregated array are part of +# what is returned and are written after the length has been +# tested, so the room for them has to be kept back when the test +# is made. Where the elements ended exactly on the limit there +# was nothing to cut, and the answer went over it in silence. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (a VARCHAR(20)); +INSERT INTO t1 VALUES ('aaaaaaaaaa'), ('bbbbbbbbbb'); +# +# 1. The limit walked across the whole of an element and past the +# end of the group. Nothing may answer longer than its cap. +# +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 4 AS within FROM t1; +cap v len within +4 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 5 AS within FROM t1; +cap v len within +5 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 6 AS within FROM t1; +cap v len within +6 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 7 AS within FROM t1; +cap v len within +7 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 8 AS within FROM t1; +cap v len within +8 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 9 AS within FROM t1; +cap v len within +9 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 10 AS within FROM t1; +cap v len within +10 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 11 AS within FROM t1; +cap v len within +11 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 12 AS within FROM t1; +cap v len within +12 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 13 AS within FROM t1; +cap v len within +13 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 14 AS within FROM t1; +cap v len within +14 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 15 AS within FROM t1; +cap v len within +15 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 16 AS within FROM t1; +cap v len within +16 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 17 AS within FROM t1; +cap v len within +17 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 18 AS within FROM t1; +cap v len within +18 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 19 AS within FROM t1; +cap v len within +19 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 20 AS within FROM t1; +cap v len within +20 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 21 AS within FROM t1; +cap v len within +21 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 22 AS within FROM t1; +cap v len within +22 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 23 AS within FROM t1; +cap v len within +23 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 24; +SELECT 24 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 24 AS within FROM t1; +cap v len within +24 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 25; +SELECT 25 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 25 AS within FROM t1; +cap v len within +25 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 26; +SELECT 26 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 26 AS within FROM t1; +cap v len within +26 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 27; +SELECT 27 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 27 AS within FROM t1; +cap v len within +27 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 28; +SELECT 28 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 28 AS within FROM t1; +cap v len within +28 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 29; +SELECT 29 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 29 AS within FROM t1; +cap v len within +29 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 30; +SELECT 30 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 30 AS within FROM t1; +cap v len within +30 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 31; +SELECT 31 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 31 AS within FROM t1; +cap v len within +31 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = DEFAULT; +# +# 2. The two aggregates side by side under one cap, and +# GROUP_CONCAT beside them, which has nothing to put on +# afterwards and has always ended on its limit exactly. +# +SET SESSION group_concat_max_len = 25; +SELECT LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS arr_len, +LENGTH(JSON_OBJECTAGG(a, 1)) AS obj_len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS gc_len, +@@group_concat_max_len AS cap FROM t1; +arr_len obj_len gc_len cap +14 16 21 25 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 3. A set that writes no character in one byte. A bracket costs +# two bytes there, so four is what has to be kept back. +# +CREATE TABLE t2 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t2 VALUES ('aaaa'), ('bbbb'); +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 16 AS within FROM t2; +cap h len within +16 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 17 AS within FROM t2; +cap h len within +17 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 18 AS within FROM t2; +cap h len within +18 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 19 AS within FROM t2; +cap h len within +19 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 20 AS within FROM t2; +cap h len within +20 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 21 AS within FROM t2; +cap h len within +21 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 22 AS within FROM t2; +cap h len within +22 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 23 AS within FROM t2; +cap h len within +23 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 24; +SELECT 24 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 24 AS within FROM t2; +cap h len within +24 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 25; +SELECT 25 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 25 AS within FROM t2; +cap h len within +25 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 26; +SELECT 26 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 26 AS within FROM t2; +cap h len within +26 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 27; +SELECT 27 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 27 AS within FROM t2; +cap h len within +27 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 28; +SELECT 28 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 28 AS within FROM t2; +cap h len within +28 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 29; +SELECT 29 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 29 AS within FROM t2; +cap h len within +29 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 30; +SELECT 30 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 30 AS within FROM t2; +cap h len within +30 005B002200610061006100610022002C002200620062006200620022005D 30 1 +SET SESSION group_concat_max_len = 31; +SELECT 31 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 31 AS within FROM t2; +cap h len within +31 005B002200610061006100610022002C002200620062006200620022005D 30 1 +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; +# +# 4. A cap that leaves no room for a whole element still leaves +# the brackets, which is what an empty group answers as well. +# +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len FROM t1; +v len +[] 2 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a IS NULL; +v +NULL +SET SESSION group_concat_max_len = DEFAULT; +# +# 5. The one place the limit cannot be kept: a cap smaller than +# the brackets themselves. An empty array is as short as this +# function goes, and in a set of four bytes to the character it +# is eight bytes, against a limit that may be set as low as +# four. There is nothing left to cut, so the brackets stand +# and the answer is over the cap - which is why the width this +# item declares carries room for them on top of the cap. +# +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t3 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 4 AS within FROM t3; +h len within +0000005B0000005D 8 0 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +# and the same cap where the brackets do fit +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 8 AS within FROM t3; +h len within +0000005B0000005D 8 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; +# +# 6. GROUP_CONCAT is not moved by any of this: it returns what +# it accumulated, and its cut lands on the byte the limit falls +# on as it always has. +# +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 4 AS within FROM t1; +cap v len within +4 aaaa 4 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 5 AS within FROM t1; +cap v len within +5 aaaaa 5 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 6 AS within FROM t1; +cap v len within +6 aaaaaa 6 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 7 AS within FROM t1; +cap v len within +7 aaaaaaa 7 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 8 AS within FROM t1; +cap v len within +8 aaaaaaaa 8 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 9 AS within FROM t1; +cap v len within +9 aaaaaaaaa 9 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 10 AS within FROM t1; +cap v len within +10 aaaaaaaaaa 10 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 11 AS within FROM t1; +cap v len within +11 aaaaaaaaaa, 11 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 12 AS within FROM t1; +cap v len within +12 aaaaaaaaaa,b 12 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 13 AS within FROM t1; +cap v len within +13 aaaaaaaaaa,bb 13 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 14 AS within FROM t1; +cap v len within +14 aaaaaaaaaa,bbb 14 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 15 AS within FROM t1; +cap v len within +15 aaaaaaaaaa,bbbb 15 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 16 AS within FROM t1; +cap v len within +16 aaaaaaaaaa,bbbbb 16 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 17 AS within FROM t1; +cap v len within +17 aaaaaaaaaa,bbbbbb 17 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 18 AS within FROM t1; +cap v len within +18 aaaaaaaaaa,bbbbbbb 18 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 19 AS within FROM t1; +cap v len within +19 aaaaaaaaaa,bbbbbbbb 19 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 20 AS within FROM t1; +cap v len within +20 aaaaaaaaaa,bbbbbbbbb 20 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 21 AS within FROM t1; +cap v len within +21 aaaaaaaaaa,bbbbbbbbbb 21 1 +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 22 AS within FROM t1; +cap v len within +22 aaaaaaaaaa,bbbbbbbbbb 21 1 +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 23 AS within FROM t1; +cap v len within +23 aaaaaaaaaa,bbbbbbbbbb 21 1 +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t1; +# +# 7. The width the item declares, which is the width a column made +# out of the answer is given. The answer is over the cap +# wherever the punctuation alone is - section 5 - so a width +# worked out from the cap and nothing else cannot hold it, and +# what reaches the column is cut without a word said. +# +CREATE TABLE t4 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t4 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +# Non-strict, so that the cut this cap makes - which is section 5 +# over again - is a warning and the statement runs to the end, +# where the width is what is being looked at. +SET SESSION sql_mode = ''; +CREATE TABLE t5 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, +JSON_OBJECTAGG(a, 1) AS obj FROM t4; +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SHOW CREATE TABLE t5; +Table Create Table +t5 CREATE TABLE `t5` ( + `arr` varchar(3) CHARACTER SET utf32 COLLATE utf32_general_ci DEFAULT NULL, + `obj` varchar(3) CHARACTER SET utf32 COLLATE utf32_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT HEX(arr) AS arr_h, JSON_VALID(arr) AS arr_ok, +HEX(obj) AS obj_h, JSON_VALID(obj) AS obj_ok FROM t5; +arr_h arr_ok obj_h obj_ok +0000005B0000005D 1 0000007B0000007D 1 +DROP TABLE t5; +SET SESSION sql_mode = DEFAULT; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; +# +# 8. That width is worked out in a wider type than it is kept in. +# A cap that fills the narrower type on its own leaves the room +# for the brackets nowhere to go, and a width that wrapped +# round would be a column too narrow to hold any answer at all. +# +SET @old_max_allowed_packet = @@GLOBAL.max_allowed_packet; +SET GLOBAL max_allowed_packet = 1073741824; +connect con_wide,localhost,root,,test; +SET SESSION group_concat_max_len = 1073741824; +# Four bytes to the character, so that the cap alone fills the +# width: a narrower set leaves room the brackets can go in. +CREATE TABLE t6 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t6 VALUES ('aa'), ('bb'); +CREATE TABLE t7 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, +JSON_OBJECTAGG(a, 1) AS obj FROM t6; +SHOW CREATE TABLE t7; +Table Create Table +t7 CREATE TABLE `t7` ( + `arr` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `obj` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT arr, obj FROM t7; +arr obj +["aa","bb"] {"aa":1, "bb":1} +DROP TABLE t7, t6; +disconnect con_wide; +connection default; +SET GLOBAL max_allowed_packet = @old_max_allowed_packet; diff --git a/mysql-test/main/func_json_agg_cap.test b/mysql-test/main/func_json_agg_cap.test new file mode 100644 index 0000000000000..7a39ce9d876f5 --- /dev/null +++ b/mysql-test/main/func_json_agg_cap.test @@ -0,0 +1,167 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # The most a JSON aggregate returns is group_concat_max_len +--echo # bytes, the same limit and the same wording GROUP_CONCAT is held +--echo # to. The brackets that go round an aggregated array are part of +--echo # what is returned and are written after the length has been +--echo # tested, so the room for them has to be kept back when the test +--echo # is made. Where the elements ended exactly on the limit there +--echo # was nothing to cut, and the answer went over it in silence. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (a VARCHAR(20)); +INSERT INTO t1 VALUES ('aaaaaaaaaa'), ('bbbbbbbbbb'); + +--echo # +--echo # 1. The limit walked across the whole of an element and past the +--echo # end of the group. Nothing may answer longer than its cap. +--echo # + +--let $cap= 4 +while ($cap < 32) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= $cap AS within FROM t1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 2. The two aggregates side by side under one cap, and +--echo # GROUP_CONCAT beside them, which has nothing to put on +--echo # afterwards and has always ended on its limit exactly. +--echo # + +SET SESSION group_concat_max_len = 25; +SELECT LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS arr_len, + LENGTH(JSON_OBJECTAGG(a, 1)) AS obj_len, + LENGTH(GROUP_CONCAT(a ORDER BY a)) AS gc_len, + @@group_concat_max_len AS cap FROM t1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 3. A set that writes no character in one byte. A bracket costs +--echo # two bytes there, so four is what has to be kept back. +--echo # + +CREATE TABLE t2 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t2 VALUES ('aaaa'), ('bbbb'); + +--let $cap= 16 +while ($cap < 32) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= $cap AS within FROM t2; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 4. A cap that leaves no room for a whole element still leaves +--echo # the brackets, which is what an empty group answers as well. +--echo # + +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len FROM t1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a IS NULL; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 5. The one place the limit cannot be kept: a cap smaller than +--echo # the brackets themselves. An empty array is as short as this +--echo # function goes, and in a set of four bytes to the character it +--echo # is eight bytes, against a limit that may be set as low as +--echo # four. There is nothing left to cut, so the brackets stand +--echo # and the answer is over the cap - which is why the width this +--echo # item declares carries room for them on top of the cap. +--echo # + +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t3 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 4 AS within FROM t3; +--echo # and the same cap where the brackets do fit +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 8 AS within FROM t3; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; + +--echo # +--echo # 6. GROUP_CONCAT is not moved by any of this: it returns what +--echo # it accumulated, and its cut lands on the byte the limit falls +--echo # on as it always has. +--echo # + +--let $cap= 4 +while ($cap < 24) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, GROUP_CONCAT(a ORDER BY a) AS v, + LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, + LENGTH(GROUP_CONCAT(a ORDER BY a)) <= $cap AS within FROM t1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +DROP TABLE t1; + +--echo # +--echo # 7. The width the item declares, which is the width a column made +--echo # out of the answer is given. The answer is over the cap +--echo # wherever the punctuation alone is - section 5 - so a width +--echo # worked out from the cap and nothing else cannot hold it, and +--echo # what reaches the column is cut without a word said. +--echo # + +CREATE TABLE t4 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t4 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +--echo # Non-strict, so that the cut this cap makes - which is section 5 +--echo # over again - is a warning and the statement runs to the end, +--echo # where the width is what is being looked at. +SET SESSION sql_mode = ''; +CREATE TABLE t5 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, + JSON_OBJECTAGG(a, 1) AS obj FROM t4; +SHOW CREATE TABLE t5; +SELECT HEX(arr) AS arr_h, JSON_VALID(arr) AS arr_ok, + HEX(obj) AS obj_h, JSON_VALID(obj) AS obj_ok FROM t5; +DROP TABLE t5; +SET SESSION sql_mode = DEFAULT; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; + +--echo # +--echo # 8. That width is worked out in a wider type than it is kept in. +--echo # A cap that fills the narrower type on its own leaves the room +--echo # for the brackets nowhere to go, and a width that wrapped +--echo # round would be a column too narrow to hold any answer at all. +--echo # + +SET @old_max_allowed_packet = @@GLOBAL.max_allowed_packet; +SET GLOBAL max_allowed_packet = 1073741824; +--connect (con_wide,localhost,root,,test) +SET SESSION group_concat_max_len = 1073741824; +--echo # Four bytes to the character, so that the cap alone fills the +--echo # width: a narrower set leaves room the brackets can go in. +CREATE TABLE t6 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t6 VALUES ('aa'), ('bb'); +CREATE TABLE t7 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, + JSON_OBJECTAGG(a, 1) AS obj FROM t6; +SHOW CREATE TABLE t7; +SELECT arr, obj FROM t7; +DROP TABLE t7, t6; +--disconnect con_wide +--connection default +SET GLOBAL max_allowed_packet = @old_max_allowed_packet; diff --git a/mysql-test/main/func_json_agg_cut.result b/mysql-test/main/func_json_agg_cut.result index 4d07a94a6d168..966dc0276ba40 100644 --- a/mysql-test/main/func_json_agg_cut.result +++ b/mysql-test/main/func_json_agg_cut.result @@ -45,18 +45,18 @@ SET SESSION group_concat_max_len = 6; SELECT 6 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; cap v ok -6 ["aaaa"] 1 +6 [] 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 7; SELECT 7 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; cap v ok -7 ["aaaa"] 1 +7 [] 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 8; SELECT 8 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; @@ -101,18 +101,18 @@ SET SESSION group_concat_max_len = 13; SELECT 13 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; cap v ok -13 ["aaaa","bbbb"] 1 +13 ["aaaa"] 1 Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 14; SELECT 14 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; cap v ok -14 ["aaaa","bbbb"] 1 +14 ["aaaa"] 1 Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 15; SELECT 15 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; @@ -331,18 +331,18 @@ SET SESSION group_concat_max_len = 4; SELECT 4 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; cap v ok -4 [1111] 1 +4 [] 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 5; SELECT 5 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; cap v ok -5 [1111] 1 +5 [] 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 6; SELECT 6 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; @@ -371,18 +371,18 @@ SET SESSION group_concat_max_len = 9; SELECT 9 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; cap v ok -9 [1111,2222] 1 +9 [1111] 1 Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 10; SELECT 10 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; cap v ok -10 [1111,2222] 1 +10 [1111] 1 Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 11; SELECT 11 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; @@ -411,12 +411,18 @@ SET SESSION group_concat_max_len = 14; SELECT 14 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; cap v ok -14 [1111,2222,3333] 1 +14 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 15; SELECT 15 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; cap v ok -15 [1111,2222,3333] 1 +15 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 16; SELECT 16 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; @@ -457,10 +463,10 @@ DROP TABLE t2; SET SESSION group_concat_max_len = 14; SELECT g, JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 GROUP BY g ORDER BY g; g v -1 ["aaaa","bbbb"] +1 ["aaaa"] 2 ["ee","ff"] Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; g v 1 {} @@ -476,14 +482,14 @@ SET SESSION group_concat_max_len = DEFAULT; SET SESSION group_concat_max_len = 14; SELECT JSON_ARRAYAGG(DISTINCT a ORDER BY a) AS v FROM t1 WHERE g = 1; v -["aaaa","bbbb"] +["aaaa"] Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; v -["dddd","cccc"] +["dddd"] Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = DEFAULT; # # 7. A cut result read back by another JSON function. @@ -493,17 +499,17 @@ SELECT JSON_EXTRACT(JSON_ARRAYAGG(a ORDER BY a), '$[0]') AS v FROM t1 WHERE g = v "aaaa" Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SELECT JSON_LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; v -2 +1 Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SELECT JSON_ARRAY(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; v -[["aaaa","bbbb"]] +[["aaaa"]] Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SELECT JSON_LENGTH(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; v 0 @@ -533,10 +539,10 @@ SET SESSION group_concat_max_len = 7; SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; h ok -5B22C3A4C3A4225D 1 +5B5D 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 8; SELECT HEX(JSON_OBJECTAGG(a, 1)) AS h, JSON_VALID(JSON_OBJECTAGG(a, 1)) AS ok FROM t3; @@ -607,9 +613,9 @@ SET @calls= 0; SET SESSION group_concat_max_len = 14; SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; v -["aaaa","bbbb"] +["aaaa"] Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = DEFAULT; SELECT @uncut AS uncut_calls, @calls AS cut_calls; uncut_calls cut_calls diff --git a/mysql-test/main/func_json_agg_limits.result b/mysql-test/main/func_json_agg_limits.result index 6daf1b53b44eb..0199fa85fdf06 100644 --- a/mysql-test/main/func_json_agg_limits.result +++ b/mysql-test/main/func_json_agg_limits.result @@ -94,7 +94,10 @@ SET SESSION group_concat_max_len = 39; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B2261616161616161616161222C2262626262626262626262222C2263636363636363636363225D 1 +5B2261616161616161616161222C2262626262626262626262225D 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 30; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; @@ -134,19 +137,19 @@ SET SESSION group_concat_max_len = 13; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B2261616161616161616161225D 1 +5B5D 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() # the cap falls inside the first member SET SESSION group_concat_max_len = 12; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; h ok -5B2261616161616161616161225D 1 +5B5D 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 5; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1 WHERE g = 1; @@ -167,7 +170,9 @@ Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 4; SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1 WHERE a = 'dd'; h -5B226464225D +5B5D +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = DEFAULT; # # 3. A cap that falls inside an escape sequence, where the bytes @@ -185,17 +190,17 @@ Warning 1260 Row 2 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 9; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok -5B2278785C227979225D 1 +5B5D 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 8; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok -5B2278785C227979225D 1 +5B5D 1 Warnings: -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() -Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = 7; SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; h ok @@ -353,7 +358,7 @@ v ["dd","ee"] SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def v 252 (format=json) 4194304 22 Y 0 0 45 +def v 252 (format=json) 4194312 22 Y 0 0 45 v {"dd":"dd", "ee":"ee"} SELECT JSON_ARRAY(1) AS v; diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result index a0c1a10d649bb..acfc5e7fdebd4 100644 --- a/mysql-test/main/func_json_charset.result +++ b/mysql-test/main/func_json_charset.result @@ -515,7 +515,7 @@ v [1,2] SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def v 252 (format=json) 16777216 14 Y 0 0 45 +def v 252 (format=json) 16777224 14 Y 0 0 45 v {"1":1, "2":2} # diff --git a/mysql-test/main/type_json.result b/mysql-test/main/type_json.result index 8e0b7e44a923f..708097558341b 100644 --- a/mysql-test/main/type_json.result +++ b/mysql-test/main/type_json.result @@ -148,8 +148,8 @@ JSON_ARRAYAGG(1) JSON_ARRAYAGG(a) [1] [{"a":"b"}] SELECT JSON_OBJECTAGG('a','b'), JSON_OBJECTAGG('a',a) FROM t1; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def JSON_OBJECTAGG('a','b') 252 (format=json) 9437184 9 Y 0 0 33 -def JSON_OBJECTAGG('a',a) 252 (format=json) 12582912 15 Y 128 0 33 +def JSON_OBJECTAGG('a','b') 252 (format=json) 9437190 9 Y 0 0 33 +def JSON_OBJECTAGG('a',a) 252 (format=json) 12582918 15 Y 128 0 33 JSON_OBJECTAGG('a','b') JSON_OBJECTAGG('a',a) {"a":"b"} {"a":{"a":"b"}} DROP TABLE t1; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index c4b3eb7546916..0914c8f3d9c19 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -7337,8 +7337,19 @@ bool Item_func_json_arrayagg::fix_fields(THD *thd, Item **ref) { bool res= Item_func_group_concat::fix_fields(thd, ref); m_tmp_json.set_charset(collation.collation); - /* account for opening and closing brackets */ - max_length= MY_MIN(max_length + 2*collation.collation->mbminlen, UINT_MAX32); + /* + Account for the opening and closing brackets, asking for the room + they take rather than working it out again: the width the item + declares and the room the limit keeps back for them are the same + figure, and two formats of it are two chances for one to move. + + Added in a type wide enough to hold the sum. A cap of a gigabyte in + a set of four bytes to the character fills the width on its own, and + an addition made in that width would carry the brackets round to + nothing. + */ + max_length= (uint32) MY_MIN((ulonglong) max_length + + reserved_result_length(), UINT_MAX32); return res; } @@ -7592,9 +7603,19 @@ Item_func_json_objectagg::fix_fields(THD *thd, Item **ref) result.set_charset(collation.collation); result_field= 0; null_value= 1; + /* + The cap, and on top of it the room the two braces take, for the + reason the sister aggregate carries the same allowance: the shortest + answer this function gives is the braces with nothing between them, + and a cap smaller than that is answered over the cap with nothing + left to cut. A width worked out from the cap alone would then be a + column too narrow to hold the answer, which is cut into it without a + word said. + */ max_length= (uint32) MY_MIN((ulonglong) thd->gconcat_max_len() / collation.collation->mbminlen - * collation.collation->mbmaxlen, UINT_MAX32); + * collation.collation->mbmaxlen + + 2 * brace_length(), UINT_MAX32); if (check_sum_func(thd, ref)) @@ -7765,7 +7786,7 @@ bool Item_func_json_objectagg::add() is why the width is asked for rather than assumed. */ { - uint32 close_len= collation.collation->mbminlen; + uint32 close_len= brace_length(); DBUG_ASSERT(m_thd); if (result.length() + close_len > m_thd->gconcat_max_len()) diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 0021b7c708139..941d2722152f5 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -951,6 +951,14 @@ class Item_func_json_arrayagg : public Item_func_group_concat const uchar *key, size_t offset) override; void cut_max_length(String *result, uint old_length, uint max_length) const override; + /* + The two brackets, which val_str() writes round the group once it is + asked for. They go in through String::append(), which writes them + in the set of the result, so each of them is as wide as the + narrowest character that set has. + */ + uint32 reserved_result_length() const override + { return 2 * collation.collation->mbminlen; } /* A row of this group could not be written out at all, the buffer having failed to grow. Neither a row whose value does not parse as @@ -1100,6 +1108,13 @@ class Item_func_json_objectagg : public Item_sum */ THD *m_thd; Json_result_marks m_marks; + /* + How wide a brace is. Both of them go in through String::append(), + which writes them in the set of the result, so each is as wide as + the narrowest character that set has. + */ + uint32 brace_length() const + { return collation.collation->mbminlen; } public: /* The opening brace is not written here. This runs while the diff --git a/sql/item_sum.cc b/sql/item_sum.cc index c5d4e1902a656..66da849474cd2 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -3830,7 +3830,20 @@ int dump_leaf_key(void* key_arg, element_count count __attribute__((unused)), { Item_func_group_concat *item= (Item_func_group_concat *) item_arg; TABLE *table= item->table; - uint max_length= table->in_use->gconcat_max_len(); + /* + Room is kept back for whatever the caller still has to write round + this, so that the limit is reached where the answer reaches it + rather than two bytes further on. GROUP_CONCAT keeps back nothing + and is not moved by any of this. + + A limit too small to hold what the caller writes leaves nothing for + the rows, and the answer is then that wrapper by itself - as short + as the function goes, and still over the limit. Nothing can be done + about that here; the width the item declares carries room for it. + */ + uint reserved= item->reserved_result_length(); + uint gconcat_max_len= table->in_use->gconcat_max_len(); + uint max_length= gconcat_max_len > reserved ? gconcat_max_len - reserved : 0; String tmp((char *)table->record[1], table->s->reclength, default_charset_info); String tmp2; diff --git a/sql/item_sum.h b/sql/item_sum.h index 354861ab6c5db..4c233d3e6ab74 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -2083,6 +2083,14 @@ class Item_func_group_concat : public Item_sum { return f->val_str(tmp, key + offset); } virtual void cut_max_length(String *result, uint old_length, uint max_length) const; + /* + How much of the length limit is spoken for by something that is not + in the buffer yet. What this accumulates is the whole of what + GROUP_CONCAT returns, so nothing; a subclass that writes anything + round the group once it is asked for is handing those bytes back as + well, and the limit is on the bytes returned. + */ + virtual uint32 reserved_result_length() const { return 0; } bool uses_non_standard_aggregator_for_distinct() const override { return distinct; } From b38e78fe748e477eabef0e018183e1e6bd7896d3 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Wed, 12 Aug 2026 04:28:27 -0400 Subject: [PATCH 26/30] Run the JSON tests under the other test-runner protocols too The runner can put every statement through a prepared statement, a cursor or a wrapping view, and three of the things these tests do come out differently when it does. 1. **A prepared SELECT is run a second time**, so that the two answers can be checked against each other. Anything counted rather than read counts twice over - the scan counter, a stored function's call count - and a statement that writes to the variable it is reading is asked the second time about a document it has already replaced. `--disable_ps2_protocol` over those sections. 2. **A check constraint is looked at when the statement is prepared**, so the warnings one draws are prepare warnings, which the runner does not show unless it is asked for them. `--enable_prepare_warnings` over the tables that draw one. 3. **A wrapping view is built on a connection of its own**, so it carries neither the session's `sql_mode` nor its connection collation; it keeps its body as printed text, so a boolean literal comes back as the number it equals; and its columns, like a cursor's temporary table, are as wide and as nullable as the expression says rather than as the answer turned out. `--disable_view_protocol` and `--disable_cursor_protocol` over the sections that read result metadata, that set a character set or an `sql_mode`, and over the few answers a declaration does not fit. No recorded result moves: every directive here is inert in the protocol the results were recorded in. --- mysql-test/main/func_concat.test | 4 ++++ mysql-test/main/func_json_agg_cut.test | 6 +++++ mysql-test/main/func_json_agg_limits.test | 7 ++++++ mysql-test/main/func_json_aliasing.test | 8 +++++++ mysql-test/main/func_json_charset.test | 7 ++++++ .../main/func_json_check_scan_count.test | 10 ++++++++ mysql-test/main/func_json_check_skip.test | 7 ++++++ mysql-test/main/func_json_columns.test | 8 +++++++ mysql-test/main/func_json_embed.test | 5 ++++ mysql-test/main/func_json_format.test | 5 ++++ mysql-test/main/func_json_invalid.test | 7 ++++++ mysql-test/main/func_json_key_span.test | 5 ++++ mysql-test/main/func_json_keys_typed.test | 7 ++++++ .../main/func_json_keys_typed_scans.test | 7 ++++++ mysql-test/main/func_json_marks.test | 5 ++++ mysql-test/main/func_json_punct_charset.test | 9 ++++++++ mysql-test/main/func_json_result_side.test | 5 ++++ mysql-test/main/func_json_scan_count.test | 7 ++++++ mysql-test/main/func_json_search_typed.test | 5 ++++ mysql-test/main/func_json_tmp_scan_count.test | 7 ++++++ .../main/func_json_valid_constraint.test | 19 +++++++++++++++ mysql-test/main/func_json_value_reserve.test | 23 +++++++++++++++++++ mysql-test/main/mysql-metadata.test | 4 ++++ mysql-test/main/type_json.test | 4 ++++ 24 files changed, 181 insertions(+) diff --git a/mysql-test/main/func_concat.test b/mysql-test/main/func_concat.test index f9d8367b404e6..51caf15644f65 100644 --- a/mysql-test/main/func_concat.test +++ b/mysql-test/main/func_concat.test @@ -316,6 +316,9 @@ SELECT CONCAT(JSON_TYPE(j), JSON_TYPE(j)) AS c FROM t1; DROP TABLE t1; --echo # The || operator in sql_mode=ORACLE concatenates the same way +# A wrapping view is created on a connection of its own, which does not +# carry this session's sql_mode, so || would be read as an OR there. +--disable_view_protocol SET @save_sql_mode=@@sql_mode; SET sql_mode=ORACLE; @@ -323,5 +326,6 @@ SELECT JSON_TYPE('{"a":1}') || 'X'; SELECT GET_FORMAT(DATE, 'USA') || 'X'; SELECT 'P' || JSON_TYPE('{"a":1}'); SET sql_mode=@save_sql_mode; +--enable_view_protocol --echo # End of 10.11 tests diff --git a/mysql-test/main/func_json_agg_cut.test b/mysql-test/main/func_json_agg_cut.test index 274c707f62468..a83ed663879c3 100644 --- a/mysql-test/main/func_json_agg_cut.test +++ b/mysql-test/main/func_json_agg_cut.test @@ -189,6 +189,11 @@ BEGIN END// DELIMITER ;// +# The counter below records how often the function was called rather than +# what the query answered, so a statement run a second time to check that it +# repeats itself would count twice. +--disable_ps2_protocol + --echo # the array, which reads every row of the group already SET @calls= 0; SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; @@ -238,6 +243,7 @@ DROP TABLE t6; DROP FUNCTION seen_once; DROP TABLE t5; +--enable_ps2_protocol --echo # --echo # 12. The limit is on the bytes returned, and the brace that diff --git a/mysql-test/main/func_json_agg_limits.test b/mysql-test/main/func_json_agg_limits.test index da5a58ea01e23..61331336691c5 100644 --- a/mysql-test/main/func_json_agg_limits.test +++ b/mysql-test/main/func_json_agg_limits.test @@ -182,11 +182,18 @@ SELECT JSON_OBJECTAGG(k, 1) AS v --echo # bytes are converted on their way to the client. --echo # +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 2; SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; SELECT JSON_ARRAY(1) AS v; SELECT JSON_SET('{}', '$.a', 1) AS v; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol DROP TABLE t1; diff --git a/mysql-test/main/func_json_aliasing.test b/mysql-test/main/func_json_aliasing.test index 87a6aea3930ea..63d99754b4491 100644 --- a/mysql-test/main/func_json_aliasing.test +++ b/mysql-test/main/func_json_aliasing.test @@ -82,6 +82,10 @@ SELECT LENGTH(g) AS mgrown, JSON_VALID(g) AS parses, --echo # 4. A user variable holding the document, written to by the --echo # expression that works out the value --echo # +# Each statement below overwrites the variable it reads, so running it a +# second time to check that it repeats itself would be asking it about a +# different document. +--disable_ps2_protocol SET @d= '{"a":1111111111}'; SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 30))) AS uvar; SET @d= '{"a":1111111111}'; @@ -94,6 +98,7 @@ SET @d= '[1111111111]'; SELECT JSON_ARRAY_APPEND(@d, '$', (@d := REPEAT('z', 30))) AS uvar_app; SET @d= '{"a":1111111111}'; SELECT JSON_MERGE(@d, (@d := '{"b":2}')) AS uvar_mrg; +--enable_ps2_protocol --echo # --echo # 5. A routine's own variable holding the document, with the @@ -166,9 +171,12 @@ SELECT LENGTH(g) AS typed_big, JSON_VALID(g) AS parses, --echo # And where the document being edited is a user variable that --echo # the value expression writes to, with the value a document. --echo # +# This one overwrites the variable it reads as well. +--disable_ps2_protocol SET @d= '{"a":1111111111}'; SELECT JSON_INSERT(@d, '$.b', JSON_ARRAY((@d := REPEAT('z', 30)), 2)) AS typed_uvar; +--enable_ps2_protocol DROP TABLE t1; diff --git a/mysql-test/main/func_json_charset.test b/mysql-test/main/func_json_charset.test index 99ffd7464432d..321addb6f7540 100644 --- a/mysql-test/main/func_json_charset.test +++ b/mysql-test/main/func_json_charset.test @@ -223,6 +223,11 @@ DROP TABLE tp; --echo # bytes are converted on the way out. --echo # +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_ARRAY(1) AS v; SELECT JSON_OBJECT('a',1) AS v; @@ -235,6 +240,8 @@ SELECT JSON_COMPACT('{}') AS v; SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol --echo # --echo # 8. A result delivered in a character set other than the one it was diff --git a/mysql-test/main/func_json_check_scan_count.test b/mysql-test/main/func_json_check_scan_count.test index 0616abe425f7b..6184f3ef23f06 100644 --- a/mysql-test/main/func_json_check_scan_count.test +++ b/mysql-test/main/func_json_check_scan_count.test @@ -25,6 +25,13 @@ SET NAMES utf8mb4; +# Everything here is counted rather than read, so a statement that is run a +# second time to check that it repeats itself would count twice. +--disable_ps2_protocol +# A check constraint is looked at while the column definition is validated, +# which for a prepared CREATE TABLE is prepare time. +--enable_prepare_warnings + CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); CREATE TABLE t2 (id INT PRIMARY KEY, j VARCHAR(64)); INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); @@ -303,3 +310,6 @@ UPDATE t1 FOR PORTION OF p FROM '2002-01-01' TO '2003-01-01' SET id = 2; SHOW STATUS LIKE 'Json_scans'; SELECT id, s, e, j FROM t1 ORDER BY s; DROP TABLE t1; + +--disable_prepare_warnings +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_check_skip.test b/mysql-test/main/func_json_check_skip.test index 3782dd970aa35..7e03cab350433 100644 --- a/mysql-test/main/func_json_check_skip.test +++ b/mysql-test/main/func_json_check_skip.test @@ -15,6 +15,11 @@ SET NAMES utf8mb4; +# Some of the statements below are looked at once when they are prepared and +# warn there rather than when they run, so the warnings only reach the client +# when prepare warnings are asked for. +--enable_prepare_warnings + --echo # --echo # 1. A document that came from a JSON function goes in and stays put. --echo # @@ -299,3 +304,5 @@ DELETE FROM t2 WHERE id = 2; SELECT id, j FROM t1 ORDER BY id; --remove_file $DATA DROP TABLE t1, t2; + +--disable_prepare_warnings diff --git a/mysql-test/main/func_json_columns.test b/mysql-test/main/func_json_columns.test index 9b8e1b54537b4..6fe19693981f8 100644 --- a/mysql-test/main/func_json_columns.test +++ b/mysql-test/main/func_json_columns.test @@ -13,6 +13,12 @@ SET NAMES utf8mb4; +# A check constraint is looked at while the column definition is validated, +# which for a prepared CREATE TABLE is prepare time, so the warnings the +# constraints below draw only reach the client when prepare warnings are +# asked for. +--enable_prepare_warnings + --echo # --echo # 1. What declaring a column JSON actually creates. --echo # @@ -427,3 +433,5 @@ SELECT g, JSON_ARRAY(JSON_SET(j, '$.z', 1)) AS v, SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 ORDER BY JSON_EXTRACT(j, '$.a'), v; DROP TABLE t1, t2, t3, t4, t5, t10; + +--disable_prepare_warnings diff --git a/mysql-test/main/func_json_embed.test b/mysql-test/main/func_json_embed.test index 36a3d9728302f..9e1eb3b2f15cf 100644 --- a/mysql-test/main/func_json_embed.test +++ b/mysql-test/main/func_json_embed.test @@ -206,9 +206,14 @@ INSERT INTO t5 VALUES (2, '{"a":1,"b":[1,2]}'); INSERT INTO t5 VALUES (3, '[ 1 ,2, 3 ]'); --echo # with nothing spliced, and with a document spliced in +# A view keeps its body as printed text, and a boolean literal prints as the +# number it equals, so read through one this array would hold a 1 rather +# than a true. +--disable_view_protocol SELECT JSON_ARRAY(1, 'a', TRUE, NULL) AS built, JSON_ARRAY(1, 'a', TRUE, NULL) = JSON_LOOSE(JSON_ARRAY(1, 'a', TRUE, NULL)) AS already_loose; +--enable_view_protocol SELECT JSON_OBJECT('a', 1, 'b', 'x') AS built, JSON_OBJECT('a', 1, 'b', 'x') = JSON_LOOSE(JSON_OBJECT('a', 1, 'b', 'x')) AS already_loose; diff --git a/mysql-test/main/func_json_format.test b/mysql-test/main/func_json_format.test index b8b3fb5562ddc..ea21750b0959a 100644 --- a/mysql-test/main/func_json_format.test +++ b/mysql-test/main/func_json_format.test @@ -99,7 +99,12 @@ SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}', '[9]') AS v; --echo # --echo # plain strings are quoted, JSON-typed arguments are embedded raw +# A view keeps its body as printed text, and a boolean literal prints as the +# number it equals, so read through one this array would hold a 1 rather +# than a true. +--disable_view_protocol SELECT JSON_ARRAY(1, 'str', NULL, TRUE) AS v; +--enable_view_protocol SELECT JSON_ARRAY() AS v, HEX(JSON_ARRAY()) AS h; SELECT JSON_OBJECT() AS v, HEX(JSON_OBJECT()) AS h; SELECT JSON_OBJECT('a', 1, 'b', 'str') AS v; diff --git a/mysql-test/main/func_json_invalid.test b/mysql-test/main/func_json_invalid.test index 1c224a1a89ee0..b665870333bf7 100644 --- a/mysql-test/main/func_json_invalid.test +++ b/mysql-test/main/func_json_invalid.test @@ -72,7 +72,14 @@ SELECT JSON_QUERY('{"a":1,', '$.a') AS v; SELECT JSON_VALUE('{"a":1,', '$.a') AS v; SELECT JSON_KEYS('{"a":1,') AS v; SELECT JSON_LENGTH('{"a":1,') AS v; +# JSON_DEPTH says it can never answer NULL, so a view column or a cursor's +# temporary table built from it cannot hold one and the NULL below would +# arrive as a zero. +--disable_cursor_protocol +--disable_view_protocol SELECT JSON_DEPTH('{"a":1,') AS v; +--enable_view_protocol +--enable_cursor_protocol SELECT JSON_TYPE('{"a":1,') AS v; SELECT JSON_EXISTS('{"a":1,', '$.a') AS v; SELECT JSON_CONTAINS('{"a":1,', '1', '$.a') AS v; diff --git a/mysql-test/main/func_json_key_span.test b/mysql-test/main/func_json_key_span.test index 9e9c64446f675..7f9757dc61422 100644 --- a/mysql-test/main/func_json_key_span.test +++ b/mysql-test/main/func_json_key_span.test @@ -51,6 +51,10 @@ SELECT JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1) AS backslash_is_valid; --echo # than one byte --echo # SET @old_coll= @@collation_connection; +# A wrapping view is created on a connection of its own, which does not +# carry the connection collation set below, so the answers would come back +# encoded in the default set instead of the wide one being tested. +--disable_view_protocol --echo # All four of the keys above run again here, and the last two --echo # are the reason the section exists: a key that closes its own @@ -95,6 +99,7 @@ SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) AS utf32_backslash_is_valid; SET @@collation_connection= @old_coll; +--enable_view_protocol --echo # --echo # 5. Removing a key reads the path the same way diff --git a/mysql-test/main/func_json_keys_typed.test b/mysql-test/main/func_json_keys_typed.test index 1a2363903db1a..4d1c9034e02e5 100644 --- a/mysql-test/main/func_json_keys_typed.test +++ b/mysql-test/main/func_json_keys_typed.test @@ -136,11 +136,18 @@ DROP TABLE t1; --echo # 10. The result metadata, which is what decides all of the above. --echo # +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_KEYS('{"a":1,"b":2}') AS v; SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; SELECT JSON_QUOTE('a') AS v; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol --echo # --echo # 11. JSON_QUOTE is documented to return a string and still does. diff --git a/mysql-test/main/func_json_keys_typed_scans.test b/mysql-test/main/func_json_keys_typed_scans.test index 904c09fc04426..ceb282a6c6603 100644 --- a/mysql-test/main/func_json_keys_typed_scans.test +++ b/mysql-test/main/func_json_keys_typed_scans.test @@ -20,6 +20,11 @@ SET NAMES utf8mb4; +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol + CREATE TABLE ts (j TEXT) CHARACTER SET sjis; CREATE TABLE tu (j TEXT) CHARACTER SET utf8mb4; INSERT INTO ts VALUES ('{"a":1,"b":2}'); @@ -35,3 +40,5 @@ SELECT JSON_ARRAY(JSON_KEYS(j)) FROM tu; --enable_result_log SHOW STATUS LIKE 'Json_scans'; DROP TABLE ts, tu; + +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_marks.test b/mysql-test/main/func_json_marks.test index 7d972737832df..a49753fbdeb20 100644 --- a/mysql-test/main/func_json_marks.test +++ b/mysql-test/main/func_json_marks.test @@ -20,7 +20,12 @@ --echo # SELECT JSON_ARRAY() AS empty_array, JSON_OBJECT() AS empty_object; +# A view keeps its body as printed text, and a boolean literal prints as the +# number it equals, so read through one this array would hold a 1 rather +# than a true. +--disable_view_protocol SELECT JSON_ARRAY(1, 'two', NULL, TRUE, 3.5) AS scalars; +--enable_view_protocol SELECT JSON_OBJECT('a', 1, 'b', 'two', 'c', NULL) AS pairs; SELECT JSON_ARRAY(JSON_ARRAY(1, 2), JSON_OBJECT('a', 1)) AS nested; SELECT JSON_OBJECT('k', JSON_ARRAY(1, 2)) AS nested_in_object; diff --git a/mysql-test/main/func_json_punct_charset.test b/mysql-test/main/func_json_punct_charset.test index 4b1114c614aba..5fdae3c24aca2 100644 --- a/mysql-test/main/func_json_punct_charset.test +++ b/mysql-test/main/func_json_punct_charset.test @@ -16,6 +16,12 @@ # back must not change for that. # +# A view column and a cursor's temporary table are both as wide as the +# expression says it is, and one of the answers below is wider than that, +# so materialising it would cut it. +--disable_cursor_protocol +--disable_view_protocol + --echo # A scalar converts into swe7 and a container does not. SELECT JSON_VALID(CONVERT('1' USING swe7)) AS scalar_reads; SELECT HEX(CONVERT('[1,2]' USING swe7)) AS container_bytes; @@ -68,3 +74,6 @@ SELECT HEX(JSON_OBJECT(CONVERT('a' USING swe7), CONVERT('1' USING swe7))) AS obj SELECT HEX(JSON_QUERY(CONVERT('1' USING swe7), '$')) AS qry; SELECT HEX(JSON_NORMALIZE(CONVERT('1' USING swe7))) AS nrm; SELECT HEX(JSON_COMPACT(CONVERT('1' USING swe7))) AS cmp; + +--enable_view_protocol +--enable_cursor_protocol diff --git a/mysql-test/main/func_json_result_side.test b/mysql-test/main/func_json_result_side.test index 97fe00fd9c832..c03e02b60fe78 100644 --- a/mysql-test/main/func_json_result_side.test +++ b/mysql-test/main/func_json_result_side.test @@ -110,6 +110,10 @@ SELECT vj AS a FROM v1 GROUP BY a --echo # and puts it inside a new one; with the column's answer taken --echo # away the same query reads that document again. --echo # +# What is counted below is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol SET optimizer_switch='derived_merge=off'; FLUSH STATUS; --disable_result_log @@ -125,6 +129,7 @@ SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY SHOW STATUS LIKE 'Json_scans'; SET SESSION debug_dbug = DEFAULT; SET optimizer_switch=DEFAULT; +--enable_ps2_protocol DROP VIEW v1; DROP TABLE t1; diff --git a/mysql-test/main/func_json_scan_count.test b/mysql-test/main/func_json_scan_count.test index 1d7d15afe9c97..14ae73c41cadf 100644 --- a/mysql-test/main/func_json_scan_count.test +++ b/mysql-test/main/func_json_scan_count.test @@ -19,6 +19,11 @@ # no query is responsible for. # +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol + CREATE TABLE t1 (id INT, js VARCHAR(64)); INSERT INTO t1 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); @@ -612,3 +617,5 @@ SELECT JSON_VALID(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary_va --echo # And the same going the other way, which was always refused --echo # SELECT HEX(CONVERT(JSON_ARRAY('ab') USING binary)) AS to_binary; + +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_search_typed.test b/mysql-test/main/func_json_search_typed.test index 7ceebc6b927a3..95a2fb596f21f 100644 --- a/mysql-test/main/func_json_search_typed.test +++ b/mysql-test/main/func_json_search_typed.test @@ -79,6 +79,10 @@ DROP TABLE t3; --echo # the server built is taken as it stands; the same query with --echo # the column's answer taken away reads it again. --echo # +# What is counted below is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol SET optimizer_switch='derived_merge=off'; FLUSH STATUS; --disable_result_log @@ -96,5 +100,6 @@ SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d SHOW STATUS LIKE 'Json_scans'; SET SESSION debug_dbug = DEFAULT; SET optimizer_switch=DEFAULT; +--enable_ps2_protocol DROP TABLE t1; diff --git a/mysql-test/main/func_json_tmp_scan_count.test b/mysql-test/main/func_json_tmp_scan_count.test index e58d571d70bfc..6555c6f527ea3 100644 --- a/mysql-test/main/func_json_tmp_scan_count.test +++ b/mysql-test/main/func_json_tmp_scan_count.test @@ -16,6 +16,11 @@ --echo # SET NAMES utf8mb4; + +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol SET optimizer_switch='derived_merge=off'; CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; @@ -587,3 +592,5 @@ DROP TABLE t39, t39b; DROP TABLE t1; DROP TABLE t13, t13m; + +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_valid_constraint.test b/mysql-test/main/func_json_valid_constraint.test index ac076cb71f17c..3fce3704902b2 100644 --- a/mysql-test/main/func_json_valid_constraint.test +++ b/mysql-test/main/func_json_valid_constraint.test @@ -8,6 +8,11 @@ SET NAMES utf8mb4; +# The constraint is looked at while the column definition is validated, +# which for a prepared CREATE TABLE is prepare time, so the warnings below +# only reach the client when prepare warnings are asked for. +--enable_prepare_warnings + --echo # --echo # A constraint reading another column --echo # @@ -22,9 +27,13 @@ CREATE TABLE t1 ( SHOW CREATE TABLE t1; --echo # what the server tells a client the columns are +# Read through a view the columns belong to the view, so the names and the +# flags reported here would be the wrapper's rather than the table's. +--disable_view_protocol --enable_metadata SELECT a, b, d, e FROM t1; --disable_metadata +--enable_view_protocol SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1' ORDER BY COLUMN_NAME; @@ -71,9 +80,11 @@ DROP TABLE t2; --echo # CREATE TABLE t3 (a VARCHAR(100) CHECK (JSON_VALID('{}'))); +--disable_view_protocol --enable_metadata SELECT a FROM t3; --disable_metadata +--enable_view_protocol INSERT INTO t3 VALUES ('not json at all'); SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t3; DROP TABLE t3; @@ -98,9 +109,11 @@ CREATE TABLE t5 ( b VARCHAR(100), CONSTRAINT c CHECK (JSON_VALID(b)) ); +--disable_view_protocol --enable_metadata SELECT a, b FROM t5; --disable_metadata +--enable_view_protocol DROP TABLE t5; --echo # @@ -123,10 +136,12 @@ CREATE TABLE t8 (d TEXT CHECK (JSON_VALID(d))); --echo # the protocol tells the client what the warnings told the writer: --echo # none of the three is a JSON column, and a column whose check is --echo # the call alone still is +--disable_view_protocol --enable_metadata SELECT a, b, c, d FROM t7; SELECT d FROM t8; --disable_metadata +--enable_view_protocol --echo # and what each does with a document says the same: quoted, where a --echo # column whose check is the call alone is spliced @@ -147,9 +162,11 @@ DROP TABLE t7, t8; CREATE TABLE t6 (j JSON, s VARCHAR(100) CHECK (JSON_VALID(s))); INSERT INTO t6 VALUES ('{"x":1}', '{"y":2}'), ('{"x":3}', '{"y":4}'); +--disable_view_protocol --enable_metadata SELECT j, s FROM t6 GROUP BY j, s; --disable_metadata +--enable_view_protocol SELECT JSON_SET('{}', '$.k', j) AS from_j, JSON_SET('{}', '$.k', s) AS from_s @@ -214,3 +231,5 @@ SELECT JSON_SET('{}','$.k',j) AS direct_nullable FROM t7 ORDER BY 1; SELECT JSON_SET('{}','$.k',j) AS direct_notnull FROM t8 ORDER BY 1; DROP TABLE lg, t7, t8; + +--disable_prepare_warnings diff --git a/mysql-test/main/func_json_value_reserve.test b/mysql-test/main/func_json_value_reserve.test index c9e903b04cd60..ce7a19536929c 100644 --- a/mysql-test/main/func_json_value_reserve.test +++ b/mysql-test/main/func_json_value_reserve.test @@ -40,6 +40,11 @@ INSERT INTO t3 VALUES (CONCAT('{"x":"', REPEAT('a',190), '"}')), --echo # reach this number, so it says what the arithmetic came to and --echo # nothing else. The Length column is the one to read. --echo # +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_SET(v, '$.p', 'z') AS one_pair FROM t1; SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS two_pairs FROM t1; @@ -50,6 +55,8 @@ SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS escaped_value FROM t2; SELECT JSON_SET(v, '$.p', s) AS wide_escape_value FROM t2; SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS document_fills_column FROM t3; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol --echo # --echo # The same numbers again, read off the column a result is put in. @@ -207,6 +214,8 @@ DROP TABLE t4; --echo # All four work it out the same way and all four are given a value --echo # whose characters have to be escaped. --echo # +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_ARRAY(s) AS array_value FROM t2; SELECT JSON_OBJECT('k', s) AS object_value FROM t2; @@ -214,6 +223,8 @@ SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS appended FROM t2; SELECT JSON_ARRAY_INSERT(JSON_SET(v, '$.x', JSON_ARRAY(1)), '$.x[0]', s) AS inserted FROM t2; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol SELECT LENGTH(JSON_ARRAY(s)) AS produced FROM t2; SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid @@ -246,9 +257,13 @@ SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid CREATE TABLE t7 (s VARCHAR(45)); INSERT INTO t7 VALUES (REPEAT('a',45)), (REPEAT('b',45)); +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_QUOTE(s) AS quoted FROM t7; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol CREATE TABLE d9 AS SELECT JSON_ARRAY(s) AS r FROM t7; SHOW CREATE TABLE d9; @@ -260,9 +275,13 @@ DROP TABLE t7; --echo # is written out again with the spacing, so the room for it is --echo # asked for twice over and no quotes are asked for round it. --echo # +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_ARRAY(JSON_QUERY(v, '$')) AS document_value FROM t2; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol --echo # --echo # A function that takes something out and writes what is left out @@ -275,9 +294,13 @@ INSERT INTO t5 VALUES ('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), ('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); +--disable_cursor_protocol +--disable_view_protocol --enable_metadata SELECT JSON_REMOVE(v, '$[0]') AS removed FROM t5; --disable_metadata +--enable_view_protocol +--enable_cursor_protocol SELECT LENGTH(v) AS document FROM t5; SELECT LENGTH(JSON_REMOVE(v, '$[0]')) AS produced FROM t5; diff --git a/mysql-test/main/mysql-metadata.test b/mysql-test/main/mysql-metadata.test index bab44496f7880..8978ce49d6832 100644 --- a/mysql-test/main/mysql-metadata.test +++ b/mysql-test/main/mysql-metadata.test @@ -6,12 +6,16 @@ --echo # SET NAMES utf8; +# The check constraints are looked at while the column definitions are +# validated, which for a prepared CREATE TABLE is prepare time. +--enable_prepare_warnings CREATE TABLE t1 ( js0 JSON, js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +--disable_prepare_warnings --replace_regex /0 rows in set [(].*[)]/0 rows in set (TIME)/ --exec $MYSQL -vvv --column-type-info --database=test -e "SELECT * FROM t1;" diff --git a/mysql-test/main/type_json.test b/mysql-test/main/type_json.test index 8effe78f803d4..7af47326d1f2a 100644 --- a/mysql-test/main/type_json.test +++ b/mysql-test/main/type_json.test @@ -74,12 +74,16 @@ select cast('{a:1}' as json); --echo # SET NAMES utf8; +# The check constraints are looked at while the column definitions are +# validated, which for a prepared CREATE TABLE is prepare time. +--enable_prepare_warnings CREATE TABLE t1 ( js0 JSON, js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +--disable_prepare_warnings --disable_view_protocol --disable_ps_protocol --enable_metadata From c93e552a0992fd68c56f90e262e591ac4cb9c24c Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Fri, 14 Aug 2026 17:11:34 -0400 Subject: [PATCH 27/30] MDEV-40700 A JSON constructor returns a document or NULL `JSON_ARRAY`, `JSON_OBJECT`, `JSON_ARRAYAGG` and `JSON_OBJECTAGG` build their result by concatenation and never parse it. A value typed as JSON is copied in verbatim, including a value that does not parse, so the result was not a document. `JSON_ARRAYAGG` over a column holding `{"a":1,` returned `[{"a": 1},{"a":1,,,{"a":1}]`. The server returned those bytes and raised only a note. These four now return NULL instead. Every value is parsed on the way in, and `Json_splice_marks::is_valid` already tracks whether all of them parsed, so the fix is to check that flag before returning. **Character sets.** The finished result is also parsed when it was written in a character set that is not ASCII-compatible. Such a set may still write the brackets and quotes as themselves - `sjis` moves only the backslash and the tilde - so the character set alone does not answer the question and the bytes have to be parsed. A compatible set is not parsed, so the extra parse is confined to the sets that can get it wrong. **Why this is worth doing.** Returning a document or NULL is exactly what `is_valid_json_static()` claims, so the four can now claim it. That is the only attestation that reaches an aggregate, which reads its elements back out of a temporary table by raw key bytes and cannot see a per-value mark. `JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js))` now parses nothing on the way out, `Json_scans` 2 -> 0. Against that, a result in a set that is not ASCII-compatible costs one parse that was not paid before. **Two questions for a reviewer.** 1. For these four the diagnostic about a bad value is now a warning rather than a note. It is the only explanation the statement gets for the NULL, and under strict mode a warning becomes an error, so the statement stops at the value. Two tests changed accordingly and now expect an error rather than a row: an `INSERT` of a construction 32 levels deep, and an `INSERT` whose trigger builds a value from an unparseable column. This is deliberate. The alternative is a statement that inserts nothing and reports no error. A value rejected for containing a character that no document can hold still raises a note. That is released behaviour which this bug does not touch, so it was left alone. 2. Once the result is decided, the remaining arguments are not evaluated. A constructor stops at the first value it rejects, and `JSON_OBJECTAGG` evaluates neither argument for the rows after the group is cut at `group_concat_max_len`. Nothing documents how many times an aggregate evaluates its arguments, and the server already skips argument evaluation elsewhere once the result is known. It does mean that an argument with a side effect - a stored function that writes, an assignment - runs for the rows before the cut and not for the rows after it. Say so if that is not acceptable. --- mysql-test/main/func_json_agg_charset.result | 24 +- mysql-test/main/func_json_agg_cut.result | 33 +- mysql-test/main/func_json_agg_cut.test | 27 +- mysql-test/main/func_json_charset.result | 15 +- mysql-test/main/func_json_columns.result | 175 ++++---- mysql-test/main/func_json_columns.test | 16 +- mysql-test/main/func_json_depth.result | 17 +- mysql-test/main/func_json_embed.result | 382 +++++++++++------- mysql-test/main/func_json_embed.test | 110 ++++- .../main/func_json_keys_typed_scans.result | 12 +- .../main/func_json_keys_typed_scans.test | 10 +- mysql-test/main/func_json_marks.result | 41 +- .../main/func_json_punct_charset.result | 8 +- mysql-test/main/func_json_scan_count.result | 37 ++ mysql-test/main/func_json_scan_count.test | 41 ++ .../main/func_json_search_wide_charset.result | 10 +- mysql-test/main/func_json_sp.result | 9 +- mysql-test/main/func_json_sp.test | 6 + mysql-test/main/func_json_splice_depth.result | 13 +- .../main/func_json_tmp_scan_count.result | 38 +- mysql-test/main/func_json_tmp_scan_count.test | 30 +- mysql-test/main/func_json_tmp_trust.result | 64 ++- mysql-test/main/func_json_tmp_trust.test | 20 +- sql/item_jsonfunc.cc | 317 +++++++++++---- sql/item_jsonfunc.h | 72 +++- 25 files changed, 1016 insertions(+), 511 deletions(-) diff --git a/mysql-test/main/func_json_agg_charset.result b/mysql-test/main/func_json_agg_charset.result index 86e964b42db84..69b12dbf5aeb6 100644 --- a/mysql-test/main/func_json_agg_charset.result +++ b/mysql-test/main/func_json_agg_charset.result @@ -290,32 +290,36 @@ g h # SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS h FROM t1; h -5B226161222C226262222C226363222C226464225D +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 SELECT CHARSET(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS cs FROM t1; cs swe7 SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid FROM t1; valid -0 +NULL Warnings: -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 SELECT g, JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid FROM t1 GROUP BY g ORDER BY g; g valid -1 0 -2 0 +1 NULL +2 NULL Warnings: -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING swe7), CONVERT(a USING swe7))) AS h FROM t1; h -7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_objectagg' at position 1 SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING swe7), CONVERT(a USING swe7))) AS valid FROM t1; valid -0 +NULL Warnings: -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_objectagg' at position 1 DROP TABLE t1, t2; SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_agg_cut.result b/mysql-test/main/func_json_agg_cut.result index 966dc0276ba40..eee842a08956e 100644 --- a/mysql-test/main/func_json_agg_cut.result +++ b/mysql-test/main/func_json_agg_cut.result @@ -587,13 +587,22 @@ SET SESSION group_concat_max_len = DEFAULT; DROP TABLE t4; DROP TABLE t1; # -# 11. A cut group is still read to the end. An argument is an -# expression, and the rows past the cut work it out exactly as -# the rows before it do - only the writing stops. Each shape -# below is run twice over the same rows, once under a cap that -# cuts and once under one that does not, and counts what the -# argument did. The two counts are the same because the -# reading is. +# 11. A cut group is still read to the end, but takes nothing +# further, so the object evaluates neither of its arguments +# for the rows after the cut. Each shape below is run twice +# over the same rows, once under a cap that cuts and once +# under one that does not, and counts what the argument did. +# +# The array counts the same either way: it discovers the cut +# when the group is asked for, by which time every row has +# been read, so there is no row left to skip. The object +# knows while the rows are still coming, and its count falls +# to the rows up to the cut. +# +# Nothing documents how many times an aggregate evaluates its +# arguments, so an argument that does something besides +# produce a value does that thing for the rows before the cut +# and not for the rows after it. # CREATE TABLE t5 (k VARCHAR(20), v VARCHAR(20)); INSERT INTO t5 VALUES @@ -636,10 +645,10 @@ Warning 1260 Row 1 was cut by JSON_OBJECTAGG() SET SESSION group_concat_max_len = DEFAULT; SELECT @uncut AS uncut_calls, @calls AS cut_calls; uncut_calls cut_calls -4 4 +4 1 # the object, on the key. The object also asks the key whether it # is NULL, and asking that of a function is running it again, so -# the count here is two a row - under either cap. +# the count is two a row - for the rows it reaches. SET @calls= 0; SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; v @@ -655,9 +664,9 @@ Warning 1260 Row 1 was cut by JSON_OBJECTAGG() SET SESSION group_concat_max_len = DEFAULT; SELECT @uncut AS uncut_calls, @calls AS cut_calls; uncut_calls cut_calls -8 8 +8 2 # a row whose key is NULL makes no pair, so its value is not read -# for it - past the cut no differently from before it +# for it either, which is a second reason not to read one CREATE TABLE t6 (k VARCHAR(20), v VARCHAR(20)); INSERT INTO t6 VALUES ('kkkk', 'aaaa'), ('llll', 'bbbb'), (NULL, 'cccc'), ('nnnn', 'dddd'); @@ -676,7 +685,7 @@ Warning 1260 Row 1 was cut by JSON_OBJECTAGG() SET SESSION group_concat_max_len = DEFAULT; SELECT @uncut AS uncut_calls, @calls AS cut_calls; uncut_calls cut_calls -3 3 +3 1 DROP TABLE t6; DROP FUNCTION seen_once; DROP TABLE t5; diff --git a/mysql-test/main/func_json_agg_cut.test b/mysql-test/main/func_json_agg_cut.test index a83ed663879c3..b93533f603413 100644 --- a/mysql-test/main/func_json_agg_cut.test +++ b/mysql-test/main/func_json_agg_cut.test @@ -168,13 +168,22 @@ DROP TABLE t4; DROP TABLE t1; --echo # ---echo # 11. A cut group is still read to the end. An argument is an ---echo # expression, and the rows past the cut work it out exactly as ---echo # the rows before it do - only the writing stops. Each shape ---echo # below is run twice over the same rows, once under a cap that ---echo # cuts and once under one that does not, and counts what the ---echo # argument did. The two counts are the same because the ---echo # reading is. +--echo # 11. A cut group is still read to the end, but takes nothing +--echo # further, so the object evaluates neither of its arguments +--echo # for the rows after the cut. Each shape below is run twice +--echo # over the same rows, once under a cap that cuts and once +--echo # under one that does not, and counts what the argument did. +--echo # +--echo # The array counts the same either way: it discovers the cut +--echo # when the group is asked for, by which time every row has +--echo # been read, so there is no row left to skip. The object +--echo # knows while the rows are still coming, and its count falls +--echo # to the rows up to the cut. +--echo # +--echo # Nothing documents how many times an aggregate evaluates its +--echo # arguments, so an argument that does something besides +--echo # produce a value does that thing for the rows before the cut +--echo # and not for the rows after it. --echo # CREATE TABLE t5 (k VARCHAR(20), v VARCHAR(20)); @@ -216,7 +225,7 @@ SELECT @uncut AS uncut_calls, @calls AS cut_calls; --echo # the object, on the key. The object also asks the key whether it --echo # is NULL, and asking that of a function is running it again, so ---echo # the count here is two a row - under either cap. +--echo # the count is two a row - for the rows it reaches. SET @calls= 0; SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; SET @uncut= @calls; @@ -227,7 +236,7 @@ SET SESSION group_concat_max_len = DEFAULT; SELECT @uncut AS uncut_calls, @calls AS cut_calls; --echo # a row whose key is NULL makes no pair, so its value is not read ---echo # for it - past the cut no differently from before it +--echo # for it either, which is a second reason not to read one CREATE TABLE t6 (k VARCHAR(20), v VARCHAR(20)); INSERT INTO t6 VALUES ('kkkk', 'aaaa'), ('llll', 'bbbb'), (NULL, 'cccc'), ('nnnn', 'dddd'); diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result index acfc5e7fdebd4..572e98f29e900 100644 --- a/mysql-test/main/func_json_charset.result +++ b/mysql-test/main/func_json_charset.result @@ -144,15 +144,17 @@ swe7_to_utf8mb4 C3A4C3A5 SELECT HEX(JSON_ARRAY(_swe7 X'7B7D')) AS embedded; embedded -5B227B7D225D +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 SELECT HEX(JSON_QUOTE(_swe7 X'7B7D')) AS quoted; quoted 22C3A4C3A522 SELECT JSON_VALID(JSON_ARRAY(_swe7 X'7B7D')) AS still_valid; still_valid -0 +NULL Warnings: -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 # a document written in swe7 itself SELECT HEX(JSON_COMPACT(CONVERT('{"a":1}' USING swe7))) AS as_swe7; as_swe7 @@ -449,11 +451,10 @@ SET SESSION check_constraint_checks = ON; SELECT JSON_ARRAY(CONVERT(j USING latin1)) AS v, JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tp; v ok -[{"a":1,] 0 +NULL NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1)) AS v FROM tp; v NULL diff --git a/mysql-test/main/func_json_columns.result b/mysql-test/main/func_json_columns.result index ebdfb8e2019ab..07cd62dba7f5f 100644 --- a/mysql-test/main/func_json_columns.result +++ b/mysql-test/main/func_json_columns.result @@ -473,79 +473,71 @@ Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument # SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v -10 [{"a":1,] -11 [not json at all] -12 [{"a":1} trailing] -13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] -Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; id still_valid -10 0 -11 0 -12 0 -13 0 +10 NULL +11 NULL +12 NULL +13 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v -10 {"k": {"a":1,} -11 {"k": not json at all} -12 {"k": {"a":1} trailing} -13 {"k": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} +10 NULL +11 NULL +12 NULL +13 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; id still_valid -10 0 -11 0 -12 0 -13 0 +10 NULL +11 NULL +12 NULL +13 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 15 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 37 +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 SELECT id, JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; id v -10 [{"a":1,] -11 [not json at all] -12 [{"a":1} trailing] -13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +10 NULL +11 NULL +12 NULL +13 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 32 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 32 SELECT id, JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; id v -10 {"10":{"a":1,} -11 {"11":not json at all} -12 {"12":{"a":1} trailing} -13 {"13":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} +10 NULL +11 NULL +12 NULL +13 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' -Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 32 +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 32 # the same column read as plain text is quoted instead of embedded SELECT id, JSON_ARRAY(CONCAT(j, '')) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v @@ -564,16 +556,15 @@ id still_valid SELECT id, JSON_ARRAY(j) AS v FROM t3 ORDER BY id; id v 1 [{"a":1}] -10 [{"a":1,] +10 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t3 ORDER BY id; id still_valid 1 1 -10 0 +10 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' # 5a. The same column reached through a scalar subquery, where the # value is fetched once and held rather than read per row. CREATE TABLE ts (id INT, j JSON); @@ -589,23 +580,22 @@ ok 1 SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 2)) AS v_poisoned; v_poisoned -[{"a":1,] +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 2))) AS ok_poisoned; ok_poisoned -0 +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 1)) AS v; v {"k": {"a": 1}} SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; v_poisoned -{"k": {"a":1,} +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 1)) AS v; v {"d": {"a": 1}} @@ -644,20 +634,18 @@ SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok FROM v_merge ORDER BY id; id v ok 1 [{"a": 1, "b": [1, 2]}] 1 -2 [{"a":1,] 0 +2 NULL NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok FROM v_tmp ORDER BY id; id v ok 1 [{"a": 1, "b": [1, 2]}] 1 -2 [{"a":1,] 0 +2 NULL NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_merge ORDER BY id; id v 1 {"a": 1, "b": [1, 2], "z": 1} @@ -728,23 +716,27 @@ j {"a": 1, "b": 2} [1, 2] {"a": 1} -# a value the constructor built that the check does not accept +# a value the constructor will not build. A released server built +# something 32 deep, which is one too many to read back, and the +# check on the column was what caught it. The constructor now +# answers NULL, and the complaint that says why is a warning, which +# a statement inserting under strict mode makes an error of - so +# the statement stops at the value rather than at the column, and +# says what is wrong with the value. CREATE TABLE t7 (j JSON); INSERT INTO t7 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); -ERROR 23000: CONSTRAINT `t7.j` failed for `test`.`t7` +ERROR HY000: Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 SELECT COUNT(*) AS stored FROM t7; stored 0 -# the same value stored into a plain LONGTEXT column +# the same value into a plain LONGTEXT column, which has no check +# on it to reach: the constructor is where it stops either way CREATE TABLE t8 (j LONGTEXT); INSERT INTO t8 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); -Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 -SELECT JSON_VALID(j) AS v, LENGTH(j) AS len FROM t8; -v len -0 65 -Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +ERROR HY000: Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT COUNT(*) AS stored FROM t8; +stored +0 # a value that is truncated on the way into the column CREATE TABLE t9 (j VARCHAR(10)); SET @@sql_mode=''; @@ -904,16 +896,15 @@ Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' SELECT id, JSON_ARRAY(j) AS v FROM tmp1 ORDER BY id; id v 1 [{"a":1}] -2 [{"a":1,] +2 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM tmp1 ORDER BY id; id still_valid 1 1 -2 0 +2 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT id, JSON_SET(j, '$.z', 1) AS v FROM tmp1 ORDER BY id; id v 1 {"a": 1, "z": 1} diff --git a/mysql-test/main/func_json_columns.test b/mysql-test/main/func_json_columns.test index 6fe19693981f8..e1fa8eba4c6e0 100644 --- a/mysql-test/main/func_json_columns.test +++ b/mysql-test/main/func_json_columns.test @@ -270,15 +270,23 @@ INSERT INTO t6 SELECT JSON_SET('{"a":1}', '$.b', 2); INSERT INTO t6 SELECT JSON_ARRAY(1,2); INSERT INTO t6 SELECT JSON_OBJECT('a',1); SELECT j FROM t6; ---echo # a value the constructor built that the check does not accept +--echo # a value the constructor will not build. A released server built +--echo # something 32 deep, which is one too many to read back, and the +--echo # check on the column was what caught it. The constructor now +--echo # answers NULL, and the complaint that says why is a warning, which +--echo # a statement inserting under strict mode makes an error of - so +--echo # the statement stops at the value rather than at the column, and +--echo # says what is wrong with the value. CREATE TABLE t7 (j JSON); ---error ER_CONSTRAINT_FAILED +--error ER_JSON_DEPTH INSERT INTO t7 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); SELECT COUNT(*) AS stored FROM t7; ---echo # the same value stored into a plain LONGTEXT column +--echo # the same value into a plain LONGTEXT column, which has no check +--echo # on it to reach: the constructor is where it stops either way CREATE TABLE t8 (j LONGTEXT); +--error ER_JSON_DEPTH INSERT INTO t8 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); -SELECT JSON_VALID(j) AS v, LENGTH(j) AS len FROM t8; +SELECT COUNT(*) AS stored FROM t8; --echo # a value that is truncated on the way into the column CREATE TABLE t9 (j VARCHAR(10)); SET @@sql_mode=''; diff --git a/mysql-test/main/func_json_depth.result b/mysql-test/main/func_json_depth.result index 1bdfeda7a16a5..eff0354ddeef8 100644 --- a/mysql-test/main/func_json_depth.result +++ b/mysql-test/main/func_json_depth.result @@ -254,19 +254,17 @@ SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS d32; d32 NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 -Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT(@d31))) AS still_valid; still_valid -0 +NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 SELECT LENGTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS produced_bytes; produced_bytes -65 +NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 # the same through an object constructor SELECT JSON_DEPTH(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS d32; d32 @@ -277,10 +275,9 @@ still_valid # two levels of constructor SELECT JSON_VALID(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d30)))) AS still_valid; still_valid -0 +NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d29)))) AS d31; d31 32 diff --git a/mysql-test/main/func_json_embed.result b/mysql-test/main/func_json_embed.result index 3eb2be4e6ef48..03e1d7db78d82 100644 --- a/mysql-test/main/func_json_embed.result +++ b/mysql-test/main/func_json_embed.result @@ -18,9 +18,14 @@ # the caller gets. # # A value that does not parse is spliced all the same, byte for -# byte, and a note is raised saying which argument it was. Reading -# the result back is still where a caller finds out that what came -# out is not a document; the note is what says so at the time. +# byte. What the constructor has then built is not a document, so +# it returns NULL instead of it, and a warning names the argument +# that was responsible. A warning and not a note, because it is +# the only word the statement gets about why it was given NULL, +# and because a statement running under strict mode then stops at +# the value rather than finishing without one. A function that +# reads its whole answer back has a second complaint to make about +# the same value, and leaves this one a note. # # Non ASCII characters below are written as the bytes that encode # them, so this file needs no character set of its own: @@ -53,108 +58,100 @@ INSERT INTO t1 VALUES (12, '{"a":1} and then some', NULL); SET SESSION check_constraint_checks = ON; SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v -10 [{"a":1,] -11 [not json at all] -12 [{"a":1} and then some] +10 NULL +11 NULL +12 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document FROM t1 WHERE id >= 10 ORDER BY id; id built_a_document -10 0 -11 0 -12 0 -Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v -10 {"k": {"a":1,} -11 {"k": not json at all} -12 {"k": {"a":1} and then some} +10 NULL +11 NULL +12 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS built_a_document FROM t1 WHERE id >= 10 ORDER BY id; id built_a_document -10 0 -11 0 -12 0 -Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 15 +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 # more than one element, so the value is spliced in the middle of a # document rather than as the whole of it SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t1 WHERE id >= 10 ORDER BY id; id v -10 [1, {"a":1,, "z"] -11 [1, not json at all, "z"] -12 [1, {"a":1} and then some, "z"] +10 NULL +11 NULL +12 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 9 +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 9 SELECT id, JSON_OBJECT('before', 1, 'k', j, 'after', 2) AS v FROM t1 WHERE id >= 10 ORDER BY id; id v -10 {"before": 1, "k": {"a":1,, "after": 2} -11 {"before": 1, "k": not json at all, "after": 2} -12 {"before": 1, "k": {"a":1} and then some, "after": 2} +10 NULL +11 NULL +12 NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 4 to function 'json_object' -Note 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 1 -Note 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 9 +Warning 4037 Unexpected end of JSON text in argument 4 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 9 # the aggregates, which splice a value per row SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10; v -[{"a":1,,not json at all,{"a":1} and then some] +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS built_a_document FROM t1 WHERE id >= 10; built_a_document -0 +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 SELECT JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10; v -{"10":{"a":1,, "11":not json at all, "12":{"a":1} and then some} +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' -Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 SELECT JSON_VALID(JSON_OBJECTAGG(id, j)) AS built_a_document FROM t1 WHERE id >= 10; built_a_document -0 +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' -Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 14 +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 # a group mixing a good row with a bad one SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id = 1 OR id = 10; v -[{"a": 1},{"a":1,] +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' # the mutators splice the same way, but read back what they built SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; @@ -183,12 +180,14 @@ Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_app Note 4038 Syntax error in JSON text in argument 3 to function 'json_array_append' at position 9 Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 13 # -# 3. Bytes that are not a document but do not look wrong afterwards. +# 3. Bytes that are not a document and would not have looked wrong +# afterwards. # -# An empty value leaves nothing behind at all, so the result parses -# and reports one element fewer than it was given. The document -# itself gives no sign of this, which is what makes the note the -# only way to tell it apart from an array that really was short. +# An empty value leaves nothing behind at all, so what is built +# round it parses and reports one element fewer than it was given. +# A document like that gives no sign of what happened to it, and +# is exactly what is not returned: the row is refused, and the +# warning says which argument it was. # SET SESSION check_constraint_checks = OFF; INSERT INTO t1 VALUES (20, '', NULL); @@ -199,77 +198,68 @@ SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS looks_fine, JSON_LENGTH(JSON_ARRAY(j)) AS elements FROM t1 WHERE id >= 20 ORDER BY id; id v looks_fine elements -20 [] 1 0 -21 [ ] 1 0 -22 [// nothing here] 0 NULL -Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 -Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 2 +20 NULL NULL NULL +21 NULL NULL NULL +22 NULL NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 SELECT id, JSON_ARRAY(1, j, 2) AS v, JSON_VALID(JSON_ARRAY(1, j, 2)) AS looks_fine, JSON_LENGTH(JSON_ARRAY(1, j, 2)) AS elements FROM t1 WHERE id >= 20 ORDER BY id; id v looks_fine elements -20 [1, , 2] 0 NULL -21 [1, , 2] 0 NULL -22 [1, // nothing here, 2] 0 NULL -Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' -Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 5 -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_array' -Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 8 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 -Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 5 +20 NULL NULL NULL +21 NULL NULL NULL +22 NULL NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 SELECT id, JSON_OBJECT('k', j) AS v, JSON_VALID(JSON_OBJECT('k', j)) AS looks_fine FROM t1 WHERE id >= 20 ORDER BY id; id v looks_fine -20 {"k": } 0 -21 {"k": } 0 -22 {"k": // nothing here} 0 -Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 -Note 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +20 NULL NULL +21 NULL NULL +22 NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 SELECT JSON_ARRAYAGG(j) AS v, JSON_VALID(JSON_ARRAYAGG(j)) AS looks_fine FROM t1 WHERE id >= 20; v looks_fine -[, ,// nothing here] 0 +NULL NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 # # 4. Nesting that only goes over the limit once the value is wrapped. # # The scanner takes 31 nested structures. A document of exactly 31 -# is a document; put it inside one more and it is not. +# is a document; put it inside one more and it is not, so the +# constructor that wrapped it has nothing to return. A mutator +# reads its whole answer back and finds out there, as it always +# did. # CREATE TABLE t2 (id INT, j JSON); INSERT INTO t2 VALUES (31, CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))); @@ -282,26 +272,23 @@ SELECT id, JSON_VALID(JSON_ARRAY(j)) AS wrapped_once, LENGTH(JSON_ARRAY(j)) AS bytes FROM t2 ORDER BY id; id wrapped_once bytes 30 1 63 -31 0 65 +31 NULL NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS wrapped_once FROM t2 ORDER BY id; id wrapped_once 30 1 -31 0 +31 NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 63 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 37 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 63 # without a table in the way SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT( CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))))) AS wrapped_once; wrapped_once -0 +NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 # a mutator splices at the depth the path reaches, and reads back SELECT id, JSON_VALID(JSON_SET('{}', '$.k', j)) AS spliced FROM t2 ORDER BY id; id spliced @@ -508,13 +495,17 @@ id built already_loose # An aggregate writes the separator before it asks for the element, # so a row that yields nothing leaves a separator with nothing on # one side of it. There are two reasons a row can yield nothing -# and they are not answered alike. A character that cannot be -# written into a document at all is a property of the data: the -# element was dropped and the separator left standing before, and -# it still is, now with a note saying so. A buffer that would not -# grow is not a property of the data, and has already stopped the -# statement by the time it is noticed, so the group is refused -# whole rather than returned with a gap in it. +# and they are not reported alike. A character that cannot be +# written into a document at all is a property of the data, and +# gets a note naming the row; a buffer that would not grow is not, +# and has already stopped the statement by the time it is noticed. +# Either way what is left is a group with a gap in it, which is not +# a document and is not returned. +# +# A KEY that cannot be written is neither of those. It goes in as +# the empty key with its pair finished round it, so the object is a +# document whose keys are not the ones that were asked for, and the +# note is the whole of what is said about it. # # cp1250 is used here because it admits every byte, so a column can # hold one that stands for no character at all. @@ -528,13 +519,13 @@ id stored 3 8881 SELECT JSON_ARRAYAGG(c) AS v FROM t6; v -[,"ok",] +NULL Warnings: Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 SELECT JSON_OBJECTAGG(id, c) AS v FROM t6; v -{"1":", "2":"ok", "3":"} +NULL Warnings: Note 4035 Broken JSON string in argument 2 to function 'json_objectagg' at position 0 Note 4035 Broken JSON string in argument 2 to function 'json_objectagg' at position 0 @@ -553,13 +544,13 @@ v # when the group is asked for rather than as they arrive SELECT JSON_ARRAYAGG(c ORDER BY id) AS v FROM t6; v -[,"ok",] +NULL Warnings: Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 SELECT JSON_ARRAYAGG(DISTINCT c) AS v FROM t6; v -["ok",,] +NULL Warnings: Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 @@ -584,12 +575,10 @@ SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj FROM t2 WHERE id = 31; wrapped_once wrapped_once_obj -0 0 +NULL NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 63 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 63 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 37 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 63 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 63 SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj FROM t2 WHERE id = 30; @@ -605,6 +594,12 @@ wrapped_once wrapped_once_obj # of either is what is said about it, and both sides say which # argument it was. # +# Said at note level, where the same constructor refusing a value +# that did not parse says it at warning level. Both answer NULL. +# The difference is that refusing a character is what a released +# server did and refusing an unparsed value is not, so only the +# second is raised to where strict mode can act on it. +# # the value side, which has always been named SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; v @@ -667,4 +662,83 @@ v NULL Warnings: Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 0 +# +# 11. What a constructor or an aggregate returns when what it put +# together is not a document. +# +# These four have no closing pass over what they built, so whatever +# went in wrong is in the answer, and every way that can happen is +# above: a value typed as JSON that does not parse, one that parses +# as nothing at all, a nesting that only goes over the limit once +# the value is wrapped, and a character no document can carry, which +# leaves an aggregate's separator with nothing beside it. +# +# None of the four is a document and none of them is returned. The +# complaint saying what was wrong is raised where it always was; +# what follows it is NULL rather than the bytes. +# +# Two cases that look like these and are not stay as they were. A +# KEY that could not be written leaves the pair around it whole, so +# the object is a document whose keys are not the ones asked for. +# A group cut to fit its length limit is cut back to the last whole +# element, so the brackets go round a document as well. +# +# a value typed as JSON that does not parse, and one that parses as +# nothing at all +SELECT id, JSON_ARRAY(j) IS NULL AS refused, +JSON_OBJECT('k', j) IS NULL AS refused_obj +FROM t1 WHERE id >= 10 ORDER BY id; +id refused refused_obj +10 1 1 +11 1 1 +12 1 1 +20 1 1 +21 1 1 +22 1 1 +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, +JSON_OBJECTAGG(id, j) IS NULL AS refused_obj +FROM t1 WHERE id >= 10; +refused refused_obj +1 1 +# a nesting that only goes over the limit once the value is wrapped +SELECT id, JSON_ARRAY(j) IS NULL AS refused, +JSON_OBJECT('k', j) IS NULL AS refused_obj +FROM t2 ORDER BY id; +id refused refused_obj +30 0 0 +31 1 1 +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, +JSON_OBJECTAGG(id, j) IS NULL AS refused_obj +FROM t2 WHERE id = 31; +refused refused_obj +1 1 +# a character no document can carry, in a value +SELECT JSON_ARRAYAGG(c) IS NULL AS refused, +JSON_OBJECTAGG(id, c) IS NULL AS refused_obj FROM t6; +refused refused_obj +1 1 +# the same character in a KEY, which leaves a document behind +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +v +{"":1, "ok":2, "":3} +# a group cut to fit its length limit +SET SESSION group_concat_max_len= 28; +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +v +[{"a": 1, "b": [1, 2]}] +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; +v +{"1":{"a": 1, "b": [1, 2]}} +SET SESSION group_concat_max_len= DEFAULT; +# and what IS a document comes back as it always did +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id <= 2 ORDER BY id; +id v +1 [{"a": 1, "b": [1, 2]}] +2 [[1, 2, 3]] +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +v +[{"a": 1, "b": [1, 2]},[1, 2, 3]] +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; +v +{"1":{"a": 1, "b": [1, 2]}, "2":[1, 2, 3]} DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/main/func_json_embed.test b/mysql-test/main/func_json_embed.test index 9e1eb3b2f15cf..aac6a13741a94 100644 --- a/mysql-test/main/func_json_embed.test +++ b/mysql-test/main/func_json_embed.test @@ -20,9 +20,14 @@ --echo # the caller gets. --echo # --echo # A value that does not parse is spliced all the same, byte for ---echo # byte, and a note is raised saying which argument it was. Reading ---echo # the result back is still where a caller finds out that what came ---echo # out is not a document; the note is what says so at the time. +--echo # byte. What the constructor has then built is not a document, so +--echo # it returns NULL instead of it, and a warning names the argument +--echo # that was responsible. A warning and not a note, because it is +--echo # the only word the statement gets about why it was given NULL, +--echo # and because a statement running under strict mode then stops at +--echo # the value rather than finishing without one. A function that +--echo # reads its whole answer back has a second complaint to make about +--echo # the same value, and leaves this one a note. --echo # --echo # Non ASCII characters below are written as the bytes that encode --echo # them, so this file needs no character set of its own: @@ -83,12 +88,14 @@ SELECT id, JSON_ARRAY_APPEND('[0]', '$', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; --echo # ---echo # 3. Bytes that are not a document but do not look wrong afterwards. +--echo # 3. Bytes that are not a document and would not have looked wrong +--echo # afterwards. --echo # ---echo # An empty value leaves nothing behind at all, so the result parses ---echo # and reports one element fewer than it was given. The document ---echo # itself gives no sign of this, which is what makes the note the ---echo # only way to tell it apart from an array that really was short. +--echo # An empty value leaves nothing behind at all, so what is built +--echo # round it parses and reports one element fewer than it was given. +--echo # A document like that gives no sign of what happened to it, and +--echo # is exactly what is not returned: the row is refused, and the +--echo # warning says which argument it was. --echo # SET SESSION check_constraint_checks = OFF; @@ -112,7 +119,10 @@ SELECT JSON_ARRAYAGG(j) AS v, JSON_VALID(JSON_ARRAYAGG(j)) AS looks_fine --echo # 4. Nesting that only goes over the limit once the value is wrapped. --echo # --echo # The scanner takes 31 nested structures. A document of exactly 31 ---echo # is a document; put it inside one more and it is not. +--echo # is a document; put it inside one more and it is not, so the +--echo # constructor that wrapped it has nothing to return. A mutator +--echo # reads its whole answer back and finds out there, as it always +--echo # did. --echo # CREATE TABLE t2 (id INT, j JSON); @@ -255,13 +265,17 @@ SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS built, --echo # An aggregate writes the separator before it asks for the element, --echo # so a row that yields nothing leaves a separator with nothing on --echo # one side of it. There are two reasons a row can yield nothing ---echo # and they are not answered alike. A character that cannot be ---echo # written into a document at all is a property of the data: the ---echo # element was dropped and the separator left standing before, and ---echo # it still is, now with a note saying so. A buffer that would not ---echo # grow is not a property of the data, and has already stopped the ---echo # statement by the time it is noticed, so the group is refused ---echo # whole rather than returned with a gap in it. +--echo # and they are not reported alike. A character that cannot be +--echo # written into a document at all is a property of the data, and +--echo # gets a note naming the row; a buffer that would not grow is not, +--echo # and has already stopped the statement by the time it is noticed. +--echo # Either way what is left is a group with a gap in it, which is not +--echo # a document and is not returned. +--echo # +--echo # A KEY that cannot be written is neither of those. It goes in as +--echo # the empty key with its pair finished round it, so the object is a +--echo # document whose keys are not the ones that were asked for, and the +--echo # note is the whole of what is said about it. --echo # --echo # cp1250 is used here because it admits every byte, so a column can --echo # hold one that stands for no character at all. @@ -308,6 +322,12 @@ SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, --echo # of either is what is said about it, and both sides say which --echo # argument it was. --echo # +--echo # Said at note level, where the same constructor refusing a value +--echo # that did not parse says it at warning level. Both answer NULL. +--echo # The difference is that refusing a character is what a released +--echo # server did and refusing an unparsed value is not, so only the +--echo # second is raised to where strict mode can act on it. +--echo # --echo # the value side, which has always been named SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; @@ -338,4 +358,62 @@ SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 2; --echo # named the argument SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; +--echo # +--echo # 11. What a constructor or an aggregate returns when what it put +--echo # together is not a document. +--echo # +--echo # These four have no closing pass over what they built, so whatever +--echo # went in wrong is in the answer, and every way that can happen is +--echo # above: a value typed as JSON that does not parse, one that parses +--echo # as nothing at all, a nesting that only goes over the limit once +--echo # the value is wrapped, and a character no document can carry, which +--echo # leaves an aggregate's separator with nothing beside it. +--echo # +--echo # None of the four is a document and none of them is returned. The +--echo # complaint saying what was wrong is raised where it always was; +--echo # what follows it is NULL rather than the bytes. +--echo # +--echo # Two cases that look like these and are not stay as they were. A +--echo # KEY that could not be written leaves the pair around it whole, so +--echo # the object is a document whose keys are not the ones asked for. +--echo # A group cut to fit its length limit is cut back to the last whole +--echo # element, so the brackets go round a document as well. +--echo # + +--disable_warnings +--echo # a value typed as JSON that does not parse, and one that parses as +--echo # nothing at all +SELECT id, JSON_ARRAY(j) IS NULL AS refused, + JSON_OBJECT('k', j) IS NULL AS refused_obj + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, + JSON_OBJECTAGG(id, j) IS NULL AS refused_obj + FROM t1 WHERE id >= 10; +--echo # a nesting that only goes over the limit once the value is wrapped +SELECT id, JSON_ARRAY(j) IS NULL AS refused, + JSON_OBJECT('k', j) IS NULL AS refused_obj + FROM t2 ORDER BY id; +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, + JSON_OBJECTAGG(id, j) IS NULL AS refused_obj + FROM t2 WHERE id = 31; +--echo # a character no document can carry, in a value +SELECT JSON_ARRAYAGG(c) IS NULL AS refused, + JSON_OBJECTAGG(id, c) IS NULL AS refused_obj FROM t6; +--echo # the same character in a KEY, which leaves a document behind +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +--enable_warnings + +--echo # a group cut to fit its length limit +SET SESSION group_concat_max_len= 28; +--disable_warnings +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; +--enable_warnings +SET SESSION group_concat_max_len= DEFAULT; + +--echo # and what IS a document comes back as it always did +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id <= 2 ORDER BY id; +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; + DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/main/func_json_keys_typed_scans.result b/mysql-test/main/func_json_keys_typed_scans.result index af53917a13269..24a80ccc5f4f7 100644 --- a/mysql-test/main/func_json_keys_typed_scans.result +++ b/mysql-test/main/func_json_keys_typed_scans.result @@ -6,10 +6,12 @@ # sjis decodes every ASCII byte to itself and is flagged as not # ASCII-compatible all the same, so the array it answers is a # perfectly good document with no mark on it, and the splice reads -# it again. utf8mb4 stands beside it, over the same document, -# where the mark is given and the splice reads nothing - which is -# what makes the count a fact about the set rather than about the -# document or the splice. +# it again. The constructor round it then reads its own answer +# once more, that flag being the only warning it has that a set +# might not write the brackets as brackets - a set that says +# nothing against itself is not read at all. utf8mb4 stands +# beside it, over the same document, where no reading happens on +# either count. # # What the answers themselves look like is in # func_json_keys_typed. @@ -23,7 +25,7 @@ FLUSH STATUS; SELECT JSON_ARRAY(JSON_KEYS(j)) FROM ts; SHOW STATUS LIKE 'Json_scans'; Variable_name Value -Json_scans 2 +Json_scans 3 FLUSH STATUS; SELECT JSON_ARRAY(JSON_KEYS(j)) FROM tu; SHOW STATUS LIKE 'Json_scans'; diff --git a/mysql-test/main/func_json_keys_typed_scans.test b/mysql-test/main/func_json_keys_typed_scans.test index ceb282a6c6603..90901e94fc7ad 100644 --- a/mysql-test/main/func_json_keys_typed_scans.test +++ b/mysql-test/main/func_json_keys_typed_scans.test @@ -9,10 +9,12 @@ --echo # sjis decodes every ASCII byte to itself and is flagged as not --echo # ASCII-compatible all the same, so the array it answers is a --echo # perfectly good document with no mark on it, and the splice reads ---echo # it again. utf8mb4 stands beside it, over the same document, ---echo # where the mark is given and the splice reads nothing - which is ---echo # what makes the count a fact about the set rather than about the ---echo # document or the splice. +--echo # it again. The constructor round it then reads its own answer +--echo # once more, that flag being the only warning it has that a set +--echo # might not write the brackets as brackets - a set that says +--echo # nothing against itself is not read at all. utf8mb4 stands +--echo # beside it, over the same document, where no reading happens on +--echo # either count. --echo # --echo # What the answers themselves look like is in --echo # func_json_keys_typed. diff --git a/mysql-test/main/func_json_marks.result b/mysql-test/main/func_json_marks.result index af7c2669dedf2..7dbf5843a2cec 100644 --- a/mysql-test/main/func_json_marks.result +++ b/mysql-test/main/func_json_marks.result @@ -40,19 +40,19 @@ from_good [{"a": 1}] SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 2)) AS from_broken; from_broken -[{"a":1,] +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 3)) AS from_empty; from_empty -[] +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' SELECT JSON_OBJECT('k', (SELECT j FROM tj WHERE id = 2)) AS from_broken_object; from_broken_object -{"k": {"a":1,} +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' # # The compact one reads perfectly well and is simply not # written the way this function writes. @@ -167,16 +167,16 @@ Warning 1260 Row 3 was cut by JSON_ARRAYAGG() SET SESSION group_concat_max_len = @save_len; SELECT JSON_ARRAYAGG(j) AS agg_broken FROM tj; agg_broken -[{"a": 1},{"a":1,,,{"a":1}] +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' SELECT JSON_OBJECTAGG(id, j) AS objagg_broken FROM tj; objagg_broken -{"1":{"a": 1}, "2":{"a":1,, "3":, "4":{"a":1}} +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' -Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' # # 6. Character sets # @@ -187,12 +187,15 @@ Note 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' SELECT HEX(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7, JSON_VALID(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7_valid; array_in_swe7 array_in_swe7_valid -5B2261225D 0 +NULL NULL Warnings: -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 SELECT HEX(JSON_ARRAYAGG(CONVERT(s USING swe7))) AS agg_in_swe7 FROM tg; agg_in_swe7 -5B2261222C2262222C2263222C2264225D +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) AS array_in_ucs2, JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) AS array_in_ucs2_valid; @@ -287,9 +290,9 @@ Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at SELECT JSON_ARRAY(CONVERT(CONVERT(JSON_OBJECT('a', 1) USING ucs2) USING binary)) AS built_from_relabelled; built_from_relabelled -[{"a": 1}] +NULL Warnings: -Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 # # 8. Values that arrive through a reference # @@ -300,9 +303,9 @@ through_derived SELECT JSON_ARRAY(x) AS through_broken_derived FROM (SELECT j AS x FROM tj WHERE id = 2) d; through_broken_derived -[{"a":1,] +NULL Warnings: -Note 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' CREATE VIEW v1 AS SELECT JSON_OBJECT('a', 1) AS x; SELECT JSON_ARRAY(x) AS through_view FROM v1; through_view diff --git a/mysql-test/main/func_json_punct_charset.result b/mysql-test/main/func_json_punct_charset.result index 530ed55d5c1b9..e02a89c4bd8b5 100644 --- a/mysql-test/main/func_json_punct_charset.result +++ b/mysql-test/main/func_json_punct_charset.result @@ -79,10 +79,14 @@ srch 222422 SELECT HEX(JSON_ARRAY(CONVERT('1' USING swe7), CONVERT('2' USING swe7))) AS arr; arr -5B2231222C202232225D +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 SELECT HEX(JSON_OBJECT(CONVERT('a' USING swe7), CONVERT('1' USING swe7))) AS obj; obj -7B2261223A202231227D +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_object' at position 1 SELECT HEX(JSON_QUERY(CONVERT('1' USING swe7), '$')) AS qry; qry NULL diff --git a/mysql-test/main/func_json_scan_count.result b/mysql-test/main/func_json_scan_count.result index 9a4e431fd61b5..8d1b1a8bd8cc8 100644 --- a/mysql-test/main/func_json_scan_count.result +++ b/mysql-test/main/func_json_scan_count.result @@ -320,6 +320,43 @@ SHOW STATUS LIKE 'Json_scans'; Variable_name Value Json_scans 2 # +# The elements a group gathers through a table are not read on the +# way back out either, whatever composed them. A constructor +# attests to every value it will ever make, so the column it +# fills says so once for all its rows, and the reading that used +# to happen per element does not happen at all - which is the one +# count here that a released server never paid, it having read +# nothing anywhere and returned whatever the bytes were. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js)) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_ARRAY(id, js)) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# The same through the tree, which gathers the elements first and +# writes them out when the group is asked for. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js) ORDER BY id) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# And a group gathering a column of a caller's own, which nothing +# has attested to, reads every element as it always did. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(j) FROM t2; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# # A name given to a value in the select list and used again later # in the same statement reaches it through a reference, and a # reference reads what the item's RESULT side passes. For diff --git a/mysql-test/main/func_json_scan_count.test b/mysql-test/main/func_json_scan_count.test index 14ae73c41cadf..9d15cba6c5476 100644 --- a/mysql-test/main/func_json_scan_count.test +++ b/mysql-test/main/func_json_scan_count.test @@ -387,6 +387,47 @@ SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3; --enable_result_log SHOW STATUS LIKE 'Json_scans'; +--echo # +--echo # The elements a group gathers through a table are not read on the +--echo # way back out either, whatever composed them. A constructor +--echo # attests to every value it will ever make, so the column it +--echo # fills says so once for all its rows, and the reading that used +--echo # to happen per element does not happen at all - which is the one +--echo # count here that a released server never paid, it having read +--echo # nothing anywhere and returned whatever the bytes were. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js)) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(JSON_ARRAY(id, js)) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same through the tree, which gathers the elements first and +--echo # writes them out when the group is asked for. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js) ORDER BY id) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And a group gathering a column of a caller's own, which nothing +--echo # has attested to, reads every element as it always did. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(j) FROM t2; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + --echo # --echo # A name given to a value in the select list and used again later --echo # in the same statement reaches it through a reference, and a diff --git a/mysql-test/main/func_json_search_wide_charset.result b/mysql-test/main/func_json_search_wide_charset.result index cf04cc5c6a0af..0b49d0879785e 100644 --- a/mysql-test/main/func_json_search_wide_charset.result +++ b/mysql-test/main/func_json_search_wide_charset.result @@ -35,10 +35,9 @@ Warnings: Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 16 SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@u, 'all', @m))) AS utf32_spliced; utf32_spliced -0 +NULL Warnings: -Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 -Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 20 +Warning 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 SELECT JSON_TYPE(JSON_SEARCH(@u, 'all', @m)) AS utf32_type; utf32_type NULL @@ -110,9 +109,8 @@ SELECT id, JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT(d USING utf32), 'all', CONVERT(n USING utf32)))) AS spliced FROM t ORDER BY id; id spliced -1 0 +1 NULL 2 1 Warnings: -Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 -Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 20 +Warning 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 DROP TABLE t; diff --git a/mysql-test/main/func_json_sp.result b/mysql-test/main/func_json_sp.result index e03e282e517a8..0b0e355c7882d 100644 --- a/mysql-test/main/func_json_sp.result +++ b/mysql-test/main/func_json_sp.result @@ -444,14 +444,17 @@ INSERT INTO t2 (j) VALUES ('{"a":1}'); SELECT j, note, JSON_VALID(note) AS note_valid FROM t2; j note note_valid {"a":1} [{"a":1}] 1 +# A row whose document does not read leaves the trigger with +# nothing to set the column to. Saying so is a warning, and an +# INSERT under strict mode makes an error of it, so the row does +# not go in at all - where a released server stored the bytes the +# constructor had spliced and let the column keep them. SET SESSION check_constraint_checks = OFF; INSERT INTO t2 (j) VALUES ('{"a":1,'); +ERROR HY000: Unexpected end of JSON text in argument 1 to function 'json_array' SET SESSION check_constraint_checks = ON; SELECT j, note, JSON_VALID(note) AS note_valid FROM t2 ORDER BY j; j note note_valid -{"a":1, [{"a":1,] 0 {"a":1} [{"a":1}] 1 -Warnings: -Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 DROP TABLE t2; DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp.test b/mysql-test/main/func_json_sp.test index 5bc533e198cda..980ce3c8a5d3c 100644 --- a/mysql-test/main/func_json_sp.test +++ b/mysql-test/main/func_json_sp.test @@ -362,7 +362,13 @@ END $$ --delimiter ; INSERT INTO t2 (j) VALUES ('{"a":1}'); SELECT j, note, JSON_VALID(note) AS note_valid FROM t2; +--echo # A row whose document does not read leaves the trigger with +--echo # nothing to set the column to. Saying so is a warning, and an +--echo # INSERT under strict mode makes an error of it, so the row does +--echo # not go in at all - where a released server stored the bytes the +--echo # constructor had spliced and let the column keep them. SET SESSION check_constraint_checks = OFF; +--error ER_JSON_EOS INSERT INTO t2 (j) VALUES ('{"a":1,'); SET SESSION check_constraint_checks = ON; SELECT j, note, JSON_VALID(note) AS note_valid FROM t2 ORDER BY j; diff --git a/mysql-test/main/func_json_splice_depth.result b/mysql-test/main/func_json_splice_depth.result index 6810fee6fac61..2ad7c15a3bb6d 100644 --- a/mysql-test/main/func_json_splice_depth.result +++ b/mysql-test/main/func_json_splice_depth.result @@ -69,9 +69,9 @@ SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); SET @d30a= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); SELECT JSON_ARRAY(JSON_EXTRACT(@d31, '$')) AS built_deep; built_deep -[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 SELECT JSON_ARRAY(JSON_EXTRACT(@d30a, '$')) AS built_limit; built_limit [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] @@ -90,8 +90,7 @@ SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k31, '$'))) AS padded_deep_depth; padded_deep_depth NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 -Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 452 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 SELECT JSON_ARRAY(JSON_EXTRACT(@k30, '$')) AS padded_limit; SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_limit_depth; padded_limit_depth @@ -107,8 +106,7 @@ AS padded_two_depth; padded_two_depth NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 483 -Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 438 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 483 # # 9. A document that was EDITED is as deep as it was or as deep as # what went into it, whichever is more, and says so without @@ -137,8 +135,7 @@ AS edited_deep_depth; edited_deep_depth NULL Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 479 -Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 443 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 479 # # 10. Wrapping, where the new array is a structure the document # being read never had. What was found is put inside it and diff --git a/mysql-test/main/func_json_tmp_scan_count.result b/mysql-test/main/func_json_tmp_scan_count.result index 7fcda73629aef..9d8c5f207b6c5 100644 --- a/mysql-test/main/func_json_tmp_scan_count.result +++ b/mysql-test/main/func_json_tmp_scan_count.result @@ -57,9 +57,9 @@ SHOW STATUS LIKE 'Json_scans'; Variable_name Value Json_scans 2 # -# 4. The same shape with a producer that answers per evaluation -# rather than for every one of them. Nothing can be said about the -# column at the time it is made, so each row is read on the way out. +# 4. The same shape with a producer that attests for its class. +# The claim holds for the column at the time it is made, so no row +# is read on the way out. # FLUSH STATUS; SELECT JSON_ARRAY(x) AS r FROM @@ -69,7 +69,7 @@ r [{"a": 2}] SHOW STATUS LIKE 'Json_scans'; Variable_name Value -Json_scans 2 +Json_scans 0 # # 5. Through a GROUP BY, which writes a temporary table of its own. # @@ -179,10 +179,10 @@ SHOW STATUS LIKE 'Json_scans'; Variable_name Value Json_scans 2 # -# 12. The same, with a producer that attests per evaluation rather -# than to every value it makes. Its column is read on the way out, -# so the total matches 11 from the other side: 11's two readings are -# the producer's own, these two are the readings back 11 was spared. +# 12. The same, with a producer that attests for its class. The +# column takes that claim, so the elements are not read back and +# nothing is counted here - where 11 pays two readings for the +# values its producer parsed itself. # FLUSH STATUS; SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS r FROM t1; @@ -190,7 +190,7 @@ r [{"a": 1},{"a": 2}] SHOW STATUS LIKE 'Json_scans'; Variable_name Value -Json_scans 2 +Json_scans 0 # # 13. An aggregate whose arguments are buffered before the group can # be built - a join with a grouping the rows do not arrive in. The @@ -361,11 +361,11 @@ Variable_name Value Json_scans 2 SET SESSION debug_dbug= @old_debug; # -# 22. The same UNION with one branch that attests per evaluation -# rather than to every value it makes. One branch not attesting is -# the whole column not attesting, there being no telling which -# branch a row came out of once it is in there. So every row is -# read on the way out - four - on top of the two the JSON_EXTRACT +# 22. The same UNION where both branches attest for their class, +# which is what a column needs: there is no telling which branch a +# row came out of once it is in there, so one branch not attesting +# would be the whole column not attesting. No row is read on the +# way out, and the two counted here are the ones the JSON_EXTRACT # branch read itself. # FLUSH STATUS; @@ -380,7 +380,7 @@ r [{"a": 2}] SHOW STATUS LIKE 'Json_scans'; Variable_name Value -Json_scans 6 +Json_scans 2 # # 23. The same again with a branch that is a column, which says # nothing about itself and so says nothing for the column it is @@ -415,8 +415,10 @@ SHOW STATUS LIKE 'Json_scans'; Variable_name Value Json_scans 2 # -# And one list that answers per evaluation is the same column not -# answering, exactly as one branch is. +# And a list whose producer attests for its class leaves the +# column attesting, exactly as a branch does. The one counted +# here is the JSON_EXTRACT list reading its own value; the +# constructor reads nothing. # FLUSH STATUS; WITH d(x) AS @@ -427,7 +429,7 @@ r [{"b": 2}] SHOW STATUS LIKE 'Json_scans'; Variable_name Value -Json_scans 5 +Json_scans 1 # # 25. A recursive CTE, whose column is built once the parts that do # not go round have agreed on a type - the parts that DO go round diff --git a/mysql-test/main/func_json_tmp_scan_count.test b/mysql-test/main/func_json_tmp_scan_count.test index 6555c6f527ea3..fd32af4f4bc10 100644 --- a/mysql-test/main/func_json_tmp_scan_count.test +++ b/mysql-test/main/func_json_tmp_scan_count.test @@ -55,9 +55,9 @@ SELECT JSON_ARRAY(x) AS r FROM SHOW STATUS LIKE 'Json_scans'; --echo # ---echo # 4. The same shape with a producer that answers per evaluation ---echo # rather than for every one of them. Nothing can be said about the ---echo # column at the time it is made, so each row is read on the way out. +--echo # 4. The same shape with a producer that attests for its class. +--echo # The claim holds for the column at the time it is made, so no row +--echo # is read on the way out. --echo # FLUSH STATUS; SELECT JSON_ARRAY(x) AS r FROM @@ -144,10 +144,10 @@ SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS r FROM t1; SHOW STATUS LIKE 'Json_scans'; --echo # ---echo # 12. The same, with a producer that attests per evaluation rather ---echo # than to every value it makes. Its column is read on the way out, ---echo # so the total matches 11 from the other side: 11's two readings are ---echo # the producer's own, these two are the readings back 11 was spared. +--echo # 12. The same, with a producer that attests for its class. The +--echo # column takes that claim, so the elements are not read back and +--echo # nothing is counted here - where 11 pays two readings for the +--echo # values its producer parsed itself. --echo # FLUSH STATUS; SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS r FROM t1; @@ -287,11 +287,11 @@ SHOW STATUS LIKE 'Json_scans'; SET SESSION debug_dbug= @old_debug; --echo # ---echo # 22. The same UNION with one branch that attests per evaluation ---echo # rather than to every value it makes. One branch not attesting is ---echo # the whole column not attesting, there being no telling which ---echo # branch a row came out of once it is in there. So every row is ---echo # read on the way out - four - on top of the two the JSON_EXTRACT +--echo # 22. The same UNION where both branches attest for their class, +--echo # which is what a column needs: there is no telling which branch a +--echo # row came out of once it is in there, so one branch not attesting +--echo # would be the whole column not attesting. No row is read on the +--echo # way out, and the two counted here are the ones the JSON_EXTRACT --echo # branch read itself. --echo # FLUSH STATUS; @@ -324,8 +324,10 @@ WITH d(x) AS SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; SHOW STATUS LIKE 'Json_scans'; --echo # ---echo # And one list that answers per evaluation is the same column not ---echo # answering, exactly as one branch is. +--echo # And a list whose producer attests for its class leaves the +--echo # column attesting, exactly as a branch does. The one counted +--echo # here is the JSON_EXTRACT list reading its own value; the +--echo # constructor reads nothing. --echo # FLUSH STATUS; WITH d(x) AS diff --git a/mysql-test/main/func_json_tmp_trust.result b/mysql-test/main/func_json_tmp_trust.result index d2242effe9525..e43d40bae7024 100644 --- a/mysql-test/main/func_json_tmp_trust.result +++ b/mysql-test/main/func_json_tmp_trust.result @@ -142,9 +142,9 @@ SELECT n + 1, t5.bad FROM r, t5 WHERE n < 2) SELECT n, JSON_ARRAY(x) AS unioned FROM r ORDER BY n; n unioned 1 [{"a": 1}] -2 [not a document] +2 NULL Warnings: -Note 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 DROP TABLE t5; # # Branches written in different character sets. The column takes @@ -167,14 +167,14 @@ HEX(x) AS bytes FROM UNION ALL SELECT JSON_QUERY(b, '$') FROM t7) d ORDER BY bytes; packed still_a_document bytes -5B005B0031002C0032005D5D 0 005B0031002C0032005D -5B007B002200610022003A0031007D5D 0 007B002200610022003A0031007D +NULL 0 005B0031002C0032005D +NULL 0 007B002200610022003A0031007D 5B5B332C345D5D 1 5B332C345D 5B7B2262223A327D5D 1 7B2262223A327D Warnings: -Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 -Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 # # The same two branches with the wide one on the right, so that it @@ -186,14 +186,14 @@ HEX(x) AS bytes FROM UNION ALL SELECT JSON_QUERY(j, '$') FROM t6) d ORDER BY bytes; packed still_a_document bytes -5B005B0031002C0032005D5D 0 005B0031002C0032005D -5B007B002200610022003A0031007D5D 0 007B002200610022003A0031007D +NULL 0 005B0031002C0032005D +NULL 0 007B002200610022003A0031007D 5B5B332C345D5D 1 5B332C345D 5B7B2262223A327D5D 1 7B2262223A327D Warnings: -Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 -Note 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 # # And both branches in the wide set, where the column takes that @@ -209,15 +209,50 @@ unioned still_a_document [{"a":1}] 1 DROP TABLE t6, t7; # -# 6. A producer that is a document only some of the time keeps its -# column untrusted. JSON_OBJECT answers per evaluation, not for the -# class, so a column fed by one is read like any other. +# 6. A producer that attests to every value it will ever make, +# rather than to the one it has just made. A constructor returns a +# document or nothing whatever it is given, so a column fed by one +# is read without being asked. So does an aggregate, and so does a +# constructor composing from one. # SELECT JSON_ARRAY(x) AS spliced FROM (SELECT JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY 1; spliced [{"a": 1}] [{"a": 2}] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_ARRAY(id, j) AS x FROM t1) d ORDER BY 1; +spliced +[[1, {"a":1}]] +[[2, [1,2]]] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_ARRAYAGG(j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +spliced +[[[1,2]]] +[[{"a":1}]] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_OBJECTAGG(id, j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +spliced +[{"1":{"a":1}}] +[{"2":[1,2]}] +# +# And in a character set that cannot write the brackets it answers +# nothing, there being no document to promise - see section 12. +# +SELECT HEX(JSON_ARRAY(x)) AS spliced FROM +(SELECT JSON_ARRAY(CONVERT(j USING swe7)) AS x FROM t1) d ORDER BY 1; +spliced +NULL +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 1977 Cannot convert 'utf8mb4' character 0x5B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 # # 7. A function that returns more than it was given. JSON_REMOVE # writes a space after every separator it copies, so taking one @@ -568,8 +603,7 @@ id d 1 NULL 2 3 Warnings: -Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 -Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 452 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 # # Each level of these is fourteen characters, so half the length is # seven times the limit and the length can tell neither of them from diff --git a/mysql-test/main/func_json_tmp_trust.test b/mysql-test/main/func_json_tmp_trust.test index 2fb43e3e0f03d..c1fdcff1c521d 100644 --- a/mysql-test/main/func_json_tmp_trust.test +++ b/mysql-test/main/func_json_tmp_trust.test @@ -143,12 +143,26 @@ SELECT JSON_ARRAY(x) AS unioned, JSON_VALID(x) AS still_a_document FROM DROP TABLE t6, t7; --echo # ---echo # 6. A producer that is a document only some of the time keeps its ---echo # column untrusted. JSON_OBJECT answers per evaluation, not for the ---echo # class, so a column fed by one is read like any other. +--echo # 6. A producer that attests to every value it will ever make, +--echo # rather than to the one it has just made. A constructor returns a +--echo # document or nothing whatever it is given, so a column fed by one +--echo # is read without being asked. So does an aggregate, and so does a +--echo # constructor composing from one. --echo # SELECT JSON_ARRAY(x) AS spliced FROM (SELECT JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_ARRAY(id, j) AS x FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_ARRAYAGG(j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_OBJECTAGG(id, j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +--echo # +--echo # And in a character set that cannot write the brackets it answers +--echo # nothing, there being no document to promise - see section 12. +--echo # +SELECT HEX(JSON_ARRAY(x)) AS spliced FROM + (SELECT JSON_ARRAY(CONVERT(j USING swe7)) AS x FROM t1) d ORDER BY 1; --echo # --echo # 7. A function that returns more than it was given. JSON_REMOVE diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 0914c8f3d9c19..be4ce451b3143 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -569,6 +569,46 @@ static inline bool is_json_compatible_charset(CHARSET_INFO *cs) } +/* + Whether a container just composed has to be refused for the character + set it was composed in. + + A container is written with punctuation round it, and a set that does + not encode that punctuation as itself holds no container: what was + composed reads as something else, or as nothing. A released server + returned those bytes. The functions that read their whole answer back + have always returned NULL there instead, the reading being what fails, + and the ones that compose their answer say the same thing here, in the + same words and at the same position. + + Which is why it is read rather than reckoned. The question above is + not the question this is: a set stops being JSON-compatible as soon as + it puts ANY character somewhere other than ASCII does, and a set can do + that and still write brackets - sjis moves the backslash and leaves the + brackets where they were, so what it composes IS a document and is + returned. A compatible set is not read at all, so the reading is + confined to the sets that might have got it wrong. +*/ +static bool json_container_charset_refused(String *str, const char *fname) +{ + json_engine_t je; + + if (is_json_compatible_charset(str->charset())) + return false; + + json_scan_start(&je, str->charset(), (const uchar *) str->ptr(), + (const uchar *) str->ptr() + str->length()); + while (json_scan_next(&je) == 0) + {} + if (!je.s.error) + return false; + + report_json_error_ex(str->ptr(), &je, fname, 0, + Sql_condition::WARN_LEVEL_WARN); + return true; +} + + /* Whether the document argument leaves a function free to return what it composes, instead of reading the whole answer again at the end to @@ -3112,15 +3152,16 @@ struct Json_splice_marks the column afterwards. Nothing on the way from the column to here reads them. - So the value is parsed as it is copied, but a value that does not - parse is copied ALL THE SAME, with a note saying so. It has always - been copied, callers have always been able to see the result, and - what a released server does is not something to take away from the - people relying on it; the note is what is new. Reading the result - back is where a caller finds out, exactly as before. What the parse - is for is to know WHETHER to say anything - and, once the trust - predicates exist, to keep an unparseable value from ever being taken - on trust by a later reader. + So the value is parsed as it is copied, and a value that does not + parse is copied ALL THE SAME, with a complaint saying so. Undoing the + copy is not this function's to do: the bytes have gone in where the + caller asked for them, and how much of a result there is around them + is the caller's to know. What is owed here is to say so, which is + what the marks are for, and each caller answers it the way its own + result works. The four that compose a document and return what they + composed return NULL rather than bytes no reader will take; the + ones that read their whole answer back find out there, exactly as + before. A value written in another character set is converted first, because the result is read in the character set it is being built in, not in @@ -3147,13 +3188,24 @@ struct Json_splice_marks 'value_depth' is how deep the item says the value goes, or JSON_DEPTH_UNKNOWN where it does not say. It is only ever read alongside is_valid. + + 'lv' is the level to complain at. A caller that has another + complaint to make about the same value - which every caller that + reads its whole answer back does, from the reading - leaves this a + note, so that the two do not say the same thing twice at the same + volume. The four that compose their answer and return it have no + second complaint and no answer either, and pass a warning: it is the + only word the statement gets about why it was given NULL, and under + strict mode it stops the statement where the value went wrong rather + than letting it finish with nothing. */ static int append_json_typed_value(String *str, const String *sv, uint depth, const char *fname, int n_param, Json_splice_marks &marks, bool caller_reads_back, bool is_valid, bool is_nice, bool need_nice, - uint value_depth) + uint value_depth, + Sql_condition::enum_warning_level lv) { StringBuffer cnv; const char *ptr= sv->ptr(); @@ -3381,12 +3433,15 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, Too deep only once it is in place. There is nothing wrong with the value itself, so a caller that reads its whole result back says it about the answer rather than about this one value - which is what it - has always said, and where it has always said it. All that is owed - here is to stop such an answer being taken on trust. + has always said, and where it has always said it. A caller that does not read back has nobody else to hear it from, so - for that one the scanner is put into error and the note goes out as - it always has. + for that one the scanner is put into error and the complaint goes out + below at whatever level that caller asked for. Clearing the mark is + what carries it further: a composed answer that reaches past the + limit cannot be read back by this server at all, so the caller + composing it has nothing to return, and the mark is where it finds + that out. */ if (je.s.error == 0 && max_level + (int) depth >= JSON_DEPTH_LIMIT) { @@ -3413,17 +3468,12 @@ static int append_json_typed_value(String *str, const String *sv, uint depth, what went wrong with a value, and reports nothing at all when that engine is not itself in error. - A note and not a warning, because a warning becomes an error inside - a statement running under strict mode, and a statement that used to - finish would then stop finishing. What is being added here is - something to read, not a new way to fail. json_valid() says the - same thing at the same level for the same reason. + At the level the caller asked for, and see there for what picks it. */ if (je.s.error) { marks.is_valid= marks.is_nice= false; - report_json_error_ex(ptr, &je, fname, n_param, - Sql_condition::WARN_LEVEL_NOTE); + report_json_error_ex(ptr, &je, fname, n_param, lv); } return reformat ? 0 : append_simple(str, ptr, length); @@ -3485,7 +3535,9 @@ __attribute__((warn_unused_result)) static int append_json_value(String *str, Item *item, String *tmp_val, uint depth, const char *fname, int n_param, Json_splice_marks &marks, - bool caller_reads_back, bool need_nice) + bool caller_reads_back, bool need_nice, + Sql_condition::enum_warning_level lv= + Sql_condition::WARN_LEVEL_NOTE) { /* Said before anything is looked at, because it holds for whatever the @@ -3562,7 +3614,7 @@ static int append_json_value(String *str, Item *item, String *tmp_val, return append_json_typed_value(str, sv, depth, fname, n_param, marks, caller_reads_back, item->is_valid_json(), item->is_nice_json(), need_nice, - item->last_depth()); + item->last_depth(), lv); /* Being a document is what gets a value spliced rather than quoted, @@ -3596,7 +3648,11 @@ static int append_json_value(String *str, Item *item, String *tmp_val, } -/* The same, reading the value out of a row rather than off an Item. */ +/* + The same, reading the value out of a row rather than off an Item. + Only JSON_ARRAYAGG reads a row back this way, so the level it + complains at is the one that composer asks for everywhere else. +*/ __attribute__((warn_unused_result)) static int append_json_value_from_field(String *str, Item *i, Field *f, const uchar *key, size_t offset, String *tmp_val, @@ -3663,7 +3719,8 @@ static int append_json_value_from_field(String *str, return append_json_typed_value(str, sv, depth, fname, n_param, marks, false, f->is_valid_json_static(), f->is_nice_json_static(), false, - f->json_static_depth()); + f->json_static_depth(), + Sql_condition::WARN_LEVEL_WARN); rc= append_escaped_value(str, sv, i->result_type() == STRING_RESULT, fname, n_param); @@ -3806,6 +3863,14 @@ bool Item_func_json_array::fix_length_and_dec(THD *thd) fix_char_length_ulonglong(char_length); tmp_val.set_charset(collation.collation); + /* + An argument can leave this with no document to return - see + val_str() - and that is decided by what the argument holds rather + than by whether it was NULL, so an argument that is never NULL does + not settle it. The constructor with no arguments at all is the one + that cannot happen to, and it is answered above. + */ + set_maybe_null(); return FALSE; } @@ -3822,21 +3887,39 @@ String *Item_func_json_array::val_str(String *str) Json_splice_marks marks(1); m_marks.clear(); + /* + Said for this evaluation rather than left where the last one put it: + a row refused below would otherwise leave it set for the row after, + which composes a document and would be answered NULL for it. + */ + null_value= 0; str->length(0); str->set_charset(collation.collation); - if (str->append('[') || - ((arg_count > 0) && - append_json_value(str, args[0], &tmp_val, 1, func_name(), 0, marks, - false, false))) + if (str->append('[')) goto err_return; - for (n_arg=1; n_arg < arg_count; n_arg++) + for (n_arg=0; n_arg < arg_count; n_arg++) { - if (str->append(STRING_WITH_LEN(json_loose_comma)) || + if ((n_arg && str->append(STRING_WITH_LEN(json_loose_comma))) || append_json_value(str, args[n_arg], &tmp_val, 1, func_name(), - (int) n_arg, marks, false, false)) + (int) n_arg, marks, false, false, + Sql_condition::WARN_LEVEL_WARN)) + goto err_return; + + /* + A value that is not a document is the whole array not being one, + and there is no other answer to compose, so the arguments after it + are not evaluated. The marks ARE the reading a closing pass would + have done, taken a value at a time as the values go in, so this is + where the question is answered. + + A released server spliced such a value as it stood and returned + bytes no reader will take. The warning naming the argument is + raised where it always was, just above. + */ + if (!marks.is_valid) goto err_return; } @@ -3859,13 +3942,14 @@ String *Item_func_json_array::val_str(String *str) marks.is_nice are cleared by whichever of them did not hold, and that is the whole of what this has to go on. - All of which holds only where the brackets written above are - brackets - see is_json_compatible_charset(). + is_valid is settled above, a value at a time, so only the brackets + are left to ask about. */ - bool is_json_compatible= is_json_compatible_charset(str->charset()); + if (json_container_charset_refused(str, func_name())) + goto err_return; - m_marks.set(str, is_json_compatible && marks.is_valid, - is_json_compatible && marks.is_nice, marks.deepest); + DBUG_ASSERT(marks.is_valid); + m_marks.set(str, true, marks.is_nice, marks.deepest); return str; } @@ -4467,24 +4551,36 @@ String *Item_func_json_object::val_str(String *str) Json_splice_marks marks(1); m_marks.clear(); + /* Said per evaluation, as in the sister constructor. */ + null_value= 0; str->length(0); str->set_charset(collation.collation); - if (str->append('{') || - (arg_count > 0 && - (append_json_keyname(str, args[0], &tmp_val, func_name(), 0) || - append_json_value(str, args[1], &tmp_val, 1, func_name(), 1, marks, - false, false)))) + if (str->append('{')) goto err_return; - for (n_arg=2; n_arg < arg_count; n_arg+=2) + /* + The first pair used to be written outside this loop, the separator + being what set it apart. It is written here instead so that the + refusal below is written once for every pair rather than twice. + */ + for (n_arg=0; n_arg < arg_count; n_arg+=2) { - if (str->append(STRING_WITH_LEN(json_loose_comma)) || + if ((n_arg && str->append(STRING_WITH_LEN(json_loose_comma))) || append_json_keyname(str, args[n_arg], &tmp_val, func_name(), (int) n_arg) || append_json_value(str, args[n_arg+1], &tmp_val, 1, func_name(), - (int) n_arg + 1, marks, false, false)) + (int) n_arg + 1, marks, false, false, + Sql_condition::WARN_LEVEL_WARN)) + goto err_return; + + /* + As in the sister constructor: one value that is not a document is + the whole object not being one, so the arguments after it are not + evaluated. + */ + if (!marks.is_valid) goto err_return; } @@ -4496,11 +4592,12 @@ String *Item_func_json_object::val_str(String *str) if (str->length() <= result_limit) { - /* As in the sister constructor above. */ - bool is_json_compatible= is_json_compatible_charset(str->charset()); + /* As in the sister constructor above, and so is the refusal. */ + if (json_container_charset_refused(str, func_name())) + goto err_return; - m_marks.set(str, is_json_compatible && marks.is_valid, - is_json_compatible && marks.is_nice, marks.deepest); + DBUG_ASSERT(marks.is_valid); + m_marks.set(str, true, marks.is_nice, marks.deepest); return str; } @@ -7385,15 +7482,20 @@ void Item_func_json_arrayagg::clear() stays written whether an element follows it or not, so a row declined here leaves a separator with nothing on one side of it. - That is what has always been returned for a value holding a - character no document can carry, and it still is: the note is the new - part, the shape is not. A row lost to a buffer that would not grow - is a different matter - nothing about the data asked for it, and the - statement is over anyway - so the group is marked and refused whole + That is what has always been written for a value holding a character + no document can carry, and it still is - the writing is not this + function's to undo, and how much of a group there is around it is not + its to know either. A row lost to a buffer that would not grow is a + different matter: nothing about the data asked for it and the + statement is over anyway, so the group is marked and refused whole when it is asked for. A row holding something that does not parse as JSON reaches neither of these. It is written out as it stands, the same as it always was. + + What becomes of the group in the two cases that leave it not a + document is settled where it is asked for, both of them through the + same mark - see val_str(). */ String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) { @@ -7409,7 +7511,7 @@ String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) m_tmp_json.length(0); rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0, marks, - false, false); + false, false, Sql_condition::WARN_LEVEL_WARN); if (!marks.is_valid) m_elements_valid= false; m_elements_depth= MY_MAX(m_elements_depth, marks.deepest); @@ -7512,6 +7614,29 @@ String* Item_func_json_arrayagg::val_str(String *str) return NULL; } + /* + A group that is not a document is not returned either, for the + reason the constructors give where they refuse the same thing: the + elements were read as they went in, that reading is the only one + there is, and what it found has nowhere else to be acted on. + + The row that made it so has already been reported - a value that + did not parse through the warning naming the argument, one carrying + a character no document can hold through the note saying which - so + what is added here is the refusal and not the complaint. + + A group cut back to fit the length limit is NOT this. It is cut to + the last whole element and the brackets go round what is left, so + it is a shorter document and still a document; the mark below + declines it all the same, that being about the answer this group + had rather than about the one it gives. + */ + if (!m_elements_valid) + { + null_value= 1; + return NULL; + } + /* Asked for a second time, the group is already inside its brackets and there is nothing left to do to it. What can be said about it @@ -7535,13 +7660,19 @@ String* Item_func_json_arrayagg::val_str(String *str) m_closed= true; } + /* + Asked once the brackets are on, for the reason given there. The + bare name, func_name() here carrying the open bracket that the cut + warning writes its own closing one after. + */ + if (json_container_charset_refused(str, json_arrayagg_name)) + goto bad_result; + /* A group that was cut to fit the length limit is cut back to the last whole element, so what stands between the brackets reads - - and it is a shorter answer than the group has, with the row it - stopped at reported. One that carried something not readable as - JSON is a group with that in it still. Either way the brackets - are around something this cannot attest to. + but it is a shorter answer than the group has, with the row it + stopped at reported, so this declines it. Never said to be nicely written: what goes between the elements is the separator this function inherits from GROUP_CONCAT, a comma @@ -7549,9 +7680,7 @@ String* Item_func_json_arrayagg::val_str(String *str) space there as well. There is no asking for another one - the grammar takes no SEPARATOR here and writes that comma itself. */ - m_marks.set(str, is_json_compatible_charset(str->charset()) && - m_elements_valid && !warning_for_row, false, - m_elements_depth); + m_marks.set(str, !warning_for_row, false, m_elements_depth); } return str; @@ -7696,31 +7825,25 @@ bool Item_func_json_objectagg::add() */ uint32 old_length= result.length(); + /* + A cut group takes nothing further, so neither argument is evaluated + for the rows after the cut. The rows are still read; see the cut at + the end of this function. + + An argument with side effects - a stored function that writes, an + assignment, a warning raised per row - therefore runs for the rows + up to the cut and not for those after it. Nothing documents how + many times an aggregate evaluates its arguments. + */ + if (m_cut) + return 0; + key= args[0]->val_str(&buf); if (args[0]->is_null()) return 0; m_row_count++; - /* - Nothing more goes into a group that has been cut, but the rows of it - are still read to the end. An argument is an expression, and working - one out once for every row of the group is what the object has always - done - a row that stops being read stops whatever the expression does, - which is not the caller's answer to give away. So only the writing - stops here; see the cut at the end of this function. - - The value is worked out and dropped rather than written, and it is - asked for the way append_json_value() would ask a value of its type. - The key was worked out above, where every row needs it: whether a row - makes a pair of it is what its being NULL decides, cut or not. - */ - if (m_cut) - { - args[1]->update_null_value(); - return 0; - } - /* Whether this pair needs a separator in front of it is whether a pair has been written already, which is what null_value says until this @@ -7744,13 +7867,16 @@ bool Item_func_json_objectagg::add() { if (rc == JSON_APPEND_OOM) goto bad_pair; - report_bad_chr_note(func_name(), 1); /* - However much of the key could be written went in and the pair was - finished around it, so the object is complete and its keys are - not what was asked for. + The escaping writes into the buffer and says afterwards how much of + what it wrote counts, and a key it refused counts for none - so + the key that goes in is the empty one, with the quotes and the + colon written round it here as they are round any other. The + object that comes of it is a document, and its keys are not the + ones that were asked for; the note is what says so, and nothing + about being a document is dropped for it. */ - m_pairs_valid= false; + report_bad_chr_note(func_name(), 1); } if (result.append(STRING_WITH_LEN("\":")) || @@ -7759,7 +7885,7 @@ bool Item_func_json_objectagg::add() buf.length(0); rc= append_json_value(&result, args[1], &buf, 1, func_name(), 1, marks, - false, false); + false, false, Sql_condition::WARN_LEVEL_WARN); if (!marks.is_valid) m_pairs_valid= false; m_pairs_depth= MY_MAX(m_pairs_depth, marks.deepest); @@ -7826,13 +7952,26 @@ String* Item_func_json_objectagg::val_str(String* str) } m_closed= true; + /* + And an object that is not a document is not returned, for the reason + the sister aggregate gives where it refuses the same thing. Only a + VALUE gets an object here: a key that could not be written leaves + the pair whole around an empty one. + + The braces are on by now, which is what the second of these reads. + */ + if (!m_pairs_valid || json_container_charset_refused(&result, func_name())) + { + null_value= 1; + return 0; + } + /* Never said to be nicely written: a key is followed here by a colon and the value straight after it, where the loose form puts a space between the two. */ - m_marks.set(&result, is_json_compatible_charset(result.charset()) && - m_pairs_valid, false, m_pairs_depth); + m_marks.set(&result, true, false, m_pairs_depth); return &result; } diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 941d2722152f5..dd3056c7ed69f 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -540,6 +540,23 @@ class Item_func_json_array: public Item_json_func static LEX_CSTRING name= {STRING_WITH_LEN("json_array") }; return name; } + /* + A document or nothing. Every value is read as it goes in - the ones + quoted here come out of json_escape() and are documents of their + own, and the ones spliced as they stand were either read on the way + or passed by something that had already read them - and a value that + did not read as one takes the whole result down to NULL rather than + into the answer. What goes between them is written here. + + Which leaves the brackets and the separators. In a character set + that might not write those as themselves the result is read back + before it is returned, and one that does not read comes back NULL, + exactly as it does for the functions that always read back. + + Said for Item_func_json_object as well, which composes the same way + through the same routines. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -964,8 +981,8 @@ class Item_func_json_arrayagg : public Item_func_group_concat having failed to grow. Neither a row whose value does not parse as JSON nor one holding a character no document can carry is this: the first is written out as it stands and the second is dropped - where it always was, both with a note. Reset for each group by - clear(). + where it always was, both with a note. Those two are answered by + the mark below instead. Reset for each group by clear(). */ bool m_bad_element; /* @@ -984,7 +1001,8 @@ class Item_func_json_arrayagg : public Item_func_group_concat them. Reset for each group by clear(), the same as above, and kept apart from it because they say different things - that one means the group is missing a row, this one that the group is complete and - still not a document. + still not a document. Either way there is no array to return, + so a group this is cleared for is refused when it is asked for. */ bool m_elements_valid; /* @@ -1027,6 +1045,24 @@ class Item_func_json_arrayagg : public Item_func_group_concat bool is_valid_json() const override { return m_marks.valid(); } bool is_nice_json() const override { return m_marks.nice(); } uint last_depth() const override { return m_marks.depth(); } + /* + A document or nothing, for the reason given where + Item_func_json_array declares the same: the elements are read as + they arrive, a group carrying one that did not read as a document + comes back NULL, and the brackets are written here. A group cut to + fit its length limit is cut back to the last whole element, so the + brackets still go round a document. + */ + bool is_valid_json_static() const override { return true; } + /* + Asked for rather than read off. The aggregate this inherits from + settles a NULL before any of the group is written, that being what + it says about a group with no rows in it and the only time it says + it, so Item_sum::is_null() reads what is already there. A group + refused here is refused as it is written out, and what is already + there is then the answer to a question nothing has asked yet. + */ + bool is_null() override { update_null_value(); return null_value; } LEX_CSTRING func_name_cstring() const override { @@ -1065,13 +1101,19 @@ class Item_func_json_objectagg : public Item_sum bool m_closed; /* Whether everything that went into this group leaves the object - around it answerable for. Cleared by a key or a value holding a - character that could not be written, which goes in as however much - of it fitted, or by a value that did not read as JSON. Reset for + around it answerable for. Cleared by a value that did not read as + JSON, or one holding a character that could not be written, which + leaves the string it was being written into unclosed. Reset for each group by clear(), and kept apart from the flag above for the same reason as in the sister aggregate - one means a pair is missing, this one that the pairs are all there and still do not make - a document. + a document. Either way there is no object to return, so a group + this is cleared for is refused when it is asked for. + + A KEY holding such a character is neither: it goes in as the empty + key with its pair finished round it, so the object is a document + whose keys are not the ones that were asked for, and the note raised + where it happens is the whole of what is said about it. */ bool m_pairs_valid; /* @@ -1084,15 +1126,16 @@ class Item_func_json_objectagg : public Item_sum The group has been cut back to fit group_concat_max_len and nothing further goes into it. A later pair small enough to fit in what is left would go in behind the pair that did not fit, putting the - object out of the order the group was read in. Reset for each group - by clear(). + object out of the order the group was read in. Once this is set, + add() evaluates neither argument. Reset for each group by clear(). */ bool m_cut; /* Which row of this group is being added, counted over the rows that have a key to make a pair of - a row whose key is NULL is not one of - them and is not counted. Only the cut warning reads it, to say - where the group stopped. + them and is not counted. Neither is a row read after the cut, whose + key is not evaluated. Only the cut warning reads it, to say where + the group stopped, and it has been raised by then. Counted per group, where the sister aggregate's row_count runs on across the groups of a statement. Nothing here needs it to run on, @@ -1147,6 +1190,13 @@ class Item_func_json_objectagg : public Item_sum bool is_valid_json() const override { return m_marks.valid(); } bool is_nice_json() const override { return m_marks.nice(); } uint last_depth() const override { return m_marks.depth(); } + /* + A document or nothing, for the reason given where the sister + aggregate declares the same. + */ + bool is_valid_json_static() const override { return true; } + /* Asked for rather than read off, also as in the sister aggregate. */ + bool is_null() override { update_null_value(); return null_value; } void clear() override; bool add() override; void reset_field() override { DBUG_ASSERT(0); } // not used From 0573234e930a9008d2a59254c89c040bfe22f92d Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Fri, 14 Aug 2026 21:54:25 -0400 Subject: [PATCH 28/30] MDEV-40769 Write a character past the first plane as two escapes A value going into a document is escaped where the document's character set cannot carry the character as it stands. JSON spells such a character as the hex of its UTF-16 form, and a character outside the first plane is two UTF-16 units, so it is written as two escapes, one unit apiece. `json_escape()` wrote one escape and put the figures of both units after it. The second unit's four figures are then not an escape - they are four more characters of the string - and a reader takes the one escape, finds half a character, and stops. RFC 8259 has one production for this escape, `%x75 4HEXDIG`, and says in as many words that a character outside the Basic Multilingual Plane is represented as a twelve-character sequence encoding the UTF-16 surrogate pair. There is no form that carries eight figures. The reader in this same file already knew. `json_handle_esc()` reads four figures, and where they come to a high surrogate it requires a backslash and a `u` after them, answering `JE_SYN` where those are not there. So the server refused what it had just written: the five functions that write a value into a document they were handed returned NULL, and the note they gave named argument 1 at an offset past the end of argument 1, that offset being an offset into their own output. The writing now puts out the first unit's figures, and where a second unit came back it writes the backslash and the `u` that introduce it before writing its figures. The room was there all along - the reserve `st_append_escaped()` asks for is twelve characters apiece, its comment naming the two-escape form. ## The test `func_json_escape_pair` takes the last character of the first plane and three from beyond it - the first one past, an ordinary one, and the last there is - and puts each through the five functions, into a document written in a set that stops at the first plane and into one that carries the character as it stands. What comes back is read once as text, for the escapes themselves, and once as bytes, where a document that can carry the character would otherwise put the connection's own character set into the answer. `JSON_VALID` reads every one of them. Unfixed, the recorded results cannot be taken at all. The first statement answers NULL, and the third stops a debug build on the promise a composing function makes about its own answer, that promise being checked by reading the bytes it was made about. --- mysql-test/main/func_json_escape_pair.result | 62 ++++++++++++++++++++ mysql-test/main/func_json_escape_pair.test | 56 ++++++++++++++++++ strings/json_lib.c | 50 ++++++++++++---- 3 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 mysql-test/main/func_json_escape_pair.result create mode 100644 mysql-test/main/func_json_escape_pair.test diff --git a/mysql-test/main/func_json_escape_pair.result b/mysql-test/main/func_json_escape_pair.result new file mode 100644 index 0000000000000..1da874618d306 --- /dev/null +++ b/mysql-test/main/func_json_escape_pair.result @@ -0,0 +1,62 @@ +CREATE TABLE t1 (n VARCHAR(20) CHARACTER SET latin1, +t VARCHAR(20) CHARACTER SET utf8mb3, +w VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES ('{}', '{}', _utf8mb4 0xEFBFBF), +('{}', '{}', _utf8mb4 0xF0908080), +('{}', '{}', _utf8mb4 0xF09F9880), +('{}', '{}', _utf8mb4 0xF48FBFBF); +# +# The last character of the first plane takes one escape, and the +# three past it take two. +# +SELECT JSON_SET(n, '$.b', w) AS written, +JSON_VALID(JSON_SET(n, '$.b', w)) AS valid FROM t1; +written valid +{"b": "\uFFFF"} 1 +{"b": "\uD800\uDC00"} 1 +{"b": "\uD83D\uDE00"} 1 +{"b": "\uDBFF\uDFFF"} 1 +# +# The same into a document written in the other set that stops at +# the first plane. That set carries the first of the four as it +# stands, so that one is not escaped at all - read here as bytes, +# to keep the connection's own set out of the answer. +# +SELECT HEX(JSON_SET(t, '$.b', w)) AS bytes, +JSON_VALID(JSON_SET(t, '$.b', w)) AS valid FROM t1; +bytes valid +7B2262223A2022EFBFBF227D 1 +7B2262223A20225C75443830305C7544433030227D 1 +7B2262223A20225C75443833445C7544453030227D 1 +7B2262223A20225C75444246465C7544464646227D 1 +# +# The four other functions that write a value into a document they +# were handed answer the same way. +# +SELECT JSON_INSERT(n, '$.b', w) AS inserted, +JSON_REPLACE(JSON_SET(n, '$.b', 1), '$.b', w) AS replaced FROM t1; +inserted replaced +{"b": "\uFFFF"} {"b": "\uFFFF"} +{"b": "\uD800\uDC00"} {"b": "\uD800\uDC00"} +{"b": "\uD83D\uDE00"} {"b": "\uD83D\uDE00"} +{"b": "\uDBFF\uDFFF"} {"b": "\uDBFF\uDFFF"} +SELECT JSON_ARRAY_APPEND(CONCAT('[', 1, ']'), '$', w) AS appended, +JSON_ARRAY_INSERT(CONCAT('[', 1, ']'), '$[0]', w) AS array_inserted +FROM t1; +appended array_inserted +[1, "\uFFFF"] ["\uFFFF", 1] +[1, "\uD800\uDC00"] ["\uD800\uDC00", 1] +[1, "\uD83D\uDE00"] ["\uD83D\uDE00", 1] +[1, "\uDBFF\uDFFF"] ["\uDBFF\uDFFF", 1] +# +# What the writing put there, read as bytes. The escape is ASCII +# whatever the document is written in, so the two units stand out +# as two of them. +# +SELECT HEX(JSON_SET(n, '$.b', w)) AS bytes FROM t1; +bytes +7B2262223A20225C7546464646227D +7B2262223A20225C75443830305C7544433030227D +7B2262223A20225C75443833445C7544453030227D +7B2262223A20225C75444246465C7544464646227D +DROP TABLE t1; diff --git a/mysql-test/main/func_json_escape_pair.test b/mysql-test/main/func_json_escape_pair.test new file mode 100644 index 0000000000000..29d5f01d0caf1 --- /dev/null +++ b/mysql-test/main/func_json_escape_pair.test @@ -0,0 +1,56 @@ +# +# Writing out a character the document cannot carry. +# +# A character outside the first plane is two UTF-16 units, and JSON +# spells it as two escapes, one unit apiece. Anything shorter is not a +# document: a reader takes the first escape, looks for the second, and +# stops where it is not. +# +# Only a document whose character set cannot carry the character is +# written this way, so the functions that meet it are the ones that take +# their character set from the document they were handed and aggregate +# nothing with it. +# + +CREATE TABLE t1 (n VARCHAR(20) CHARACTER SET latin1, + t VARCHAR(20) CHARACTER SET utf8mb3, + w VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES ('{}', '{}', _utf8mb4 0xEFBFBF), + ('{}', '{}', _utf8mb4 0xF0908080), + ('{}', '{}', _utf8mb4 0xF09F9880), + ('{}', '{}', _utf8mb4 0xF48FBFBF); + +--echo # +--echo # The last character of the first plane takes one escape, and the +--echo # three past it take two. +--echo # +SELECT JSON_SET(n, '$.b', w) AS written, + JSON_VALID(JSON_SET(n, '$.b', w)) AS valid FROM t1; + +--echo # +--echo # The same into a document written in the other set that stops at +--echo # the first plane. That set carries the first of the four as it +--echo # stands, so that one is not escaped at all - read here as bytes, +--echo # to keep the connection's own set out of the answer. +--echo # +SELECT HEX(JSON_SET(t, '$.b', w)) AS bytes, + JSON_VALID(JSON_SET(t, '$.b', w)) AS valid FROM t1; + +--echo # +--echo # The four other functions that write a value into a document they +--echo # were handed answer the same way. +--echo # +SELECT JSON_INSERT(n, '$.b', w) AS inserted, + JSON_REPLACE(JSON_SET(n, '$.b', 1), '$.b', w) AS replaced FROM t1; +SELECT JSON_ARRAY_APPEND(CONCAT('[', 1, ']'), '$', w) AS appended, + JSON_ARRAY_INSERT(CONCAT('[', 1, ']'), '$[0]', w) AS array_inserted + FROM t1; + +--echo # +--echo # What the writing put there, read as bytes. The escape is ASCII +--echo # whatever the document is written in, so the two units stand out +--echo # as two of them. +--echo # +SELECT HEX(JSON_SET(n, '$.b', w)) AS bytes FROM t1; + +DROP TABLE t1; diff --git a/strings/json_lib.c b/strings/json_lib.c index 5fbfbc4cbeeac..c0e718754f624 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -1821,7 +1821,7 @@ int json_escape(CHARSET_INFO *str_cs, { /* We have to use /uXXXX escaping. */ uchar utf16buf[4]; - uchar code_str[8]; + uchar code_str[4]; int u_len= my_uni_utf16(0, c_chr, utf16buf, utf16buf + 4); code_str[0]= hexconv[utf16buf[0] >> 4]; @@ -1829,22 +1829,48 @@ int json_escape(CHARSET_INFO *str_cs, code_str[2]= hexconv[utf16buf[1] >> 4]; code_str[3]= hexconv[utf16buf[1] & 15]; - if (u_len > 2) + if ((c_len= json_append_ascii(json_cs, json, json_end, + code_str, code_str + 4)) <= 0) { - code_str[4]= hexconv[utf16buf[2] >> 4]; - code_str[5]= hexconv[utf16buf[2] & 15]; - code_str[6]= hexconv[utf16buf[3] >> 4]; - code_str[7]= hexconv[utf16buf[3] & 15]; + /* JSON buffer is depleted. */ + return JSON_ERROR_OUT_OF_SPACE; } - - if ((c_len= json_append_ascii(json_cs, json, json_end, - code_str, code_str+u_len*2)) > 0) + json+= c_len; + + if (u_len > 2) { + /* + A character outside the first plane is two UTF-16 units, and + each unit is written as an escape of its own: a reader takes + \uXXXX one at a time and puts the pair back together itself. + The second unit's figures after the first escape and nothing + to introduce them is not a second escape - it is four more + characters of the string, and the reader stops at the one + escape with half a character in hand. + */ + code_str[0]= hexconv[utf16buf[2] >> 4]; + code_str[1]= hexconv[utf16buf[2] & 15]; + code_str[2]= hexconv[utf16buf[3] >> 4]; + code_str[3]= hexconv[utf16buf[3] & 15]; + + if ((c_len= my_ci_wc_mb(json_cs, '\\', json, json_end)) <= 0 || + (c_len= my_ci_wc_mb(json_cs, 'u', json+= c_len, + json_end)) <= 0) + { + /* JSON buffer is depleted. */ + return JSON_ERROR_OUT_OF_SPACE; + } + json+= c_len; + + if ((c_len= json_append_ascii(json_cs, json, json_end, + code_str, code_str + 4)) <= 0) + { + /* JSON buffer is depleted. */ + return JSON_ERROR_OUT_OF_SPACE; + } json+= c_len; - continue; } - /* JSON buffer is depleted. */ - return JSON_ERROR_OUT_OF_SPACE; + continue; } } else /* c_len == 0, an illegal symbol. */ From 205e5d57520653151e99e4c3655ea749f9ca4735 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Sat, 15 Aug 2026 04:38:59 -0400 Subject: [PATCH 29/30] Pass on what an argument said when only its value is returned `CASE` and its abbreviations - `COALESCE`, `IFNULL`, `IF`, `NVL2`, `NULLIF` and `DECODE` - a scalar subquery, and the expression cache that stands in front of one, all return a value that one of their arguments made, byte for byte. None of them said anything about it, so a document a JSON function had already read and attested to arrived at the next JSON function with nothing said about it, and was read again to find out what was already known. Each of them now writes down which argument the value came from while it is reading it, and puts the three questions about that value to that argument. The argument cannot be found again afterwards: `COALESCE` and `IFNULL` find theirs by reading it, and the rest are told which one by a condition that may hold a subquery or a stored program, so asking a second time would run those a second time. The same read carries a caller's request for a document down to the argument. Without it an argument that returns a view of a value it keeps - a stored program's variable is one - handed that view out where a copy was owed, the request having stopped at the item in front of it. `is_valid_json_static()` is answered as well, a temporary table being written before any row exists and taking what can be said in advance. That is the conjunction over the arguments that can be returned, which for a `CASE` is the `THEN` arguments and the `ELSE` and never the `WHEN` conditions: `reorder_args()` has already put the conditions first, so the rest are the tail of the argument list. `Item_cache_str` keeps a copy of what it was made over, and now takes that item's answers beside the copy the way `Item_copy_string` does. A debug build holds a passed-on answer against the bytes it is about at the moment both are in hand, which is the reading `Json_result_marks::set()` makes where an answer is first given. New test `func_json_pass_through_scan_count`, which counts the readings each shape costs. No recorded answer moves: a reading that is skipped could only have found out what was already known. --- .../func_json_pass_through_scan_count.result | 307 ++++++++++++++++++ .../func_json_pass_through_scan_count.test | 238 ++++++++++++++ sql/item.cc | 24 +- sql/item.h | 105 +++++- sql/item_cmpfunc.cc | 10 +- sql/item_cmpfunc.h | 51 ++- sql/item_func.h | 50 +++ sql/item_jsonfunc.cc | 44 +++ sql/item_subselect.cc | 30 ++ sql/item_subselect.h | 8 + 10 files changed, 853 insertions(+), 14 deletions(-) create mode 100644 mysql-test/main/func_json_pass_through_scan_count.result create mode 100644 mysql-test/main/func_json_pass_through_scan_count.test diff --git a/mysql-test/main/func_json_pass_through_scan_count.result b/mysql-test/main/func_json_pass_through_scan_count.result new file mode 100644 index 0000000000000..0e8d54406e67a --- /dev/null +++ b/mysql-test/main/func_json_pass_through_scan_count.result @@ -0,0 +1,307 @@ +# +# How many times a document is read to find out that it is one when +# it reaches a JSON function through an expression that returns a +# value one of its arguments made: CASE and its abbreviations, and a +# scalar subquery. +# +# No query can see the difference - the reading could only have found +# out what was already known - so nothing but Json_scans says whether +# it happened. The counts below are the record of it. A count that +# goes up is a reading that came back. +# +# A debug build reads values back to check what was claimed about +# them, and none of those readings are counted: they are the debug +# build's work and not the server's. A count of 0 below therefore +# means nothing read, not that a reading went uncounted. +# +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +# +# 1. TWO rows through a function that attests to what it returns, +# spliced with nothing in between. JSON_EXTRACT reads its input +# once a row; the splice reads nothing. Every count below is held +# against this one. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 2. The same value returned by a searched CASE. What the CASE +# returns is what the THEN argument returned, so what that argument +# said about it is said about it here. +# +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN JSON_EXTRACT(j, '$') +ELSE JSON_ARRAY() END) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 3. A simple CASE, which picks its argument by comparing rather +# than by testing, and the ELSE rather than a THEN. +# +FLUSH STATUS; +SELECT JSON_ARRAY(CASE id WHEN 0 THEN JSON_ARRAY() +ELSE JSON_EXTRACT(j, '$') END) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 4. COALESCE and IFNULL, which find their argument by reading it. +# +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 5. IFNULL again, this time returning its second argument. +# +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(NULL, '$'), JSON_EXTRACT(j, '$'))) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 6. IF and NVL2, which are told which argument to return by a +# switch. The switch is not what either returns and is not asked. +# +FLUSH STATUS; +SELECT JSON_ARRAY(IF(id > 0, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(NVL2(id, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 7. NULLIF, which compares its first argument and then returns it. +# The function that makes the document therefore runs more than once +# a row and each run reads its own input, which is why this count is +# the highest here. Those readings belong to the argument; the one +# the splice was making is the one that goes. +# +FLUSH STATUS; +SELECT JSON_ARRAY(NULLIF(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 8. A scalar subquery, whose value is kept in a cache and read out +# of it. What the cache keeps is a copy of what the item under it +# returned, taken while the two were about the same characters. +# +# The subquery reads a table, so that it is still a subquery by the +# time it is run: one over no table at all is replaced by the +# expression inside it and there is nothing left to pass anything on. +# A correlated one is also run behind an expression cache, which is +# another value passed on and has to say the same thing. +# +# First with nothing spliced, which is what the subquery costs on +# its own, and then with the splice on top of it. +# +FLUSH STATUS; +SELECT (SELECT JSON_EXTRACT(x.j, '$') FROM t1 x +WHERE x.id = t1.id) AS r FROM t1 ORDER BY id; +r +{"a": 1} +{"b": 2} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_EXTRACT(x.j, '$') FROM t1 x +WHERE x.id = t1.id)) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# And with a producer that reads nothing of its own, so that the +# only reading left to count would be the splice's. +# +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_OBJECT('a', y.id) FROM t1 y +WHERE y.id = t1.id)) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 9. Nested one inside another, so that the value is passed on twice +# before it is spliced. +# +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(IF(id > 0, JSON_EXTRACT(j, '$'), NULL), +JSON_ARRAY())) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 10. A value nothing attests to, passed through the same way. The +# column says nothing about itself, so the splice reads it - and +# passing it through changes neither the answer nor the count. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +r +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN j ELSE JSON_ARRAY() END) AS r +FROM t1 ORDER BY id; +r +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 11. Through a temporary table, which is written before any row +# exists and so cannot be told what a row turned out to be. What +# can be said in advance is what every argument that can be returned +# says, and both of these do. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) +ELSE JSON_ARRAY() END AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# The same for a subquery, whose one argument is the select list of +# the query inside it. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, (SELECT JSON_OBJECT('a', y.id) FROM t1 y +WHERE y.id = t1.id) AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 12. The same with one argument that says nothing. One is enough +# to leave the column with nothing said about it, and every row is +# read on the way out - including the rows the other argument made. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) +ELSE j END AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 13. A GROUP BY, which writes a temporary table of its own, with +# the same expression in the select list. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, COALESCE(JSON_OBJECT('a', id), JSON_ARRAY()) AS x +FROM t1 GROUP BY id) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 14. The WHEN expressions are not among the arguments that can be +# returned, so what they are made of settles nothing. A CASE whose +# conditions read a column still attests for its THEN and ELSE. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, CASE WHEN j LIKE '%a%' THEN JSON_OBJECT('a', id) +ELSE JSON_OBJECT('b', id) END AS x FROM t1) d +ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +SET optimizer_switch=DEFAULT; +# +# 15. DECODE, which picks its argument the way a simple CASE does. +# +SET sql_mode=ORACLE; +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +FLUSH STATUS; +SELECT JSON_ARRAY(DECODE(id, 0, JSON_ARRAY(), JSON_EXTRACT(j, '$'))) AS r +FROM t2 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t2; +SET sql_mode=DEFAULT; diff --git a/mysql-test/main/func_json_pass_through_scan_count.test b/mysql-test/main/func_json_pass_through_scan_count.test new file mode 100644 index 0000000000000..b1534beb325e9 --- /dev/null +++ b/mysql-test/main/func_json_pass_through_scan_count.test @@ -0,0 +1,238 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a document is read to find out that it is one when +--echo # it reaches a JSON function through an expression that returns a +--echo # value one of its arguments made: CASE and its abbreviations, and a +--echo # scalar subquery. +--echo # +--echo # No query can see the difference - the reading could only have found +--echo # out what was already known - so nothing but Json_scans says whether +--echo # it happened. The counts below are the record of it. A count that +--echo # goes up is a reading that came back. +--echo # +--echo # A debug build reads values back to check what was claimed about +--echo # them, and none of those readings are counted: they are the debug +--echo # build's work and not the server's. A count of 0 below therefore +--echo # means nothing read, not that a reading went uncounted. +--echo # + +SET NAMES utf8mb4; + +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol +SET optimizer_switch='derived_merge=off'; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +--echo # +--echo # 1. TWO rows through a function that attests to what it returns, +--echo # spliced with nothing in between. JSON_EXTRACT reads its input +--echo # once a row; the splice reads nothing. Every count below is held +--echo # against this one. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 2. The same value returned by a searched CASE. What the CASE +--echo # returns is what the THEN argument returned, so what that argument +--echo # said about it is said about it here. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN JSON_EXTRACT(j, '$') + ELSE JSON_ARRAY() END) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 3. A simple CASE, which picks its argument by comparing rather +--echo # than by testing, and the ELSE rather than a THEN. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(CASE id WHEN 0 THEN JSON_ARRAY() + ELSE JSON_EXTRACT(j, '$') END) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 4. COALESCE and IFNULL, which find their argument by reading it. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 5. IFNULL again, this time returning its second argument. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(NULL, '$'), JSON_EXTRACT(j, '$'))) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 6. IF and NVL2, which are told which argument to return by a +--echo # switch. The switch is not what either returns and is not asked. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(IF(id > 0, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY(NVL2(id, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 7. NULLIF, which compares its first argument and then returns it. +--echo # The function that makes the document therefore runs more than once +--echo # a row and each run reads its own input, which is why this count is +--echo # the highest here. Those readings belong to the argument; the one +--echo # the splice was making is the one that goes. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(NULLIF(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 8. A scalar subquery, whose value is kept in a cache and read out +--echo # of it. What the cache keeps is a copy of what the item under it +--echo # returned, taken while the two were about the same characters. +--echo # +--echo # The subquery reads a table, so that it is still a subquery by the +--echo # time it is run: one over no table at all is replaced by the +--echo # expression inside it and there is nothing left to pass anything on. +--echo # A correlated one is also run behind an expression cache, which is +--echo # another value passed on and has to say the same thing. +--echo # +--echo # First with nothing spliced, which is what the subquery costs on +--echo # its own, and then with the splice on top of it. +--echo # +FLUSH STATUS; +SELECT (SELECT JSON_EXTRACT(x.j, '$') FROM t1 x + WHERE x.id = t1.id) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_EXTRACT(x.j, '$') FROM t1 x + WHERE x.id = t1.id)) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And with a producer that reads nothing of its own, so that the +--echo # only reading left to count would be the splice's. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_OBJECT('a', y.id) FROM t1 y + WHERE y.id = t1.id)) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 9. Nested one inside another, so that the value is passed on twice +--echo # before it is spliced. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(IF(id > 0, JSON_EXTRACT(j, '$'), NULL), + JSON_ARRAY())) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 10. A value nothing attests to, passed through the same way. The +--echo # column says nothing about itself, so the splice reads it - and +--echo # passing it through changes neither the answer nor the count. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN j ELSE JSON_ARRAY() END) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 11. Through a temporary table, which is written before any row +--echo # exists and so cannot be told what a row turned out to be. What +--echo # can be said in advance is what every argument that can be returned +--echo # says, and both of these do. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) + ELSE JSON_ARRAY() END AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same for a subquery, whose one argument is the select list of +--echo # the query inside it. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, (SELECT JSON_OBJECT('a', y.id) FROM t1 y + WHERE y.id = t1.id) AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 12. The same with one argument that says nothing. One is enough +--echo # to leave the column with nothing said about it, and every row is +--echo # read on the way out - including the rows the other argument made. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) + ELSE j END AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 13. A GROUP BY, which writes a temporary table of its own, with +--echo # the same expression in the select list. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, COALESCE(JSON_OBJECT('a', id), JSON_ARRAY()) AS x + FROM t1 GROUP BY id) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 14. The WHEN expressions are not among the arguments that can be +--echo # returned, so what they are made of settles nothing. A CASE whose +--echo # conditions read a column still attests for its THEN and ELSE. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, CASE WHEN j LIKE '%a%' THEN JSON_OBJECT('a', id) + ELSE JSON_OBJECT('b', id) END AS x FROM t1) d + ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t1; +SET optimizer_switch=DEFAULT; +--enable_ps2_protocol + +--echo # +--echo # 15. DECODE, which picks its argument the way a simple CASE does. +--echo # +--disable_ps2_protocol +SET sql_mode=ORACLE; +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +FLUSH STATUS; +SELECT JSON_ARRAY(DECODE(id, 0, JSON_ARRAY(), JSON_EXTRACT(j, '$'))) AS r + FROM t2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t2; +SET sql_mode=DEFAULT; +--enable_ps2_protocol diff --git a/sql/item.cc b/sql/item.cc index cece67b607b30..2d1b5c19b924a 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -9760,21 +9760,21 @@ String *Item_cache_wrapper::val_str(String* str) DBUG_ENTER("Item_cache_wrapper::val_str"); if (!expr_cache) { - String *tmp= orig_item->val_str(str); + String *tmp= m_value_arm.read(orig_item, str); null_value= orig_item->null_value; DBUG_RETURN(tmp); } if ((cached_value= check_cache())) { - String *tmp= cached_value->val_str(str); + String *tmp= m_value_arm.read(cached_value, str); null_value= cached_value->null_value; DBUG_RETURN(tmp); } cache(); if ((null_value= expr_value->null_value)) DBUG_RETURN(NULL); - DBUG_RETURN(expr_value->val_str(str)); + DBUG_RETURN(m_value_arm.read(expr_value, str)); } @@ -11337,11 +11337,13 @@ Item *Item_cache_decimal::convert_to_basic_const_item(THD *thd) bool Item_cache_str::cache_value() { + bool kept= true; if (!example) { DBUG_ASSERT(value_cached == FALSE); return FALSE; } + m_marks.clear(); value_cached= TRUE; THD *thd= current_thd; const bool err= thd->is_error(); @@ -11361,12 +11363,24 @@ bool Item_cache_str::cache_value() (select c from t1 where a=t2.a) from t2; */ - value_buff.copy(*value); + kept= !value_buff.copy(*value); value= &value_buff; } else - value_buff.copy(); + kept= !value_buff.copy(); value_buff.mark_as_const(); + /* + The item was evaluated just above and nothing has evaluated it since, + so this is the one moment its answers are about the bytes being kept + here. They are asked of its result side because str_result() is what + was read. String::copy() takes the character set along with the + bytes, so the characters answered about are the characters kept - a + copy that could not get the room keeps nothing this can attest to. + */ + if (value && kept) + m_marks.set(value, example->is_valid_json_result(), + example->is_nice_json_result(), + example->last_depth_result()); return TRUE; } diff --git a/sql/item.h b/sql/item.h index 539062f8a69e6..82290ed2a2703 100644 --- a/sql/item.h +++ b/sql/item.h @@ -935,6 +935,48 @@ class Json_result_marks }; +/* + What an item that returns a value one of its arguments made needs in + order to pass on what that argument said about it. Such an item copies + the bytes out and adds nothing of its own to them, so the three + questions above are answered by putting them to the argument that made + the value. + + Which argument that was is written down while the value is being read. + It cannot be worked out afterwards: some of these items find their + argument by reading it, and the rest are told which one by a condition + that may hold a subquery or a stored program, so asking a second time + would run those a second time. +*/ +class Json_value_arm +{ + Item *m_arm; + /* + Whether the value is being read for a caller that asked for a + document. An argument that keeps a value of its own returns a view + of it from val_str() and a copy from val_json(), and an item that + passes values on has to pass that request on with them. + */ + bool m_read_as_json; +public: + Json_value_arm() : m_arm(NULL), m_read_as_json(false) {} + /* + A copy is a new item that has not been evaluated, so it has read no + argument and there is nothing for it to pass on. Json_result_marks + is not copied for the same reason. + */ + Json_value_arm(const Json_value_arm &) + : m_arm(NULL), m_read_as_json(false) {} + /* Read the value off the argument that is returning it. */ + String *read(Item *arm, String *str); + /* Have the reads above made with val_json() rather than val_str(). */ + void want_json(bool want) { m_read_as_json= want; } + bool valid() const; + bool nice() const; + uint depth() const; +}; + + class Item :public Value_source, public Type_all_attributes { @@ -6688,6 +6730,13 @@ class Item_cache_wrapper :public Item_result_field */ Item_cache *expr_value; + /* + What this item returns is what the expression returned or what the + cache kept of it, and it adds nothing of its own to either. See + Json_value_arm. + */ + Json_value_arm m_value_arm; + List parameters; Item *check_cache(); @@ -6711,6 +6760,30 @@ class Item_cache_wrapper :public Item_result_field double val_real() override; longlong val_int() override; String *val_str(String* tmp) override; + /* + Asked for a document, which is a request whatever returns the value + has to see - see Json_value_arm. val_str() below is what reads it. + */ + String *val_json(String* tmp) override + { + m_value_arm.want_json(true); + String *res= val_str(tmp); + m_value_arm.want_json(false); + return res; + } + bool is_valid_json() const override + { return !null_value && m_value_arm.valid(); } + bool is_nice_json() const override + { return !null_value && m_value_arm.nice(); } + uint last_depth() const override + { return null_value ? JSON_DEPTH_UNKNOWN : m_value_arm.depth(); } + /* + A property of the expression rather than of any value it has + produced. The cache keeps every character of what the expression + returns, so what holds of the expression holds of the cache. + */ + bool is_valid_json_static() const override + { return orig_item->is_valid_json_static(); } bool val_native(THD *thd, Native *to) override; my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; @@ -8588,7 +8661,18 @@ class Item_cache_str: public Item_cache char buffer[STRING_BUFFER_USUAL_SIZE]; String *value, value_buff; bool is_varbinary; - + /* + What the item this was cached from said about the value at the + moment it was cached, which is the moment the two are about the same + bytes. Asking that item later would pair an answer about whatever + it has gone on to produce with the copy kept here - the copy being + the point of this item, and being what val_str() below returns. + + Taken beside the copy exactly as Item_copy_string does over the value + it keeps. + */ + Json_result_marks m_marks; + public: Item_cache_str(THD *thd, const Item *item): Item_cache(thd, item->type_handler()), value(0), @@ -8608,6 +8692,25 @@ class Item_cache_str: public Item_cache int save_in_field(Field *field, bool no_conversions) override; bool cache_value() override; Item *convert_to_basic_const_item(THD *thd) override; + /* + Asked next to val_str(), which returns the bytes these are about. + A cache emptied by clear() or set_null() keeps no value at all, and + what was said about the one it used to keep is not said about that. + */ + bool is_valid_json() const override + { return !null_value && m_marks.valid(); } + bool is_nice_json() const override + { return !null_value && m_marks.nice(); } + uint last_depth() const override + { return null_value ? JSON_DEPTH_UNKNOWN : m_marks.depth(); } + /* + A property of the item being cached rather than of any value it has + produced, so it is answered whether or not anything has been cached + yet. The copy keeps every character of what that item returns, so + what holds of the one holds of the other. + */ + bool is_valid_json_static() const override + { return example && example->is_valid_json_static(); } protected: Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index 3c0878a8f06f5..dce85df48808e 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -2469,14 +2469,14 @@ String * Item_func_ifnull::str_op(String *str) { DBUG_ASSERT(fixed()); - String *res =args[0]->val_str(str); + String *res = read_arm(args[0], str); if (!args[0]->null_value) { null_value=0; res->set_charset(collation.collation); return res; } - res=args[1]->val_str(str); + res= read_arm(args[1], str); if ((null_value=args[1]->null_value)) return 0; res->set_charset(collation.collation); @@ -2975,7 +2975,7 @@ Item_func_nullif::str_op(String *str) null_value=1; return 0; } - res= args[2]->val_str(str); + res= read_arm(args[2], str); null_value= args[2]->null_value; return res; } @@ -3126,7 +3126,7 @@ String *Item_func_case::str_op(String *str) return 0; } null_value= 0; - if (!(res=item->val_str(str))) + if (!(res= read_arm(item, str))) null_value= 1; return res; } @@ -3516,7 +3516,7 @@ String *Item_func_coalesce::str_op(String *str) for (uint i=0 ; i < arg_count ; i++) { String *res; - if ((res=args[i]->val_str(str))) + if ((res= read_arm(args[i], str))) return res; } null_value=1; diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index a661c695068df..25530bb66f3d6 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1218,6 +1218,9 @@ class Item_func_coalesce :public Item_func_case_expression return name; } table_map not_null_tables() const override { return 0; } + /* Any of the arguments can be the one whose value is returned. */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(0, arg_count); } protected: Item *shallow_copy(THD *thd) const override @@ -1305,6 +1308,9 @@ class Item_func_ifnull :public Item_func_case_abbreviation2 } table_map not_null_tables() const override { return 0; } + /* Either argument can be the one whose value is returned. */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(0, 2); } protected: Item *shallow_copy(THD *thd) const override @@ -1351,15 +1357,30 @@ class Item_func_case_abbreviation2_switch: public Item_func_case_abbreviation2 { return val_decimal_from_item(find_item(), decimal_value); } + /* + Item::val_str_from_item(), reading the argument through read_arm() so + that which argument it was is written down and a caller that asked + for a document has its request reach it. + */ String *str_op(String *str) override { - return val_str_from_item(find_item(), str); + DBUG_ASSERT(fixed()); + Item *arm= find_item(); + String *res= read_arm(arm, str); + if (res) + res->set_charset(collation.collation); + if ((null_value= arm->null_value)) + res= NULL; + return res; } bool native_op(THD *thd, Native *to) override { return val_native_with_conversion_from_item(thd, find_item(), to, type_handler()); } + /* args[0] is the switch; the value comes from one of the other two. */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(1, 2); } }; @@ -1489,6 +1510,13 @@ class Item_func_nullif :public Item_func_case_expression void update_used_tables() override; table_map not_null_tables() const override { return 0; } bool is_null() override; + /* + The value is always the right "a", args[2] - see the comment on the + constructor. The left "a" is only ever compared, and args[1] is only + ever compared against. + */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(2, 1); } Item* propagate_equal_fields(THD *thd, const Context &ctx, COND_EQUAL *cond) override { @@ -2392,6 +2420,7 @@ class Item_func_case :public Item_func_case_expression DTCollation cmp_collation; bool aggregate_then_and_else_arguments(THD *thd, uint count); virtual Item **else_expr_addr() const= 0; + virtual uint when_count() const= 0; virtual Item *find_item()= 0; inline void print_when_then_arguments(String *str, enum_query_type query_type, @@ -2419,6 +2448,22 @@ class Item_func_case :public Item_func_case_expression } CHARSET_INFO *compare_collation() const { return cmp_collation.collation; } bool need_parentheses_in_default() override { return true; } + /* + The value comes from one of the THEN arguments or from the ELSE, and + reorder_args() has put exactly those last: the optional CASE + expression comes first, then the WHEN expressions, then the THEN + expressions and the ELSE. So the arguments that can be returned are + the last one per WHEN plus the ELSE where there is one. + + Asking the WHEN expressions too would answer no for every CASE ever + written - what they return is a condition, and this item never + returns it. + */ + bool is_valid_json_static() const override + { + uint count= when_count() + MY_TEST(else_expr_addr()); + return args_are_valid_json_static(arg_count - count, count); + } }; @@ -2431,7 +2476,7 @@ class Item_func_case :public Item_func_case_expression */ class Item_func_case_searched: public Item_func_case { - uint when_count() const { return arg_count / 2; } + uint when_count() const override { return arg_count / 2; } bool with_else() const { return arg_count % 2; } Item **else_expr_addr() const override { return with_else() ? &args[arg_count - 1] : 0; } @@ -2476,7 +2521,7 @@ class Item_func_case_simple: public Item_func_case, { protected: uint m_found_types; - uint when_count() const { return (arg_count - 1) / 2; } + uint when_count() const override { return (arg_count - 1) / 2; } bool with_else() const { return arg_count % 2 == 0; } Item **else_expr_addr() const override { return with_else() ? &args[arg_count - 1] : 0; } diff --git a/sql/item_func.h b/sql/item_func.h index bbb95e61d4f3a..37a4890c3fc3b 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1092,6 +1092,31 @@ class Item_func_hybrid_field_type: public Item_hybrid_func */ class Item_func_case_expression: public Item_func_hybrid_field_type { +protected: + /* + What this item returns is what one of its arguments returned, byte + for byte, so the questions below are put to that argument. Every + path that returns a string value reads it through here. + */ + Json_value_arm m_value_arm; + String *read_arm(Item *arm, String *str) + { return m_value_arm.read(arm, str); } + + /* + Whether every one of 'count' arguments starting at 'first' returns a + document or nothing at all each time it is evaluated. One of them + will be the one this item returns and nothing here knows which, so + what can be said before any of them is evaluated is only what they + all say. + */ + bool args_are_valid_json_static(uint first, uint count) const + { + for (uint i= first; i < first + count; i++) + if (!args[i]->is_valid_json_static()) + return false; + return true; + } + public: Item_func_case_expression(THD *thd) :Item_func_hybrid_field_type(thd) @@ -1109,6 +1134,31 @@ class Item_func_case_expression: public Item_func_hybrid_field_type Item_func_hybrid_field_type(thd, list) { } bool find_not_null_fields(table_map allowed) override { return false; } + + /* + Asked for a document rather than for a string, which is a request the + argument has to see - see Json_value_arm. val_str() is what reaches + str_op(), and str_op() is what reads the argument. + */ + String *val_json(String *str) override + { + m_value_arm.want_json(true); + String *res= val_str(str); + m_value_arm.want_json(false); + return res; + } + + /* + Nothing is said where the evaluation produced no value: NULL is not a + document, and the argument written down would then be whichever one + was read last rather than one this item returned. + */ + bool is_valid_json() const override + { return !null_value && m_value_arm.valid(); } + bool is_nice_json() const override + { return !null_value && m_value_arm.nice(); } + uint last_depth() const override + { return null_value ? JSON_DEPTH_UNKNOWN : m_value_arm.depth(); } }; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index be4ce451b3143..ff34ccd6015ae 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -1237,6 +1237,50 @@ void Json_result_marks::set(const String *str, bool valid, bool nice, } +String *Json_value_arm::read(Item *arm, String *str) +{ + m_arm= arm; + String *res= m_read_as_json ? arm->val_json(str) : arm->val_str(str); + /* + An item that passes a value on passes the argument's answers on with + it, unread. This is the one moment both are in hand, so a debug + build holds the one against the other here, for the reason given at + Json_result_marks::set(): nothing else ever would. + + The reading costs nothing where there is no answer to hold, which is + every argument that is not a document. + */ + DBUG_ASSERT(!res || !arm->is_valid_json() || + json_value_reads_as_document(res)); + DBUG_ASSERT(!res || !arm->is_nice_json() || json_value_is_nice(res)); + return res; +} + + +/* + The three below are the argument's own answers, passed straight on. + An item that has read no argument yet says what an item with nothing + to say says. +*/ + +bool Json_value_arm::valid() const +{ + return m_arm && m_arm->is_valid_json(); +} + + +bool Json_value_arm::nice() const +{ + return m_arm && m_arm->is_nice_json(); +} + + +uint Json_value_arm::depth() const +{ + return m_arm ? m_arm->last_depth() : JSON_DEPTH_UNKNOWN; +} + + #define report_json_error(js, je, n_param) \ report_json_error_ex(js->ptr(), je, func_name(), n_param, \ Sql_condition::WARN_LEVEL_WARN) diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 0d4c2e5361a1a..070cc75a0db8d 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1569,6 +1569,36 @@ bool Item_singlerow_subselect::val_native(THD *thd, Native *to) } +/* + A row subquery has more than one cache and none of them is the value, + which is what 'value' is NULL for; a subquery that has not been fixed + yet has none at all. +*/ + +bool Item_singlerow_subselect::is_valid_json() const +{ + return value && value->is_valid_json(); +} + + +bool Item_singlerow_subselect::is_nice_json() const +{ + return value && value->is_nice_json(); +} + + +uint Item_singlerow_subselect::last_depth() const +{ + return value ? value->last_depth() : JSON_DEPTH_UNKNOWN; +} + + +bool Item_singlerow_subselect::is_valid_json_static() const +{ + return value && value->is_valid_json_static(); +} + + my_decimal *Item_singlerow_subselect::val_decimal(my_decimal *decimal_value) { DBUG_ASSERT(fixed()); diff --git a/sql/item_subselect.h b/sql/item_subselect.h index ef2fa8b1fe168..9eb3fbf0707b2 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -323,6 +323,14 @@ class Item_singlerow_subselect :public Item_subselect my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; + /* + val_str() above returns what the cache holds, so the cache is what is + asked. Out of line because Item_cache is not a complete type here. + */ + bool is_valid_json() const override; + bool is_nice_json() const override; + uint last_depth() const override; + bool is_valid_json_static() const override; const Type_handler *type_handler() const override; bool fix_length_and_dec() override; From 304318b1f125f42b526e18639375091c4ad38e42 Mon Sep 17 00:00:00 2001 From: Arcadiy Ivanov Date: Sat, 15 Aug 2026 14:22:43 -0400 Subject: [PATCH 30/30] Ask the character-set wrapper what it wraps, instead of casting `is_json_type()` looks through the wrapper that aggregating character sets puts round whichever argument has to move, and it finds that wrapper with a `dynamic_cast`. A `dynamic_cast` that fails is the expensive one. It walks the base classes of whatever it was handed and cannot stop early, absence being provable only by exhausting the graph. `Item` derives from two classes, so every item that reaches this is walked as the multiple-inheritance case rather than the single-inheritance one. The answer is no for every argument that is not a document, and the question is put to every argument there is: once per value written into a container by `append_json_value()`, and once per argument sized by `json_value_reserve()`. So the item is asked what it is put round instead of being made to prove what it is. `Item::conv_charset_arg()` returns the wrapper's argument from the wrapper and NULL from everything else, which is the whole of what the look through needs, and it is one virtual call. A `Functype` of the wrapper's own would have been one comparison too, and it would have been read by more than the one caller asking. `Functype` is dispatched on by engines outside the tree - two of the values carry a comment naming which ones - and several of them take `UNKNOWN_FUNC` to mean "printed by name". Spider does: it writes `convert(` in front of the argument under that label, and the ` USING` clause after it under a test of its own, and a wrapper that stopped answering `UNKNOWN_FUNC` fell out of both. What reached the data node was the argument with the conversion taken off it, select `pkey`,`txt` from `auto_test_remote`.`tbl_a` where ((`txt`) = 'beta') which compares the bytes as they stand and can answer a different row. Nothing says so: the conversion is dropped from the statement rather than refused. In the tree that is repairable. The engines outside it are not reachable, and there is no reason to think they were written differently - this one was not. A virtual is read by nobody who was not asking, so none of them observes anything. This is measurable rather than tidiness. The failing cast is a few hundred cycles, and the workload that prompted the look - triggers building a digest document for every row written - puts the question once per value in it. Most of what it costs is work the code did before any of this branch, `append_json_value()` having asked the same question the same way for as long as it has been asking it. `charset_conv_pushdown` reads the statement the data node was sent out of its general log, under both of the handlers the suite runs, so the clause that can go missing is written down where a later change to what the wrapper answers has to walk past it. No result moves, and the statement Spider sends is the one it sent before. --- sql/item.h | 37 +++++++++ sql/item_jsonfunc.cc | 14 +++- sql/item_strfunc.h | 7 ++ .../include/charset_conv_pushdown_deinit.inc | 11 +++ .../include/charset_conv_pushdown_init.inc | 23 ++++++ ...arset_conv_pushdown,group_by_handler.rdiff | 11 +++ .../bugfix/r/charset_conv_pushdown.result | 63 +++++++++++++++ .../spider/bugfix/t/charset_conv_pushdown.cnf | 3 + .../bugfix/t/charset_conv_pushdown.test | 76 +++++++++++++++++++ 9 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_deinit.inc create mode 100644 storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_init.inc create mode 100644 storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown,group_by_handler.rdiff create mode 100644 storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.cnf create mode 100644 storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.test diff --git a/sql/item.h b/sql/item.h index 82290ed2a2703..e4432ecece8dd 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1941,6 +1941,43 @@ class Item :public Value_source, virtual bool is_nice_json_result() const { return is_nice_json(); } virtual uint last_depth_result() const { return last_depth(); } + /* + The argument of the wrapper that aggregating character sets puts + round an item whose value has to move to another one, and that + CONVERT(x USING cs) is written as, or NULL from every other item - + which is to say, this asks "are you that wrapper, and if so what + are you put round". + + It is here so that a caller that has to look through the wrapper to + reach the item underneath can be told, instead of working it out + from the object it was handed. is_json_type() is that caller: the + wrapper round a document is put there for the character set and + leaves a document, so a look that stopped at it would take the + documents moved between character sets for values that have to be + quoted. It is asked once per value written into a container and + once per argument sized, which between them is nearly every + argument the JSON functions are given. + + The two ways of working it out from the object are both worse. A + dynamic_cast that fails walks the base classes of what it was + handed and cannot stop early, absence being provable only by + exhausting the graph, and Item derives from two classes, so every + item that reaches it is walked as the multiple-inheritance case; + no is the answer for all but a few. A Functype of the wrapper's + own would be one comparison, but Functype is dispatched on by + engines outside the tree, several of which take UNKNOWN_FUNC to + mean "printed by name" - a wrapper that stopped answering it would + fall out of their dispatch, and what they build would lose the + conversion rather than refuse it. A virtual answers the one + question that is being asked and is read by nothing else. + + NULL is read as "not that wrapper, nothing to look through", never + as an error, and anything that comes to derive from the wrapper + inherits the answer - which is what the cast this replaces would + have said about it too. + */ + virtual Item *conv_charset_arg() const { return NULL; } + /* Return decimal representation of item with fixed point. diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index ff34ccd6015ae..2b859f651e8c3 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -3109,10 +3109,18 @@ bool is_json_type(const Item *item) { if (Type_handler_json_common::is_json_type_handler(item->type_handler())) return true; - const Item_func_conv_charset *func; - if (!(func= dynamic_cast(item->real_item()))) + /* + The wrapper is asked what it is put round rather than made to + prove what it is, which is one virtual call where a dynamic_cast + walks the base classes of whatever it was handed before it can + say no. Every argument that is not a document reaches here and + is told no, and on the arms this is called from those are most of + them. + */ + Item *arg= item->real_item()->conv_charset_arg(); + if (!arg) return false; - item= func->arguments()[0]; + item= arg; } return false; } diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 1419d25e7c2ec..9d61b758ec5f6 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -2094,6 +2094,13 @@ class Item_func_conv_charset :public Item_str_func static LEX_CSTRING name= {STRING_WITH_LEN("convert") }; return name; } + /* + What this wrapper is put round. A caller looking through it for the + item underneath is told, rather than working out from the object + that this is what it was handed - see Item::conv_charset_arg(), + which says why that is worth a virtual of its own. + */ + Item *conv_charset_arg() const override { return args[0]; } bool is_valid_json() const override { return m_marks.valid(); } bool is_nice_json() const override { return m_marks.nice(); } uint last_depth() const override { return m_marks.depth(); } diff --git a/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_deinit.inc b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_deinit.inc new file mode 100644 index 0000000000000..76b7582abfe49 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_deinit.inc @@ -0,0 +1,11 @@ +--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP +--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP +--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP +--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP +--disable_warnings +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_result_log +--enable_query_log +--enable_warnings diff --git a/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_init.inc b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_init.inc new file mode 100644 index 0000000000000..345c6fc501072 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_init.inc @@ -0,0 +1,23 @@ +--disable_warnings +--disable_query_log +--disable_result_log +--source ../t/test_init.inc +--enable_result_log +--enable_query_log +--enable_warnings +--let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1 +let $MASTER_1_COMMENT_2_1= + COMMENT='table "tbl_a", srv "s_2_1"'; +--let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES +let $CHILD2_1_DROP_TABLES= + DROP TABLE IF EXISTS tbl_a; +--let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES +let $CHILD2_1_CREATE_TABLES= + CREATE TABLE tbl_a ( + pkey int NOT NULL, + txt varchar(16) NOT NULL, + PRIMARY KEY (pkey) + ) $CHILD2_1_ENGINE DEFAULT CHARACTER SET utf8; +--let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES +let $CHILD2_1_SELECT_TABLES= + SELECT pkey, txt FROM tbl_a ORDER BY pkey; diff --git a/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown,group_by_handler.rdiff b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown,group_by_handler.rdiff new file mode 100644 index 0000000000000..c49606197b51c --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown,group_by_handler.rdiff @@ -0,0 +1,11 @@ +--- charset_conv_pushdown.result 2026-08-15 16:02:58.308945369 -0400 ++++ charset_conv_pushdown.reject 2026-08-15 16:03:11.788881446 -0400 +@@ -43,7 +43,7 @@ + SELECT argument FROM mysql.general_log + WHERE command_type != 'Execute' AND argument LIKE 'select %'; + argument +-select `pkey`,`txt` from `auto_test_remote`.`tbl_a` where ((convert(`txt` using latin1)) = 'beta') ++select t0.`pkey` `pkey`,t0.`txt` `txt` from `auto_test_remote`.`tbl_a` t0 where ((convert(t0.`txt` using latin1)) = 'beta') + SELECT argument FROM mysql.general_log + WHERE command_type != 'Execute' AND argument LIKE 'select %' + diff --git a/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown.result b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown.result new file mode 100644 index 0000000000000..a827a9cf8de58 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown.result @@ -0,0 +1,63 @@ +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 + +# A character-set conversion in a pushed-down condition keeps its +# conversion. The data node is told which set to convert into; a +# condition that arrived there without the USING clause would +# compare the bytes as they stand and answer a different row. + +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +connection child2_1; +SET @old_log_output = @@global.log_output; +SET GLOBAL log_output = 'TABLE,FILE'; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +connection child2_1; +CHILD2_1_CREATE_TABLES +connection master_1; +CREATE TABLE tbl_a ( +pkey int NOT NULL, +txt varchar(16) NOT NULL, +PRIMARY KEY (pkey) +) ENGINE=Spider COMMENT='table "tbl_a", srv "s_2_1"' +INSERT INTO tbl_a (pkey, txt) VALUES +(1, 'alpha'), +(2, 'beta'), +(3, 'gamma'), +(4, 'delta'); +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +SELECT pkey, txt FROM tbl_a WHERE CONVERT(txt USING latin1) = 'beta'; +pkey txt +2 beta + +# The statement the data node was sent +connection child2_1; +SELECT argument FROM mysql.general_log +WHERE command_type != 'Execute' AND argument LIKE 'select %'; +argument +select `pkey`,`txt` from `auto_test_remote`.`tbl_a` where ((convert(`txt` using latin1)) = 'beta') +SELECT argument FROM mysql.general_log +WHERE command_type != 'Execute' AND argument LIKE 'select %' + +deinit +connection master_1; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +DROP DATABASE IF EXISTS auto_test_remote; +SET GLOBAL log_output = @old_log_output; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 + +end of test diff --git a/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.cnf b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.cnf new file mode 100644 index 0000000000000..05dfd8a0bcea9 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.cnf @@ -0,0 +1,3 @@ +!include include/default_mysqld.cnf +!include ../my_1_1.cnf +!include ../my_2_1.cnf diff --git a/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.test b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.test new file mode 100644 index 0000000000000..38e651c42333e --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.test @@ -0,0 +1,76 @@ +--source ../include/charset_conv_pushdown_init.inc +--echo +--echo # A character-set conversion in a pushed-down condition keeps its +--echo # conversion. The data node is told which set to convert into; a +--echo # condition that arrived there without the USING clause would +--echo # compare the bytes as they stand and answer a different row. +--echo + +--connection master_1 +--disable_warnings +CREATE DATABASE auto_test_local; +USE auto_test_local; + +--connection child2_1 +SET @old_log_output = @@global.log_output; +SET GLOBAL log_output = 'TABLE,FILE'; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +--enable_warnings + +--connection child2_1 +--disable_query_log +echo CHILD2_1_CREATE_TABLES; +eval $CHILD2_1_CREATE_TABLES; +--enable_query_log + +--connection master_1 +--disable_query_log +echo CREATE TABLE tbl_a ( + pkey int NOT NULL, + txt varchar(16) NOT NULL, + PRIMARY KEY (pkey) +) $MASTER_1_ENGINE $MASTER_1_COMMENT_2_1; +eval CREATE TABLE tbl_a ( + pkey int NOT NULL, + txt varchar(16) NOT NULL, + PRIMARY KEY (pkey) +) $MASTER_1_ENGINE $MASTER_1_COMMENT_2_1; +--enable_query_log +INSERT INTO tbl_a (pkey, txt) VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'), + (4, 'delta'); + +--connection child2_1 +TRUNCATE TABLE mysql.general_log; + +--connection master_1 +--disable_ps2_protocol +SELECT pkey, txt FROM tbl_a WHERE CONVERT(txt USING latin1) = 'beta'; +--enable_ps2_protocol + +--echo +--echo # The statement the data node was sent +--connection child2_1 +--disable_view_protocol +--disable_ps2_protocol +SELECT argument FROM mysql.general_log + WHERE command_type != 'Execute' AND argument LIKE 'select %'; +--enable_ps2_protocol +--enable_view_protocol + +--echo +--echo deinit +--disable_warnings +--connection master_1 +DROP DATABASE IF EXISTS auto_test_local; + +--connection child2_1 +DROP DATABASE IF EXISTS auto_test_remote; +SET GLOBAL log_output = @old_log_output; +--enable_warnings +--source ../include/charset_conv_pushdown_deinit.inc +--echo +--echo end of test