diff --git a/include/json_lib.h b/include/json_lib.h index d71fcd243be76..91053ffaf2e9f 100644 --- a/include/json_lib.h +++ b/include/json_lib.h @@ -12,7 +12,7 @@ extern "C" { /* When error happens, the c_next of the JSON engine contains the - character that caused the error, and the c_str is the position + character that caused the error, and the error_pos is the position in string where the error occurs. */ enum json_errors { @@ -41,6 +41,7 @@ typedef struct st_json_string_t my_wc_t c_next; /* UNICODE of the last read character */ int c_next_len; /* character lenght of the last read character. */ int error; /* error code. */ + const uchar *error_pos; /* Where in the string that error happened. */ CHARSET_INFO *cs; /* Character set of the JSON string. */ @@ -52,6 +53,43 @@ typedef struct st_json_string_t void json_string_set_cs(json_string_t *s, CHARSET_INFO *i_cs); void json_string_set_str(json_string_t *s, const uchar *str, const uchar *end); + +/* + Refuse the string, and remember where the refusing happened. + + The position is taken here rather than read off c_str at the point + the refusal is reported, because the scanner does not come to rest + where it failed. A caller is free to go on asking for the next token + after being told no - not all of them are obliged to stop, and see + where the depth refusals are written for why they cannot be made to - + and c_str then walks past the place the refusal happened, so a reading + taken later describes somewhere else. + + The first refusal is the one kept, for the same reason: a scan that + carries on after failing reaches handlers that refuse it again on + their own account, and what the caller is told has to be what stopped + the scan rather than whatever it ran into afterwards. + + Returns the code it was given, so the sites written as + `return s->error= JE_SYN` keep their shape. A code of zero refuses + nothing, which is what the sites that pass a result along need. + + Everything that a position is reported for is refused through here, + so error and error_pos are set together and mean each other. The + geometry reader puts codes of its own into error without coming here; + it reports those by name and never asks where, and it means to + replace whatever the scanner said, which is why it still assigns. +*/ +static inline int json_error(json_string_t *s, int code) +{ + if (code && !s->error) + { + s->error= code; + s->error_pos= s->c_str; + } + return code; +} + #define json_next_char(j) \ ((j)->c_next_len= (j)->wc((j)->cs, &(j)->c_next, (j)->c_str, (j)->str_end)) #define json_eos(j) ((j)->c_str >= (j)->str_end) @@ -235,6 +273,14 @@ typedef struct st_json_engine_t } json_engine_t; +#ifndef DBUG_OFF +/* + Told about every reading of a value, when somebody has asked to be. + See json_scan_start() for why this is where the telling happens. +*/ +extern void (*json_scan_start_hook)(void); +#endif + int json_scan_start(json_engine_t *je, CHARSET_INFO *i_cs, const uchar *str, const uchar *end); int json_scan_next(json_engine_t *j); diff --git a/mysql-test/main/func_concat.result b/mysql-test/main/func_concat.result index 4b91fb97c6c8c..d31eaf0942eb3 100644 --- a/mysql-test/main/func_concat.result +++ b/mysql-test/main/func_concat.result @@ -304,3 +304,72 @@ f NULL DROP TABLE t1; # End of 10.5 tests +# Start of 10.11 tests +# +# CONCAT keeps the value of its first argument when that argument +# answers in the buffer it was handed, pointing at bytes it does +# not own +# +SELECT JSON_TYPE('{"a":1}'); +JSON_TYPE('{"a":1}') +OBJECT +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'X'); +CONCAT(JSON_TYPE('{"a":1}'), 'X') +OBJECTX +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B'); +CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B') +OBJECTAB +SELECT CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]')); +CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]')) +OBJECTARRAY +SELECT CONCAT(JSON_TYPE('{"a":1}'), ''); +CONCAT(JSON_TYPE('{"a":1}'), '') +OBJECT +SELECT LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X')); +LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X')) +7 +SELECT GET_FORMAT(DATE, 'USA'); +GET_FORMAT(DATE, 'USA') +%m.%d.%Y +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), 'X'); +CONCAT(GET_FORMAT(DATE, 'USA'), 'X') +%m.%d.%YX +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO')); +CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO')) +%m.%d.%Y%Y-%m-%d +# Every other argument position was always right +SELECT CONCAT('P', JSON_TYPE('{"a":1}')); +CONCAT('P', JSON_TYPE('{"a":1}')) +POBJECT +SELECT CONCAT('P', GET_FORMAT(DATE, 'USA')); +CONCAT('P', GET_FORMAT(DATE, 'USA')) +P%m.%d.%Y +SELECT CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X'); +CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X') +OBJECT-X +# The same over table data +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'), ('[1,2]'); +SELECT CONCAT(JSON_TYPE(j), '.') AS c FROM t1; +c +OBJECT. +ARRAY. +SELECT CONCAT(JSON_TYPE(j), JSON_TYPE(j)) AS c FROM t1; +c +OBJECTOBJECT +ARRAYARRAY +DROP TABLE t1; +# The || operator in sql_mode=ORACLE concatenates the same way +SET @save_sql_mode=@@sql_mode; +SET sql_mode=ORACLE; +SELECT JSON_TYPE('{"a":1}') || 'X'; +JSON_TYPE('{"a":1}') || 'X' +OBJECTX +SELECT GET_FORMAT(DATE, 'USA') || 'X'; +GET_FORMAT(DATE, 'USA') || 'X' +%m.%d.%YX +SELECT 'P' || JSON_TYPE('{"a":1}'); +'P' || JSON_TYPE('{"a":1}') +POBJECT +SET sql_mode=@save_sql_mode; +# End of 10.11 tests diff --git a/mysql-test/main/func_concat.test b/mysql-test/main/func_concat.test index f93b150f88f90..51caf15644f65 100644 --- a/mysql-test/main/func_concat.test +++ b/mysql-test/main/func_concat.test @@ -281,3 +281,51 @@ SELECT CONCAT_WS(' ', a, b, PASSWORD(c)) AS f FROM t1 GROUP BY f WITH ROLLUP; DROP TABLE t1; --echo # End of 10.5 tests + +--echo # Start of 10.11 tests + +--echo # +--echo # CONCAT keeps the value of its first argument when that argument +--echo # answers in the buffer it was handed, pointing at bytes it does +--echo # not own +--echo # + +SELECT JSON_TYPE('{"a":1}'); +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'X'); +SELECT CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B'); +SELECT CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]')); +SELECT CONCAT(JSON_TYPE('{"a":1}'), ''); +SELECT LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X')); + +SELECT GET_FORMAT(DATE, 'USA'); +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), 'X'); +SELECT CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO')); + +--echo # Every other argument position was always right + +SELECT CONCAT('P', JSON_TYPE('{"a":1}')); +SELECT CONCAT('P', GET_FORMAT(DATE, 'USA')); +SELECT CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X'); + +--echo # The same over table data + +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'), ('[1,2]'); +SELECT CONCAT(JSON_TYPE(j), '.') AS c FROM t1; +SELECT CONCAT(JSON_TYPE(j), JSON_TYPE(j)) AS c FROM t1; +DROP TABLE t1; + +--echo # The || operator in sql_mode=ORACLE concatenates the same way +# A wrapping view is created on a connection of its own, which does not +# carry this session's sql_mode, so || would be read as an OR there. +--disable_view_protocol + +SET @save_sql_mode=@@sql_mode; +SET sql_mode=ORACLE; +SELECT JSON_TYPE('{"a":1}') || 'X'; +SELECT GET_FORMAT(DATE, 'USA') || 'X'; +SELECT 'P' || JSON_TYPE('{"a":1}'); +SET sql_mode=@save_sql_mode; +--enable_view_protocol + +--echo # End of 10.11 tests diff --git a/mysql-test/main/func_gconcat.result b/mysql-test/main/func_gconcat.result index 6c50416ee2717..a727895576599 100644 --- a/mysql-test/main/func_gconcat.result +++ b/mysql-test/main/func_gconcat.result @@ -1556,3 +1556,68 @@ Warning 1260 Row 1 was cut by GROUP_CONCAT() disconnect u; connection default; # End of 10.6 tests +# +# A group with an OFFSET past its last row, asked for more than once +# +# Nothing says how often a statement asks for the result of a group. +# HAVING on the alias is the shortest one that asks twice, and the +# answer must not depend on how often it was asked. +# +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f'); +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +v + +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v + +# two conditions, so it is asked a third time +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%' AND v NOT LIKE 'a%'; +v + +# DISTINCT reaches the same walk by the other route +SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v + +# a group per row of output, so more than one group is replayed +SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 +GROUP BY g HAVING v LIKE '%' ORDER BY g; +g v +1 +2 +# an offset that stops inside the group, which must not move +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +b,c +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 3) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +d +# and no LIMIT at all +SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1 HAVING v LIKE '%'; +v +a,b,c,d +# the same through the aggregate that is built on this one, where an +# element written a second time lands outside the brackets already +# put round the group +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +v +[] +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +[] +SELECT JSON_VALID(JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4)) AS v +FROM t1 WHERE g = 1 HAVING v LIKE '%'; +v +1 +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 +HAVING v LIKE '%'; +v +["b","c"] +DROP TABLE t1; +# End of 10.11 tests diff --git a/mysql-test/main/func_gconcat.test b/mysql-test/main/func_gconcat.test index 90bfc9188e7dd..650d438710fb8 100644 --- a/mysql-test/main/func_gconcat.test +++ b/mysql-test/main/func_gconcat.test @@ -1137,3 +1137,46 @@ disconnect u; connection default; --echo # End of 10.6 tests + +--echo # +--echo # A group with an OFFSET past its last row, asked for more than once +--echo # +--echo # Nothing says how often a statement asks for the result of a group. +--echo # HAVING on the alias is the shortest one that asks twice, and the +--echo # answer must not depend on how often it was asked. +--echo # +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f'); +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +--echo # two conditions, so it is asked a third time +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%' AND v NOT LIKE 'a%'; +--echo # DISTINCT reaches the same walk by the other route +SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +--echo # a group per row of output, so more than one group is replayed +SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 + GROUP BY g HAVING v LIKE '%' ORDER BY g; +--echo # an offset that stops inside the group, which must not move +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 3) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +--echo # and no LIMIT at all +SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1 HAVING v LIKE '%'; + +--echo # the same through the aggregate that is built on this one, where an +--echo # element written a second time lands outside the brackets already +--echo # put round the group +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +SELECT JSON_VALID(JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4)) AS v + FROM t1 WHERE g = 1 HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1 + HAVING v LIKE '%'; +DROP TABLE t1; + +--echo # End of 10.11 tests diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index a5b21ad20294c..d0e7b870547e0 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -280,7 +280,7 @@ create table t1 as select json_object('id', 87, 'name', 'carrot') as f; show create table t1; Table Create Table t1 CREATE TABLE `t1` ( - `f` varchar(46) DEFAULT NULL + `f` varchar(52) DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci select * from t1; f @@ -305,7 +305,7 @@ json_quote('foo') show create table t1; Table Create Table t1 CREATE TABLE `t1` ( - `json_quote('foo')` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + `json_quote('foo')` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table t1; select json_merge('string'); @@ -770,8 +770,8 @@ JSON_QUOTE(_utf8'foo') AS c2; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `c1` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, - `c2` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + `c1` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL, + `c2` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci DROP TABLE t1; # @@ -780,6 +780,8 @@ DROP TABLE t1; select json_array(1,user(),compress(5.140264e+307)); json_array(1,user(),compress(5.140264e+307)) NULL +Warnings: +Note 4035 Broken JSON string in argument 3 to function 'json_array' at position 0 # # MDEV-16869 String functions don't respect character set of JSON_VALUE. # @@ -834,7 +836,7 @@ CREATE TABLE t2 SELECT JSON_ARRAY_INSERT(fld, '$.[0]', '0') FROM t1; SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( - `JSON_ARRAY_INSERT(fld, '$.[0]', '0')` varchar(21) DEFAULT NULL + `JSON_ARRAY_INSERT(fld, '$.[0]', '0')` varchar(40) DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci DROP TABLE t1, t2; SET sql_mode=default; @@ -1615,7 +1617,7 @@ insert into t1 values (concat('x64-', repeat('b', 60))); insert into t1 values (concat('x64-', repeat('c', 60))); select json_arrayagg(a) from t1; json_arrayagg(a) -["x64-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] +[] Warnings: Warning 1260 Row 1 was cut by JSON_ARRAYAGG() drop table t1; @@ -1741,9 +1743,7 @@ SET @save_collation_connection= @@collation_connection; SET collation_connection='utf16_bin'; SELECT JSON_EXTRACT('{"a": 1,"b": 2}','$.a'); JSON_EXTRACT('{"a": 1,"b": 2}','$.a') -NULL -Warnings: -Warning 4036 Character disallowed in JSON in argument 1 to function 'json_extract' at position 2 +1 SET @@collation_connection= @save_collation_connection; # End of 10.5 tests # @@ -1832,9 +1832,9 @@ create temporary table t (c varchar(20) character set latin1); insert into t values ('ab'), ('cd'), ('ef'); select json_arrayagg(c order by c) from t; json_arrayagg(c order by c) -["ab","cd"] +["ab"] Warnings: -Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() drop temporary table t; # End of 10.6 tests SELECT json_extract(t.j, '$') diff --git a/mysql-test/main/func_json_agg_cap.result b/mysql-test/main/func_json_agg_cap.result new file mode 100644 index 0000000000000..c6eb191f20e5d --- /dev/null +++ b/mysql-test/main/func_json_agg_cap.result @@ -0,0 +1,760 @@ +# +# The most a JSON aggregate returns is group_concat_max_len +# bytes, the same limit and the same wording GROUP_CONCAT is held +# to. The brackets that go round an aggregated array are part of +# what is returned and are written after the length has been +# tested, so the room for them has to be kept back when the test +# is made. Where the elements ended exactly on the limit there +# was nothing to cut, and the answer went over it in silence. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (a VARCHAR(20)); +INSERT INTO t1 VALUES ('aaaaaaaaaa'), ('bbbbbbbbbb'); +# +# 1. The limit walked across the whole of an element and past the +# end of the group. Nothing may answer longer than its cap. +# +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 4 AS within FROM t1; +cap v len within +4 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 5 AS within FROM t1; +cap v len within +5 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 6 AS within FROM t1; +cap v len within +6 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 7 AS within FROM t1; +cap v len within +7 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 8 AS within FROM t1; +cap v len within +8 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 9 AS within FROM t1; +cap v len within +9 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 10 AS within FROM t1; +cap v len within +10 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 11 AS within FROM t1; +cap v len within +11 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 12 AS within FROM t1; +cap v len within +12 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 13 AS within FROM t1; +cap v len within +13 [] 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 14 AS within FROM t1; +cap v len within +14 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 15 AS within FROM t1; +cap v len within +15 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 16 AS within FROM t1; +cap v len within +16 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 17 AS within FROM t1; +cap v len within +17 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 18 AS within FROM t1; +cap v len within +18 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 19 AS within FROM t1; +cap v len within +19 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 20 AS within FROM t1; +cap v len within +20 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 21 AS within FROM t1; +cap v len within +21 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 22 AS within FROM t1; +cap v len within +22 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 23 AS within FROM t1; +cap v len within +23 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 24; +SELECT 24 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 24 AS within FROM t1; +cap v len within +24 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 25; +SELECT 25 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 25 AS within FROM t1; +cap v len within +25 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 26; +SELECT 26 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 26 AS within FROM t1; +cap v len within +26 ["aaaaaaaaaa"] 14 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 27; +SELECT 27 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 27 AS within FROM t1; +cap v len within +27 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 28; +SELECT 28 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 28 AS within FROM t1; +cap v len within +28 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 29; +SELECT 29 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 29 AS within FROM t1; +cap v len within +29 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 30; +SELECT 30 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 30 AS within FROM t1; +cap v len within +30 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = 31; +SELECT 31 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 31 AS within FROM t1; +cap v len within +31 ["aaaaaaaaaa","bbbbbbbbbb"] 27 1 +SET SESSION group_concat_max_len = DEFAULT; +# +# 2. The two aggregates side by side under one cap, and +# GROUP_CONCAT beside them, which has nothing to put on +# afterwards and has always ended on its limit exactly. +# +SET SESSION group_concat_max_len = 25; +SELECT LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS arr_len, +LENGTH(JSON_OBJECTAGG(a, 1)) AS obj_len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS gc_len, +@@group_concat_max_len AS cap FROM t1; +arr_len obj_len gc_len cap +14 16 21 25 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 3. A set that writes no character in one byte. A bracket costs +# two bytes there, so four is what has to be kept back. +# +CREATE TABLE t2 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t2 VALUES ('aaaa'), ('bbbb'); +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 16 AS within FROM t2; +cap h len within +16 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 17 AS within FROM t2; +cap h len within +17 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 18 AS within FROM t2; +cap h len within +18 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 19 AS within FROM t2; +cap h len within +19 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 20 AS within FROM t2; +cap h len within +20 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 21 AS within FROM t2; +cap h len within +21 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 22 AS within FROM t2; +cap h len within +22 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 23 AS within FROM t2; +cap h len within +23 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 24; +SELECT 24 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 24 AS within FROM t2; +cap h len within +24 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 25; +SELECT 25 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 25 AS within FROM t2; +cap h len within +25 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 26; +SELECT 26 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 26 AS within FROM t2; +cap h len within +26 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 27; +SELECT 27 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 27 AS within FROM t2; +cap h len within +27 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 28; +SELECT 28 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 28 AS within FROM t2; +cap h len within +28 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 29; +SELECT 29 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 29 AS within FROM t2; +cap h len within +29 005B002200610061006100610022005D 16 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 30; +SELECT 30 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 30 AS within FROM t2; +cap h len within +30 005B002200610061006100610022002C002200620062006200620022005D 30 1 +SET SESSION group_concat_max_len = 31; +SELECT 31 AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 31 AS within FROM t2; +cap h len within +31 005B002200610061006100610022002C002200620062006200620022005D 30 1 +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; +# +# 4. A cap that leaves no room for a whole element still leaves +# the brackets, which is what an empty group answers as well. +# +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len FROM t1; +v len +[] 2 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a IS NULL; +v +NULL +SET SESSION group_concat_max_len = DEFAULT; +# +# 5. The one place the limit cannot be kept: a cap smaller than +# the brackets themselves. An empty array is as short as this +# function goes, and in a set of four bytes to the character it +# is eight bytes, against a limit that may be set as low as +# four. There is nothing left to cut, so the brackets stand +# and the answer is over the cap - which is why the width this +# item declares carries room for them on top of the cap. +# +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t3 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 4 AS within FROM t3; +h len within +0000005B0000005D 8 0 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +# and the same cap where the brackets do fit +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, +LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 8 AS within FROM t3; +h len within +0000005B0000005D 8 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; +# +# 6. GROUP_CONCAT is not moved by any of this: it returns what +# it accumulated, and its cut lands on the byte the limit falls +# on as it always has. +# +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 4 AS within FROM t1; +cap v len within +4 aaaa 4 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 5 AS within FROM t1; +cap v len within +5 aaaaa 5 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 6 AS within FROM t1; +cap v len within +6 aaaaaa 6 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 7 AS within FROM t1; +cap v len within +7 aaaaaaa 7 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 8 AS within FROM t1; +cap v len within +8 aaaaaaaa 8 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 9 AS within FROM t1; +cap v len within +9 aaaaaaaaa 9 1 +Warnings: +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +Warning 1260 Row 1 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 10 AS within FROM t1; +cap v len within +10 aaaaaaaaaa 10 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 11 AS within FROM t1; +cap v len within +11 aaaaaaaaaa, 11 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 12 AS within FROM t1; +cap v len within +12 aaaaaaaaaa,b 12 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 13 AS within FROM t1; +cap v len within +13 aaaaaaaaaa,bb 13 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 14 AS within FROM t1; +cap v len within +14 aaaaaaaaaa,bbb 14 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 15 AS within FROM t1; +cap v len within +15 aaaaaaaaaa,bbbb 15 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 16 AS within FROM t1; +cap v len within +16 aaaaaaaaaa,bbbbb 16 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 17 AS within FROM t1; +cap v len within +17 aaaaaaaaaa,bbbbbb 17 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 18 AS within FROM t1; +cap v len within +18 aaaaaaaaaa,bbbbbbb 18 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 19 AS within FROM t1; +cap v len within +19 aaaaaaaaaa,bbbbbbbb 19 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 20 AS within FROM t1; +cap v len within +20 aaaaaaaaaa,bbbbbbbbb 20 1 +Warnings: +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +Warning 1260 Row 2 was cut by GROUP_CONCAT() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 21 AS within FROM t1; +cap v len within +21 aaaaaaaaaa,bbbbbbbbbb 21 1 +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 22 AS within FROM t1; +cap v len within +22 aaaaaaaaaa,bbbbbbbbbb 21 1 +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, GROUP_CONCAT(a ORDER BY a) AS v, +LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, +LENGTH(GROUP_CONCAT(a ORDER BY a)) <= 23 AS within FROM t1; +cap v len within +23 aaaaaaaaaa,bbbbbbbbbb 21 1 +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t1; +# +# 7. The width the item declares, which is the width a column made +# out of the answer is given. The answer is over the cap +# wherever the punctuation alone is - section 5 - so a width +# worked out from the cap and nothing else cannot hold it, and +# what reaches the column is cut without a word said. +# +CREATE TABLE t4 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t4 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +# Non-strict, so that the cut this cap makes - which is section 5 +# over again - is a warning and the statement runs to the end, +# where the width is what is being looked at. +SET SESSION sql_mode = ''; +CREATE TABLE t5 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, +JSON_OBJECTAGG(a, 1) AS obj FROM t4; +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SHOW CREATE TABLE t5; +Table Create Table +t5 CREATE TABLE `t5` ( + `arr` varchar(3) CHARACTER SET utf32 COLLATE utf32_general_ci DEFAULT NULL, + `obj` varchar(3) CHARACTER SET utf32 COLLATE utf32_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT HEX(arr) AS arr_h, JSON_VALID(arr) AS arr_ok, +HEX(obj) AS obj_h, JSON_VALID(obj) AS obj_ok FROM t5; +arr_h arr_ok obj_h obj_ok +0000005B0000005D 1 0000007B0000007D 1 +DROP TABLE t5; +SET SESSION sql_mode = DEFAULT; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; +# +# 8. That width is worked out in a wider type than it is kept in. +# A cap that fills the narrower type on its own leaves the room +# for the brackets nowhere to go, and a width that wrapped +# round would be a column too narrow to hold any answer at all. +# +SET @old_max_allowed_packet = @@GLOBAL.max_allowed_packet; +SET GLOBAL max_allowed_packet = 1073741824; +connect con_wide,localhost,root,,test; +SET SESSION group_concat_max_len = 1073741824; +# Four bytes to the character, so that the cap alone fills the +# width: a narrower set leaves room the brackets can go in. +CREATE TABLE t6 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t6 VALUES ('aa'), ('bb'); +CREATE TABLE t7 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, +JSON_OBJECTAGG(a, 1) AS obj FROM t6; +SHOW CREATE TABLE t7; +Table Create Table +t7 CREATE TABLE `t7` ( + `arr` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL, + `obj` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT arr, obj FROM t7; +arr obj +["aa","bb"] {"aa":1, "bb":1} +DROP TABLE t7, t6; +disconnect con_wide; +connection default; +SET GLOBAL max_allowed_packet = @old_max_allowed_packet; diff --git a/mysql-test/main/func_json_agg_cap.test b/mysql-test/main/func_json_agg_cap.test new file mode 100644 index 0000000000000..7a39ce9d876f5 --- /dev/null +++ b/mysql-test/main/func_json_agg_cap.test @@ -0,0 +1,167 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # The most a JSON aggregate returns is group_concat_max_len +--echo # bytes, the same limit and the same wording GROUP_CONCAT is held +--echo # to. The brackets that go round an aggregated array are part of +--echo # what is returned and are written after the length has been +--echo # tested, so the room for them has to be kept back when the test +--echo # is made. Where the elements ended exactly on the limit there +--echo # was nothing to cut, and the answer went over it in silence. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (a VARCHAR(20)); +INSERT INTO t1 VALUES ('aaaaaaaaaa'), ('bbbbbbbbbb'); + +--echo # +--echo # 1. The limit walked across the whole of an element and past the +--echo # end of the group. Nothing may answer longer than its cap. +--echo # + +--let $cap= 4 +while ($cap < 32) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= $cap AS within FROM t1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 2. The two aggregates side by side under one cap, and +--echo # GROUP_CONCAT beside them, which has nothing to put on +--echo # afterwards and has always ended on its limit exactly. +--echo # + +SET SESSION group_concat_max_len = 25; +SELECT LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS arr_len, + LENGTH(JSON_OBJECTAGG(a, 1)) AS obj_len, + LENGTH(GROUP_CONCAT(a ORDER BY a)) AS gc_len, + @@group_concat_max_len AS cap FROM t1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 3. A set that writes no character in one byte. A bracket costs +--echo # two bytes there, so four is what has to be kept back. +--echo # + +CREATE TABLE t2 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t2 VALUES ('aaaa'), ('bbbb'); + +--let $cap= 16 +while ($cap < 32) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= $cap AS within FROM t2; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 4. A cap that leaves no room for a whole element still leaves +--echo # the brackets, which is what an empty group answers as well. +--echo # + +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len FROM t1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a IS NULL; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 5. The one place the limit cannot be kept: a cap smaller than +--echo # the brackets themselves. An empty array is as short as this +--echo # function goes, and in a set of four bytes to the character it +--echo # is eight bytes, against a limit that may be set as low as +--echo # four. There is nothing left to cut, so the brackets stand +--echo # and the answer is over the cap - which is why the width this +--echo # item declares carries room for them on top of the cap. +--echo # + +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t3 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 4 AS within FROM t3; +--echo # and the same cap where the brackets do fit +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS len, + LENGTH(JSON_ARRAYAGG(a ORDER BY a)) <= 8 AS within FROM t3; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; + +--echo # +--echo # 6. GROUP_CONCAT is not moved by any of this: it returns what +--echo # it accumulated, and its cut lands on the byte the limit falls +--echo # on as it always has. +--echo # + +--let $cap= 4 +while ($cap < 24) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, GROUP_CONCAT(a ORDER BY a) AS v, + LENGTH(GROUP_CONCAT(a ORDER BY a)) AS len, + LENGTH(GROUP_CONCAT(a ORDER BY a)) <= $cap AS within FROM t1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +DROP TABLE t1; + +--echo # +--echo # 7. The width the item declares, which is the width a column made +--echo # out of the answer is given. The answer is over the cap +--echo # wherever the punctuation alone is - section 5 - so a width +--echo # worked out from the cap and nothing else cannot hold it, and +--echo # what reaches the column is cut without a word said. +--echo # + +CREATE TABLE t4 (a VARCHAR(20)) CHARACTER SET utf32; +INSERT INTO t4 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 4; +--echo # Non-strict, so that the cut this cap makes - which is section 5 +--echo # over again - is a warning and the statement runs to the end, +--echo # where the width is what is being looked at. +SET SESSION sql_mode = ''; +CREATE TABLE t5 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, + JSON_OBJECTAGG(a, 1) AS obj FROM t4; +SHOW CREATE TABLE t5; +SELECT HEX(arr) AS arr_h, JSON_VALID(arr) AS arr_ok, + HEX(obj) AS obj_h, JSON_VALID(obj) AS obj_ok FROM t5; +DROP TABLE t5; +SET SESSION sql_mode = DEFAULT; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; + +--echo # +--echo # 8. That width is worked out in a wider type than it is kept in. +--echo # A cap that fills the narrower type on its own leaves the room +--echo # for the brackets nowhere to go, and a width that wrapped +--echo # round would be a column too narrow to hold any answer at all. +--echo # + +SET @old_max_allowed_packet = @@GLOBAL.max_allowed_packet; +SET GLOBAL max_allowed_packet = 1073741824; +--connect (con_wide,localhost,root,,test) +SET SESSION group_concat_max_len = 1073741824; +--echo # Four bytes to the character, so that the cap alone fills the +--echo # width: a narrower set leaves room the brackets can go in. +CREATE TABLE t6 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t6 VALUES ('aa'), ('bb'); +CREATE TABLE t7 AS SELECT JSON_ARRAYAGG(a ORDER BY a) AS arr, + JSON_OBJECTAGG(a, 1) AS obj FROM t6; +SHOW CREATE TABLE t7; +SELECT arr, obj FROM t7; +DROP TABLE t7, t6; +--disconnect con_wide +--connection default +SET GLOBAL max_allowed_packet = @old_max_allowed_packet; diff --git a/mysql-test/main/func_json_agg_charset.result b/mysql-test/main/func_json_agg_charset.result new file mode 100644 index 0000000000000..69b12dbf5aeb6 --- /dev/null +++ b/mysql-test/main/func_json_agg_charset.result @@ -0,0 +1,325 @@ +# +# How the JSON aggregates deliver their result to the client. +# +# A result computed in one character set and asked for in another is +# converted on the way out. Two separate questions decide what the +# client is left holding: which character set the result was computed +# in, and whether what is in it survives the conversion to the one +# that was asked for. +# +# swe7 is the vehicle for the first of those. It has no code point +# for the brackets and braces a JSON document is built from, so a +# result converted into it arrives with those replaced by '?'. That +# is what a conversion having happened looks like, and it is visible +# without a single non-ASCII byte in this file. A client asking for +# latin1 sees nothing to remark on, latin1 having the punctuation: +# the conversion happens just the same and the result arrives intact. +# +# The later sections ask the other question, leaving the client alone +# and computing the result in a character set of its own. There the +# punctuation has to go down at the width that character set asks +# for, and where the character set has no punctuation to write, what +# is written is not a document at all. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES (1, 'aa'), (1, 'bb'), (2, 'cc'), (2, 'dd'); +# +# 1. An aggregate delivered next to the functions that are not. +# +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('aa','bb') AS v; +v +?"aa", "bb"? +SELECT JSON_OBJECT('aa','bb') AS v; +v +?"aa": "bb"? +SELECT JSON_SET('{}','$.a','b') AS v; +v +?"a": "b"? +SELECT JSON_KEYS('{"aa":1}') AS v; +v +?"aa"? +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +?"aa","bb","cc","dd"? +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +v +?"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"? +# one group per row of output, so the delivery is repeated +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ?"aa","bb"? +2 ?"cc","dd"? +SELECT g, JSON_OBJECTAGG(a,a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ?"aa":"aa", "bb":"bb"? +2 ?"cc":"cc", "dd":"dd"? +SET SESSION character_set_results = DEFAULT; +# +# 2. The value itself, which the delivery does not touch. +# +# Read inside the server rather than sent to the client, the aggregate +# is the same either way. HEX() answers in ASCII, so the bytes can be +# shown under both settings and compared. +# +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, +JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +valid len +1 4 +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1; +v +[["aa","bb","cc","dd"]] +SET SESSION character_set_results = swe7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, +JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +valid len +1 4 +SET SESSION character_set_results = DEFAULT; +# +# 3. What the function says it returns. +# +# CHARSET() and COLLATION() report what the expression was resolved to, +# not what the returned value ended up labelled with, so neither of +# them can see a difference in delivery. A probe built from these two +# alone, or from ASCII values under a character set that can hold them, +# observes nothing at all. +# +SELECT CHARSET(JSON_ARRAYAGG(a)) AS cs, COLLATION(JSON_ARRAYAGG(a)) AS co +FROM t1; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_OBJECTAGG(a,a)) AS cs, COLLATION(JSON_OBJECTAGG(a,a)) AS co +FROM t1; +cs co +utf8mb4 utf8mb4_general_ci +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +["aa","bb","cc","dd"] +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +v +{"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"} +SET SESSION character_set_results = DEFAULT; +# +# 4. Stored rather than delivered. +# +# A value on its way into a column is converted from the character set +# the expression was resolved to, which is a separate question from the +# one above. +# +CREATE TABLE t2 (v VARCHAR(50) CHARACTER SET latin1); +INSERT INTO t2 SELECT JSON_ARRAYAGG(a) FROM t1; +INSERT INTO t2 SELECT JSON_OBJECTAGG(a,a) FROM t1; +SELECT v, HEX(v) AS h FROM t2 ORDER BY v; +v h +["aa","bb","cc","dd"] 5B226161222C226262222C226363222C226464225D +{"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"} 7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +SET SESSION character_set_results = swe7; +SELECT v FROM t2 ORDER BY v; +v +?"aa","bb","cc","dd"? +?"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"? +SET SESSION character_set_results = DEFAULT; +# +# 5. The other ways into the same result. +# +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1; +v +?"aa","bb","cc","dd"? +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1; +v +?"dd","cc","bb","aa"? +# a group of one writes no separator, so it exercises less than it looks +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'aa'; +v +?"aa"? +# no rows at all: NULL, which is never delivered as a string +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +v +NULL +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1 WHERE g = 99; +v +NULL +SET SESSION character_set_results = DEFAULT; +# +# 6. A result computed in a character set that ASCII is not a part of. +# +# Every character of such a result is two or four bytes wide, the +# brackets the aggregate puts around the elements included. Written +# as single bytes those two would leave everything between them one +# byte out of step, and the result would read as an entirely +# different set of characters rather than as a document. +# +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1; +h +005B0022006100610022002C0022006200620022002C0022006300630022002C0022006400640022005D +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS valid FROM t1; +valid +1 +SELECT CONVERT(JSON_ARRAYAGG(CONVERT(a USING ucs2)) USING utf8mb4) AS v FROM t1; +v +["aa","bb","cc","dd"] +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS h FROM t1; +h +005B0022006100610022002C0022006200620022002C0022006300630022002C0022006400640022005D +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS valid FROM t1; +valid +1 +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS h FROM t1; +h +0000005B000000220000006100000061000000220000002C000000220000006200000062000000220000002C000000220000006300000063000000220000002C000000220000006400000064000000220000005D +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS valid FROM t1; +valid +1 +# a group of one, which is bracket, element, bracket and nothing else +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1 WHERE a = 'aa'; +h +005B0022006100610022005D +# the same values where one byte is one character, which is the case +# the brackets were always written for +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING latin1))) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf8mb4))) AS h FROM t1; +h +5B226161222C226262222C226363222C226464225D +# +# 7. The sibling aggregate, in the same character sets. +# +# It reaches the same question by another route. Its opening brace +# is not written next to its closing one: the closing brace goes on +# when the result is asked for, and the opening brace has to be there +# before the first pair is added. Where that brace is written decides +# whether it can be written in the character set the result is in, and +# whether the two ends of the object match. +# +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h +FROM t1; +h +007B0022006100610022003A0022006100610022002C00200022006200620022003A0022006200620022002C00200022006300630022003A0022006300630022002C00200022006400640022003A0022006400640022007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid FROM t1; +valid +1 +SELECT CONVERT(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2)) USING utf8mb4) AS v +FROM t1; +v +{"aa":"aa", "bb":"bb", "cc":"cc", "dd":"dd"} +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf16), CONVERT(a USING utf16))) AS h +FROM t1; +h +007B0022006100610022003A0022006100610022002C00200022006200620022003A0022006200620022002C00200022006300630022003A0022006300630022002C00200022006400640022003A0022006400640022007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf16), +CONVERT(a USING utf16))) AS valid FROM t1; +valid +1 +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf32), CONVERT(a USING utf32))) AS h +FROM t1; +h +0000007B000000220000006100000061000000220000003A000000220000006100000061000000220000002C00000020000000220000006200000062000000220000003A000000220000006200000062000000220000002C00000020000000220000006300000063000000220000003A000000220000006300000063000000220000002C00000020000000220000006400000064000000220000003A000000220000006400000064000000220000007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf32), +CONVERT(a USING utf32))) AS valid FROM t1; +valid +1 +# one pair, where no separator is written between pairs at all +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h +FROM t1 WHERE a = 'aa'; +h +007B0022006100610022003A0022006100610022007D +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid +FROM t1 WHERE a = 'aa'; +valid +1 +# a group each, so the opening brace is written more than once +SELECT g, HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS h +FROM t1 GROUP BY g ORDER BY g; +g h +1 007B0022006100610022003A0022006100610022002C00200022006200620022003A0022006200620022007D +2 007B0022006300630022003A0022006300630022002C00200022006400640022003A0022006400640022007D +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid +FROM t1 GROUP BY g ORDER BY g; +g valid +1 1 +2 1 +# a copy of the aggregate, which is made without fixing it again +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), +CONVERT(a USING ucs2))) AS valid +FROM t1 GROUP BY g WITH ROLLUP; +g valid +1 1 +2 1 +NULL 1 +# no rows, which is a NULL and never an empty object +SELECT JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2)) AS v +FROM t1 WHERE g = 99; +v +NULL +# and where one byte is one character, which must not move +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING latin1), +CONVERT(a USING latin1))) AS h FROM t1; +h +7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf8mb4), +CONVERT(a USING utf8mb4))) AS h FROM t1; +h +7B226161223A226161222C20226262223A226262222C20226363223A226363222C20226464223A226464227D +SELECT g, HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1 GROUP BY g ORDER BY g; +g h +1 7B226161223A226161222C20226262223A226262227D +2 7B226363223A226363222C20226464223A226464227D +# +# 8. swe7 as the character set the aggregate is resolved to. +# +# Everywhere above, swe7 is only what the client asked for. Here it +# is what the expression resolves to, so the punctuation is written +# in it - and swe7 puts national letters where the brackets and the +# braces belong. What is written is therefore not a document, and +# JSON_VALID() answers 0 for it. +# +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS h FROM t1; +h +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +SELECT CHARSET(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS cs FROM t1; +cs +swe7 +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid FROM t1; +valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +SELECT g, JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid +FROM t1 GROUP BY g ORDER BY g; +g valid +1 NULL +2 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING swe7), +CONVERT(a USING swe7))) AS h FROM t1; +h +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_objectagg' at position 1 +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING swe7), +CONVERT(a USING swe7))) AS valid FROM t1; +valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_objectagg' at position 1 +DROP TABLE t1, t2; +SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_agg_charset.test b/mysql-test/main/func_json_agg_charset.test new file mode 100644 index 0000000000000..4587ddd4e63da --- /dev/null +++ b/mysql-test/main/func_json_agg_charset.test @@ -0,0 +1,213 @@ +--source include/have_utf8mb4.inc +--source include/have_ucs2.inc +--source include/have_utf16.inc +--source include/have_utf32.inc + +--echo # +--echo # How the JSON aggregates deliver their result to the client. +--echo # +--echo # A result computed in one character set and asked for in another is +--echo # converted on the way out. Two separate questions decide what the +--echo # client is left holding: which character set the result was computed +--echo # in, and whether what is in it survives the conversion to the one +--echo # that was asked for. +--echo # +--echo # swe7 is the vehicle for the first of those. It has no code point +--echo # for the brackets and braces a JSON document is built from, so a +--echo # result converted into it arrives with those replaced by '?'. That +--echo # is what a conversion having happened looks like, and it is visible +--echo # without a single non-ASCII byte in this file. A client asking for +--echo # latin1 sees nothing to remark on, latin1 having the punctuation: +--echo # the conversion happens just the same and the result arrives intact. +--echo # +--echo # The later sections ask the other question, leaving the client alone +--echo # and computing the result in a character set of its own. There the +--echo # punctuation has to go down at the width that character set asks +--echo # for, and where the character set has no punctuation to write, what +--echo # is written is not a document at all. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES (1, 'aa'), (1, 'bb'), (2, 'cc'), (2, 'dd'); + +--echo # +--echo # 1. An aggregate delivered next to the functions that are not. +--echo # + +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('aa','bb') AS v; +SELECT JSON_OBJECT('aa','bb') AS v; +SELECT JSON_SET('{}','$.a','b') AS v; +SELECT JSON_KEYS('{"aa":1}') AS v; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +--echo # one group per row of output, so the delivery is repeated +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +SELECT g, JSON_OBJECTAGG(a,a) AS v FROM t1 GROUP BY g ORDER BY g; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 2. The value itself, which the delivery does not touch. +--echo # +--echo # Read inside the server rather than sent to the client, the aggregate +--echo # is the same either way. HEX() answers in ASCII, so the bytes can be +--echo # shown under both settings and compared. +--echo # + +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, + JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1; +SET SESSION character_set_results = swe7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS valid, + JSON_LENGTH(JSON_ARRAYAGG(a)) AS len FROM t1; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 3. What the function says it returns. +--echo # +--echo # CHARSET() and COLLATION() report what the expression was resolved to, +--echo # not what the returned value ended up labelled with, so neither of +--echo # them can see a difference in delivery. A probe built from these two +--echo # alone, or from ASCII values under a character set that can hold them, +--echo # observes nothing at all. +--echo # + +SELECT CHARSET(JSON_ARRAYAGG(a)) AS cs, COLLATION(JSON_ARRAYAGG(a)) AS co + FROM t1; +SELECT CHARSET(JSON_OBJECTAGG(a,a)) AS cs, COLLATION(JSON_OBJECTAGG(a,a)) AS co + FROM t1; +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 4. Stored rather than delivered. +--echo # +--echo # A value on its way into a column is converted from the character set +--echo # the expression was resolved to, which is a separate question from the +--echo # one above. +--echo # + +CREATE TABLE t2 (v VARCHAR(50) CHARACTER SET latin1); +INSERT INTO t2 SELECT JSON_ARRAYAGG(a) FROM t1; +INSERT INTO t2 SELECT JSON_OBJECTAGG(a,a) FROM t1; +SELECT v, HEX(v) AS h FROM t2 ORDER BY v; +SET SESSION character_set_results = swe7; +SELECT v FROM t2 ORDER BY v; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 5. The other ways into the same result. +--echo # + +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1; +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1; +--echo # a group of one writes no separator, so it exercises less than it looks +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'aa'; +--echo # no rows at all: NULL, which is never delivered as a string +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +SELECT JSON_OBJECTAGG(a,a) AS v FROM t1 WHERE g = 99; +SET SESSION character_set_results = DEFAULT; + +--echo # +--echo # 6. A result computed in a character set that ASCII is not a part of. +--echo # +--echo # Every character of such a result is two or four bytes wide, the +--echo # brackets the aggregate puts around the elements included. Written +--echo # as single bytes those two would leave everything between them one +--echo # byte out of step, and the result would read as an entirely +--echo # different set of characters rather than as a document. +--echo # + +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS valid FROM t1; +SELECT CONVERT(JSON_ARRAYAGG(CONVERT(a USING ucs2)) USING utf8mb4) AS v FROM t1; +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf16))) AS valid FROM t1; +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING utf32))) AS valid FROM t1; +--echo # a group of one, which is bracket, element, bracket and nothing else +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING ucs2))) AS h FROM t1 WHERE a = 'aa'; +--echo # the same values where one byte is one character, which is the case +--echo # the brackets were always written for +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING latin1))) AS h FROM t1; +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING utf8mb4))) AS h FROM t1; +--echo # +--echo # 7. The sibling aggregate, in the same character sets. +--echo # +--echo # It reaches the same question by another route. Its opening brace +--echo # is not written next to its closing one: the closing brace goes on +--echo # when the result is asked for, and the opening brace has to be there +--echo # before the first pair is added. Where that brace is written decides +--echo # whether it can be written in the character set the result is in, and +--echo # whether the two ends of the object match. +--echo # + +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h + FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid FROM t1; +SELECT CONVERT(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2)) USING utf8mb4) AS v + FROM t1; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf16), CONVERT(a USING utf16))) AS h + FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf16), + CONVERT(a USING utf16))) AS valid FROM t1; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf32), CONVERT(a USING utf32))) AS h + FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING utf32), + CONVERT(a USING utf32))) AS valid FROM t1; +--echo # one pair, where no separator is written between pairs at all +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2))) AS h + FROM t1 WHERE a = 'aa'; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid + FROM t1 WHERE a = 'aa'; +--echo # a group each, so the opening brace is written more than once +SELECT g, HEX(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS h + FROM t1 GROUP BY g ORDER BY g; +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid + FROM t1 GROUP BY g ORDER BY g; +--echo # a copy of the aggregate, which is made without fixing it again +SELECT g, JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING ucs2), + CONVERT(a USING ucs2))) AS valid + FROM t1 GROUP BY g WITH ROLLUP; +--echo # no rows, which is a NULL and never an empty object +SELECT JSON_OBJECTAGG(CONVERT(a USING ucs2), CONVERT(a USING ucs2)) AS v + FROM t1 WHERE g = 99; +--echo # and where one byte is one character, which must not move +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING latin1), + CONVERT(a USING latin1))) AS h FROM t1; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING utf8mb4), + CONVERT(a USING utf8mb4))) AS h FROM t1; +SELECT g, HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1 GROUP BY g ORDER BY g; + +--echo # +--echo # 8. swe7 as the character set the aggregate is resolved to. +--echo # +--echo # Everywhere above, swe7 is only what the client asked for. Here it +--echo # is what the expression resolves to, so the punctuation is written +--echo # in it - and swe7 puts national letters where the brackets and the +--echo # braces belong. What is written is therefore not a document, and +--echo # JSON_VALID() answers 0 for it. +--echo # +SELECT HEX(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS h FROM t1; +SELECT CHARSET(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS cs FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid FROM t1; +SELECT g, JSON_VALID(JSON_ARRAYAGG(CONVERT(a USING swe7))) AS valid + FROM t1 GROUP BY g ORDER BY g; +SELECT HEX(JSON_OBJECTAGG(CONVERT(a USING swe7), + CONVERT(a USING swe7))) AS h FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(a USING swe7), + CONVERT(a USING swe7))) AS valid FROM t1; + +DROP TABLE t1, t2; +SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_agg_cut.result b/mysql-test/main/func_json_agg_cut.result new file mode 100644 index 0000000000000..eee842a08956e --- /dev/null +++ b/mysql-test/main/func_json_agg_cut.result @@ -0,0 +1,1007 @@ +# +# A group too long for group_concat_max_len is cut back to the last +# whole element or pair, so what is returned is still a document. +# Both aggregates honour the setting and both say so with a warning. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES +(1, 'aaaa'), (1, 'bbbb'), (1, 'cccc'), (1, 'dddd'), +(2, 'ee'), (2, 'ff'); +# +# 1. The cut walked across every offset within an element. The uncut +# group 1 is ["aaaa","bbbb","cccc","dddd"], 29 bytes. +# +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 1; +v +["aaaa","bbbb","cccc","dddd"] +SET SESSION group_concat_max_len = 3; +Warnings: +Warning 1292 Truncated incorrect group_concat_max_len value: '3' +SELECT 3 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +3 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +4 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +5 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +6 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +7 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +8 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +9 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +10 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +11 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +12 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +13 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +14 ["aaaa"] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +15 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +16 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +17 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +18 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +cap v ok +19 ["aaaa","bbbb"] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 2. The same walk for the object, which used to ignore the setting +# altogether. The uncut group 1 is +# {"aaaa":"aaaa", "bbbb":"bbbb", "cccc":"cccc", "dddd":"dddd"}. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 1; +v +{"aaaa":"aaaa", "bbbb":"bbbb", "cccc":"cccc", "dddd":"dddd"} +SET SESSION group_concat_max_len = 3; +Warnings: +Warning 1292 Truncated incorrect group_concat_max_len value: '3' +SELECT 3 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +3 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +4 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +5 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +6 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +7 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +8 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +9 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +10 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +11 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +12 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +13 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +14 {} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +15 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +16 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +17 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +18 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +cap v ok +19 {"aaaa":"aaaa"} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 3. A cap smaller than the first element leaves an empty container +# rather than half of one. +# +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +v ok +[] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SELECT JSON_OBJECTAGG(a, a) AS v, +JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +v ok +{} 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 4. Elements that are not quoted strings. +# +CREATE TABLE t2 (n INT, j VARCHAR(20)); +INSERT INTO t2 VALUES (1111, '{"k":1}'), (2222, '{"k":2}'), (3333, '{"k":3}'); +# numbers, where the old cut left a trailing separator +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +4 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +5 [] 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +6 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +7 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +8 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +9 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +10 [1111] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +11 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +12 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +13 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +14 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +15 [1111,2222] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; +cap v ok +16 [1111,2222,3333] 1 +SET SESSION group_concat_max_len = DEFAULT; +# documents, where the old cut landed inside an object +SET SESSION group_concat_max_len = 12; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +v ok +[{"k":1}] 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, +JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +v ok +[{"k":1},{"k":2}] 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT JSON_OBJECTAGG(n, JSON_COMPACT(j)) AS v, +JSON_VALID(JSON_OBJECTAGG(n, JSON_COMPACT(j))) AS ok FROM t2; +v ok +{"1111":{"k":1}} 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; +# +# 5. A cut group and an uncut group in the same statement, and the +# row the warning names. +# +SET SESSION group_concat_max_len = 14; +SELECT g, JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ["aaaa"] +2 ["ee","ff"] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 {} +2 {"ee":"ee"} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 6. DISTINCT and ORDER BY inside the array, which reach the cut by +# a different route. +# +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(DISTINCT a ORDER BY a) AS v FROM t1 WHERE g = 1; +v +["aaaa"] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +v +["dddd"] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 7. A cut result read back by another JSON function. +# +SET SESSION group_concat_max_len = 14; +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a ORDER BY a), '$[0]') AS v FROM t1 WHERE g = 1; +v +"aaaa" +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +v +1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_ARRAY(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +v +[["aaaa"]] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_LENGTH(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +v +0 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SELECT JSON_ARRAY(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +v +[{}] +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 8. Characters wider than a byte. The cap counts bytes, and an +# element boundary is a boundary whatever the characters cost. +# +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t3 VALUES (_utf8mb4 X'C3A4C3A4'), (_utf8mb4 X'C3B6C3B6'); +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +h ok +5B22C3A4C3A4225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, +JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_OBJECTAGG(a, 1)) AS h, +JSON_VALID(JSON_OBJECTAGG(a, 1)) AS ok FROM t3; +h ok +7B7D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; +# +# 9. Nothing to cut: a group that fits raises no warning, and an +# empty group is unaffected. +# +SET SESSION group_concat_max_len = 1024; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 2; +v +["ee","ff"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +v +{"ee":"ee", "ff":"ff"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +v +NULL +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +v +NULL +SET SESSION group_concat_max_len = DEFAULT; +# +# 10. A NULL key is not a pair of the object, so it is not the row +# the warning names either. +# +CREATE TABLE t4 (k VARCHAR(20), v INT); +INSERT INTO t4 VALUES (NULL, 1), ('kkkk', 2), ('llll', 3), ('mmmm', 4); +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, v) AS v FROM t4; +v +{"kkkk":2} +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; +DROP TABLE t1; +# +# 11. A cut group is still read to the end, but takes nothing +# further, so the object evaluates neither of its arguments +# for the rows after the cut. Each shape below is run twice +# over the same rows, once under a cap that cuts and once +# under one that does not, and counts what the argument did. +# +# The array counts the same either way: it discovers the cut +# when the group is asked for, by which time every row has +# been read, so there is no row left to skip. The object +# knows while the rows are still coming, and its count falls +# to the rows up to the cut. +# +# Nothing documents how many times an aggregate evaluates its +# arguments, so an argument that does something besides +# produce a value does that thing for the rows before the cut +# and not for the rows after it. +# +CREATE TABLE t5 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t5 VALUES +('kkkk', 'aaaa'), ('llll', 'bbbb'), ('mmmm', 'cccc'), ('nnnn', 'dddd'); +CREATE FUNCTION seen_once(x VARCHAR(20)) RETURNS VARCHAR(20) NO SQL +BEGIN +SET @calls= @calls + 1; +RETURN x; +END// +# the array, which reads every row of the group already +SET @calls= 0; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +v +["aaaa","bbbb","cccc","dddd"] +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +v +["aaaa"] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +4 4 +# the object, on the value of the pair +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +v +{"kkkk":"aaaa", "llll":"bbbb", "mmmm":"cccc", "nnnn":"dddd"} +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +v +{} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +4 1 +# the object, on the key. The object also asks the key whether it +# is NULL, and asking that of a function is running it again, so +# the count is two a row - for the rows it reaches. +SET @calls= 0; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +v +{"kkkk":"aaaa", "llll":"bbbb", "mmmm":"cccc", "nnnn":"dddd"} +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +v +{} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +8 2 +# a row whose key is NULL makes no pair, so its value is not read +# for it either, which is a second reason not to read one +CREATE TABLE t6 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t6 VALUES +('kkkk', 'aaaa'), ('llll', 'bbbb'), (NULL, 'cccc'), ('nnnn', 'dddd'); +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +v +{"kkkk":"aaaa", "llll":"bbbb", "nnnn":"dddd"} +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +v +{} +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +uncut_calls cut_calls +3 1 +DROP TABLE t6; +DROP FUNCTION seen_once; +DROP TABLE t5; +# +# 12. The limit is on the bytes returned, and the brace that +# closes the object is one of them. It goes on after the +# length has been tested, so the room for it has to be kept +# back when the test is made. +# +CREATE TABLE t7 (a VARCHAR(20)); +INSERT INTO t7 VALUES ('aaaa'), ('bbbb'), ('cccc'); +SET SESSION group_concat_max_len = 4; +SELECT 4 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 4 AS within FROM t7; +cap v len within +4 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 5; +SELECT 5 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 5 AS within FROM t7; +cap v len within +5 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 6; +SELECT 6 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 6 AS within FROM t7; +cap v len within +6 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 7; +SELECT 7 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 7 AS within FROM t7; +cap v len within +7 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 8; +SELECT 8 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 8 AS within FROM t7; +cap v len within +8 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 9; +SELECT 9 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 9 AS within FROM t7; +cap v len within +9 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 10; +SELECT 10 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 10 AS within FROM t7; +cap v len within +10 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 11; +SELECT 11 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 11 AS within FROM t7; +cap v len within +11 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 12 AS within FROM t7; +cap v len within +12 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 13 AS within FROM t7; +cap v len within +13 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 14 AS within FROM t7; +cap v len within +14 {} 2 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 15 AS within FROM t7; +cap v len within +15 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 16 AS within FROM t7; +cap v len within +16 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 17 AS within FROM t7; +cap v len within +17 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 18 AS within FROM t7; +cap v len within +18 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 19 AS within FROM t7; +cap v len within +19 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 20 AS within FROM t7; +cap v len within +20 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 21 AS within FROM t7; +cap v len within +21 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 22; +SELECT 22 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 22 AS within FROM t7; +cap v len within +22 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 23; +SELECT 23 AS cap, JSON_OBJECTAGG(a, a) AS v, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 23 AS within FROM t7; +cap v len within +23 {"aaaa":"aaaa"} 15 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +Warning 1260 Row 2 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t7; +# A set that writes no character in one byte: the brace costs two +# bytes there, so two is what has to be kept back. +CREATE TABLE t8 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t8 VALUES ('aa'), ('bb'); +SET SESSION group_concat_max_len = 12; +SELECT 12 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 12 AS within FROM t8; +cap h len within +12 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 13; +SELECT 13 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 13 AS within FROM t8; +cap h len within +13 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 14; +SELECT 14 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 14 AS within FROM t8; +cap h len within +14 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 15; +SELECT 15 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 15 AS within FROM t8; +cap h len within +15 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 16; +SELECT 16 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 16 AS within FROM t8; +cap h len within +16 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 17; +SELECT 17 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 17 AS within FROM t8; +cap h len within +17 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 18; +SELECT 18 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 18 AS within FROM t8; +cap h len within +18 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 19; +SELECT 19 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 19 AS within FROM t8; +cap h len within +19 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 20; +SELECT 20 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 20 AS within FROM t8; +cap h len within +20 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = 21; +SELECT 21 AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, +LENGTH(JSON_OBJECTAGG(a, a)) AS len, +LENGTH(JSON_OBJECTAGG(a, a)) <= 21 AS within FROM t8; +cap h len within +21 007B007D 4 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t8; diff --git a/mysql-test/main/func_json_agg_cut.test b/mysql-test/main/func_json_agg_cut.test new file mode 100644 index 0000000000000..b93533f603413 --- /dev/null +++ b/mysql-test/main/func_json_agg_cut.test @@ -0,0 +1,294 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # A group too long for group_concat_max_len is cut back to the last +--echo # whole element or pair, so what is returned is still a document. +--echo # Both aggregates honour the setting and both say so with a warning. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES + (1, 'aaaa'), (1, 'bbbb'), (1, 'cccc'), (1, 'dddd'), + (2, 'ee'), (2, 'ff'); + +--echo # +--echo # 1. The cut walked across every offset within an element. The uncut +--echo # group 1 is ["aaaa","bbbb","cccc","dddd"], 29 bytes. +--echo # + +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 1; + +--let $cap= 3 +while ($cap < 20) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_ARRAYAGG(a ORDER BY a) AS v, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 2. The same walk for the object, which used to ignore the setting +--echo # altogether. The uncut group 1 is +--echo # {"aaaa":"aaaa", "bbbb":"bbbb", "cccc":"cccc", "dddd":"dddd"}. +--echo # + +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 1; + +--let $cap= 3 +while ($cap < 20) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_OBJECTAGG(a, a) AS v, + JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 3. A cap smaller than the first element leaves an empty container +--echo # rather than half of one. +--echo # + +SET SESSION group_concat_max_len = 4; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t1 WHERE g = 1; +SELECT JSON_OBJECTAGG(a, a) AS v, + JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 4. Elements that are not quoted strings. +--echo # + +CREATE TABLE t2 (n INT, j VARCHAR(20)); +INSERT INTO t2 VALUES (1111, '{"k":1}'), (2222, '{"k":2}'), (3333, '{"k":3}'); + +--echo # numbers, where the old cut left a trailing separator +--let $cap= 4 +while ($cap < 17) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_ARRAYAGG(n ORDER BY n) AS v, + JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t2; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; + +--echo # documents, where the old cut landed inside an object +SET SESSION group_concat_max_len = 12; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, + JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +SET SESSION group_concat_max_len = 17; +SELECT JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n) AS v, + JSON_VALID(JSON_ARRAYAGG(JSON_COMPACT(j) ORDER BY n)) AS ok FROM t2; +SET SESSION group_concat_max_len = 20; +SELECT JSON_OBJECTAGG(n, JSON_COMPACT(j)) AS v, + JSON_VALID(JSON_OBJECTAGG(n, JSON_COMPACT(j))) AS ok FROM t2; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 5. A cut group and an uncut group in the same statement, and the +--echo # row the warning names. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT g, JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 GROUP BY g ORDER BY g; +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 6. DISTINCT and ORDER BY inside the array, which reach the cut by +--echo # a different route. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(DISTINCT a ORDER BY a) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 7. A cut result read back by another JSON function. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a ORDER BY a), '$[0]') AS v FROM t1 WHERE g = 1; +SELECT JSON_LENGTH(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(a ORDER BY a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_LENGTH(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAY(JSON_OBJECTAGG(a, a)) AS v FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 8. Characters wider than a byte. The cap counts bytes, and an +--echo # element boundary is a boundary whatever the characters cost. +--echo # + +CREATE TABLE t3 (a VARCHAR(20)) CHARACTER SET utf8mb4; +INSERT INTO t3 VALUES (_utf8mb4 X'C3A4C3A4'), (_utf8mb4 X'C3B6C3B6'); +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a ORDER BY a)) AS h, + JSON_VALID(JSON_ARRAYAGG(a ORDER BY a)) AS ok FROM t3; +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_OBJECTAGG(a, 1)) AS h, + JSON_VALID(JSON_OBJECTAGG(a, 1)) AS ok FROM t3; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; + +--echo # +--echo # 9. Nothing to cut: a group that fits raises no warning, and an +--echo # empty group is unaffected. +--echo # + +SET SESSION group_concat_max_len = 1024; +SELECT JSON_ARRAYAGG(a ORDER BY a) AS v FROM t1 WHERE g = 2; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 10. A NULL key is not a pair of the object, so it is not the row +--echo # the warning names either. +--echo # + +CREATE TABLE t4 (k VARCHAR(20), v INT); +INSERT INTO t4 VALUES (NULL, 1), ('kkkk', 2), ('llll', 3), ('mmmm', 4); +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, v) AS v FROM t4; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t4; + +DROP TABLE t1; + +--echo # +--echo # 11. A cut group is still read to the end, but takes nothing +--echo # further, so the object evaluates neither of its arguments +--echo # for the rows after the cut. Each shape below is run twice +--echo # over the same rows, once under a cap that cuts and once +--echo # under one that does not, and counts what the argument did. +--echo # +--echo # The array counts the same either way: it discovers the cut +--echo # when the group is asked for, by which time every row has +--echo # been read, so there is no row left to skip. The object +--echo # knows while the rows are still coming, and its count falls +--echo # to the rows up to the cut. +--echo # +--echo # Nothing documents how many times an aggregate evaluates its +--echo # arguments, so an argument that does something besides +--echo # produce a value does that thing for the rows before the cut +--echo # and not for the rows after it. +--echo # + +CREATE TABLE t5 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t5 VALUES + ('kkkk', 'aaaa'), ('llll', 'bbbb'), ('mmmm', 'cccc'), ('nnnn', 'dddd'); + +DELIMITER //; +CREATE FUNCTION seen_once(x VARCHAR(20)) RETURNS VARCHAR(20) NO SQL +BEGIN + SET @calls= @calls + 1; + RETURN x; +END// +DELIMITER ;// + +# The counter below records how often the function was called rather than +# what the query answered, so a statement run a second time to check that it +# repeats itself would count twice. +--disable_ps2_protocol + +--echo # the array, which reads every row of the group already +SET @calls= 0; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_ARRAYAGG(seen_once(v) ORDER BY v) AS v FROM t5; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; + +--echo # the object, on the value of the pair +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t5; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; + +--echo # the object, on the key. The object also asks the key whether it +--echo # is NULL, and asking that of a function is running it again, so +--echo # the count is two a row - for the rows it reaches. +SET @calls= 0; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(seen_once(k), v) AS v FROM t5; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; + +--echo # a row whose key is NULL makes no pair, so its value is not read +--echo # for it either, which is a second reason not to read one +CREATE TABLE t6 (k VARCHAR(20), v VARCHAR(20)); +INSERT INTO t6 VALUES + ('kkkk', 'aaaa'), ('llll', 'bbbb'), (NULL, 'cccc'), ('nnnn', 'dddd'); +SET @calls= 0; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +SET @uncut= @calls; +SET @calls= 0; +SET SESSION group_concat_max_len = 14; +SELECT JSON_OBJECTAGG(k, seen_once(v)) AS v FROM t6; +SET SESSION group_concat_max_len = DEFAULT; +SELECT @uncut AS uncut_calls, @calls AS cut_calls; +DROP TABLE t6; + +DROP FUNCTION seen_once; +DROP TABLE t5; +--enable_ps2_protocol + +--echo # +--echo # 12. The limit is on the bytes returned, and the brace that +--echo # closes the object is one of them. It goes on after the +--echo # length has been tested, so the room for it has to be kept +--echo # back when the test is made. +--echo # + +CREATE TABLE t7 (a VARCHAR(20)); +INSERT INTO t7 VALUES ('aaaa'), ('bbbb'), ('cccc'); + +--let $cap= 4 +while ($cap < 24) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, JSON_OBJECTAGG(a, a) AS v, + LENGTH(JSON_OBJECTAGG(a, a)) AS len, + LENGTH(JSON_OBJECTAGG(a, a)) <= $cap AS within FROM t7; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t7; + +--echo # A set that writes no character in one byte: the brace costs two +--echo # bytes there, so two is what has to be kept back. +CREATE TABLE t8 (a VARCHAR(20)) CHARACTER SET ucs2; +INSERT INTO t8 VALUES ('aa'), ('bb'); + +--let $cap= 12 +while ($cap < 22) +{ + eval SET SESSION group_concat_max_len = $cap; + eval SELECT $cap AS cap, HEX(JSON_OBJECTAGG(a, a)) AS h, + LENGTH(JSON_OBJECTAGG(a, a)) AS len, + LENGTH(JSON_OBJECTAGG(a, a)) <= $cap AS within FROM t8; + --inc $cap +} +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t8; diff --git a/mysql-test/main/func_json_agg_limits.result b/mysql-test/main/func_json_agg_limits.result new file mode 100644 index 0000000000000..0199fa85fdf06 --- /dev/null +++ b/mysql-test/main/func_json_agg_limits.result @@ -0,0 +1,374 @@ +# +# Behavioral baseline: the JSON aggregate functions. +# +# Both aggregates are held to group_concat_max_len, and a group too +# long for it is cut back to the last whole element or pair, so what +# comes back is still a document wherever the cap falls. This test +# walks the cap across every interesting byte offset and records the +# exact result each time. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES +(1, 'aaaaaaaaaa'), (1, 'bbbbbbbbbb'), (1, 'cccccccccc'), +(2, 'dd'), (2, 'ee'); +# +# 1. Uncut results. +# +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +["aaaaaaaaaa","bbbbbbbbbb","cccccccccc","dd","ee"] +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +h +5B2261616161616161616161222C2262626262626262626262222C2263636363636363636363222C226464222C226565225D +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1; +ok +1 +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"] +2 ["dd","ee"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +v +{"aaaaaaaaaa":"aaaaaaaaaa", "bbbbbbbbbb":"bbbbbbbbbb", "cccccccccc":"cccccccccc", "dd":"dd", "ee":"ee"} +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1; +h +7B2261616161616161616161223A2261616161616161616161222C202262626262626262626262223A2262626262626262626262222C202263636363636363636363223A2263636363636363636363222C20226464223A226464222C20226565223A226565227D +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1; +ok +1 +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +g v +1 {"aaaaaaaaaa":"aaaaaaaaaa", "bbbbbbbbbb":"bbbbbbbbbb", "cccccccccc":"cccccccccc"} +2 {"dd":"dd", "ee":"ee"} +# the aggregates over an empty input +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +v +NULL +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +v +NULL +# a single row +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'dd'; +v +["dd"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'dd'; +v +{"dd":"dd"} +# NULL and non-string members +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT NULL) d; +v +[1,null] +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +v +[1,2] +SELECT JSON_OBJECTAGG(k, x) AS v +FROM (SELECT 'a' AS k, 1 AS x UNION ALL SELECT 'b', NULL) d; +v +{"a":1, "b":null} +# DISTINCT and ORDER BY inside the aggregate +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1 WHERE g = 1; +v +["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"] +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +v +["cccccccccc","bbbbbbbbbb","aaaaaaaaaa"] +# an aggregate over documents rather than strings +SELECT JSON_ARRAYAGG(JSON_OBJECT('k', a)) AS v FROM t1 WHERE g = 2; +v +[{"k": "dd"},{"k": "ee"}] +SELECT JSON_ARRAYAGG(JSON_COMPACT(CONCAT('{"k":"', a, '"}'))) AS v +FROM t1 WHERE g = 2; +v +[{"k":"dd"},{"k":"ee"}] +# +# 2. The cap, walked byte by byte. The uncut result for group 1 is +# ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"], 40 bytes. +# +SET SESSION group_concat_max_len = 40; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C2262626262626262626262222C2263636363636363636363225D 1 +SET SESSION group_concat_max_len = 39; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C2262626262626262626262225D 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 30; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161222C2262626262626262626262225D 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 20; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# the cap falls just after the separator +SET SESSION group_concat_max_len = 15; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# the cap falls on the opening quote of the next member +SET SESSION group_concat_max_len = 14; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B2261616161616161616161225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# the cap falls on the separator itself +SET SESSION group_concat_max_len = 13; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +# the cap falls inside the first member +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 WHERE g = 1; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +# the smallest the cap can be +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1 WHERE a = 'dd'; +h +5B5D +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 3. A cap that falls inside an escape sequence, where the bytes +# after it would not be a complete character. +# +CREATE TABLE t2 (a VARCHAR(20)); +INSERT INTO t2 VALUES ('xx"yy'), ('zzzz'); +SET SESSION group_concat_max_len = 10; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B2278785C227979225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 6; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +h ok +5B5D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +Warning 1260 Row 1 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; +# +# 3a. A cap over members that are not quoted strings, where there is +# no quote for a cut to land after and nothing to re-close. +# +CREATE TABLE t3 (n INT, j JSON); +INSERT INTO t3 VALUES (111, '{"k":1}'), (222, '{"k":2}'), (333, '{"k":3}'); +# numeric members +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +h ok +5B3131312C3232325D 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +h ok +5B3131315D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# document members, where the cap falls inside an object +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +h ok +5B7B226B223A317D5D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = 17; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +h ok +5B7B226B223A317D2C7B226B223A327D5D 1 +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +# members built by a constructor rather than read from a column +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS h, +JSON_VALID(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS ok FROM t3; +h ok +5B7B226B223A203131317D5D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; +# +# 4. A cut result consumed by another JSON function. The cut value +# is passed on as a document argument like any other. +# +SET SESSION group_concat_max_len = 14; +SELECT JSON_SET(JSON_ARRAYAGG(a), '$[0]', 1) AS v FROM t1 WHERE g = 1; +v +[1] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a), '$[0]') AS v FROM t1 WHERE g = 1; +v +"aaaaaaaaaa" +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +[["aaaaaaaaaa"]] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_DEPTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +2 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_LENGTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SELECT JSON_COMPACT(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +v +["aaaaaaaaaa"] +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +# an uncut group and a cut group in the same statement +SET SESSION group_concat_max_len = 14; +SELECT g, HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok +FROM t1 GROUP BY g ORDER BY g; +g h ok +1 5B2261616161616161616161225D 1 +2 5B226464222C226565225D 1 +Warnings: +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +Warning 1260 Row 2 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = DEFAULT; +# +# 5. JSON_OBJECTAGG under the same cap. +# +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h, JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok +FROM t1 WHERE g = 1; +h ok +7B7D 1 +Warnings: +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +Warning 1260 Row 1 was cut by JSON_OBJECTAGG() +SET SESSION group_concat_max_len = DEFAULT; +# duplicate keys are kept, not merged +SELECT JSON_OBJECTAGG(g, a) AS v FROM t1 WHERE g = 1; +v +{"1":"aaaaaaaaaa", "1":"bbbbbbbbbb", "1":"cccccccccc"} +SELECT JSON_VALID(JSON_OBJECTAGG(g, a)) AS ok FROM t1 WHERE g = 1; +ok +1 +# a key that needs escaping +SELECT JSON_OBJECTAGG(k, 1) AS v +FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +v +{"a\"b":1, "c\\d":1} +SELECT JSON_VALID(JSON_OBJECTAGG(k, 1)) AS ok +FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +ok +1 +# a NULL key +SELECT JSON_OBJECTAGG(k, 1) AS v +FROM (SELECT NULL AS k UNION ALL SELECT 'b') d; +v +{"b":1} +# +# 6. The result metadata the aggregates report, which decides how the +# bytes are converted on their way to the client. +# +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 4194312 11 Y 0 0 45 +v +["dd","ee"] +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 4194312 22 Y 0 0 45 +v +{"dd":"dd", "ee":"ee"} +SELECT JSON_ARRAY(1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 40 3 Y 0 39 45 +v +[1] +SELECT JSON_SET('{}', '$.a', 1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 84 8 Y 0 39 45 +v +{"a": 1} +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_limits.test b/mysql-test/main/func_json_agg_limits.test new file mode 100644 index 0000000000000..61331336691c5 --- /dev/null +++ b/mysql-test/main/func_json_agg_limits.test @@ -0,0 +1,199 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: the JSON aggregate functions. +--echo # +--echo # Both aggregates are held to group_concat_max_len, and a group too +--echo # long for it is cut back to the last whole element or pair, so what +--echo # comes back is still a document wherever the cap falls. This test +--echo # walks the cap across every interesting byte offset and records the +--echo # exact result each time. +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 (g INT, a VARCHAR(20)); +INSERT INTO t1 VALUES + (1, 'aaaaaaaaaa'), (1, 'bbbbbbbbbb'), (1, 'cccccccccc'), + (2, 'dd'), (2, 'ee'); + +--echo # +--echo # 1. Uncut results. +--echo # + +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1; +SELECT JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t1; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g ORDER BY g; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h FROM t1; +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok FROM t1; +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g ORDER BY g; +--echo # the aggregates over an empty input +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99; +--echo # a single row +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'dd'; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'dd'; +--echo # NULL and non-string members +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT NULL) d; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +SELECT JSON_OBJECTAGG(k, x) AS v + FROM (SELECT 'a' AS k, 1 AS x UNION ALL SELECT 'b', NULL) d; +--echo # DISTINCT and ORDER BY inside the aggregate +SELECT JSON_ARRAYAGG(DISTINCT a) AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAYAGG(a ORDER BY a DESC) AS v FROM t1 WHERE g = 1; +--echo # an aggregate over documents rather than strings +SELECT JSON_ARRAYAGG(JSON_OBJECT('k', a)) AS v FROM t1 WHERE g = 2; +SELECT JSON_ARRAYAGG(JSON_COMPACT(CONCAT('{"k":"', a, '"}'))) AS v + FROM t1 WHERE g = 2; + +--echo # +--echo # 2. The cap, walked byte by byte. The uncut result for group 1 is +--echo # ["aaaaaaaaaa","bbbbbbbbbb","cccccccccc"], 40 bytes. +--echo # + +SET SESSION group_concat_max_len = 40; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 39; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 30; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 20; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cap falls just after the separator +SET SESSION group_concat_max_len = 15; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cap falls on the opening quote of the next member +SET SESSION group_concat_max_len = 14; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cap falls on the separator itself +SET SESSION group_concat_max_len = 13; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the cap falls inside the first member +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 WHERE g = 1; +--echo # the smallest the cap can be +SET SESSION group_concat_max_len = 4; +SELECT HEX(JSON_ARRAYAGG(a)) AS h FROM t1 WHERE a = 'dd'; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 3. A cap that falls inside an escape sequence, where the bytes +--echo # after it would not be a complete character. +--echo # + +CREATE TABLE t2 (a VARCHAR(20)); +INSERT INTO t2 VALUES ('xx"yy'), ('zzzz'); +SET SESSION group_concat_max_len = 10; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 8; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 7; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = 6; +SELECT HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok FROM t2; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 3a. A cap over members that are not quoted strings, where there is +--echo # no quote for a cut to land after and nothing to re-close. +--echo # + +CREATE TABLE t3 (n INT, j JSON); +INSERT INTO t3 VALUES (111, '{"k":1}'), (222, '{"k":2}'), (333, '{"k":3}'); +--echo # numeric members +SET SESSION group_concat_max_len = 9; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_ARRAYAGG(n ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(n ORDER BY n)) AS ok FROM t3; +--echo # document members, where the cap falls inside an object +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +SET SESSION group_concat_max_len = 17; +SELECT HEX(JSON_ARRAYAGG(j ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(j ORDER BY n)) AS ok FROM t3; +--echo # members built by a constructor rather than read from a column +SET SESSION group_concat_max_len = 12; +SELECT HEX(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS h, + JSON_VALID(JSON_ARRAYAGG(JSON_OBJECT('k', n) ORDER BY n)) AS ok FROM t3; +SET SESSION group_concat_max_len = DEFAULT; +DROP TABLE t3; + +--echo # +--echo # 4. A cut result consumed by another JSON function. The cut value +--echo # is passed on as a document argument like any other. +--echo # + +SET SESSION group_concat_max_len = 14; +SELECT JSON_SET(JSON_ARRAYAGG(a), '$[0]', 1) AS v FROM t1 WHERE g = 1; +SELECT JSON_EXTRACT(JSON_ARRAYAGG(a), '$[0]') AS v FROM t1 WHERE g = 1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_DEPTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_LENGTH(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +SELECT JSON_COMPACT(JSON_ARRAYAGG(a)) AS v FROM t1 WHERE g = 1; +--echo # an uncut group and a cut group in the same statement +SET SESSION group_concat_max_len = 14; +SELECT g, HEX(JSON_ARRAYAGG(a)) AS h, JSON_VALID(JSON_ARRAYAGG(a)) AS ok + FROM t1 GROUP BY g ORDER BY g; +SET SESSION group_concat_max_len = DEFAULT; + +--echo # +--echo # 5. JSON_OBJECTAGG under the same cap. +--echo # + +SET SESSION group_concat_max_len = 5; +SELECT HEX(JSON_OBJECTAGG(a, a)) AS h, JSON_VALID(JSON_OBJECTAGG(a, a)) AS ok + FROM t1 WHERE g = 1; +SET SESSION group_concat_max_len = DEFAULT; +--echo # duplicate keys are kept, not merged +SELECT JSON_OBJECTAGG(g, a) AS v FROM t1 WHERE g = 1; +SELECT JSON_VALID(JSON_OBJECTAGG(g, a)) AS ok FROM t1 WHERE g = 1; +--echo # a key that needs escaping +SELECT JSON_OBJECTAGG(k, 1) AS v + FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +SELECT JSON_VALID(JSON_OBJECTAGG(k, 1)) AS ok + FROM (SELECT 'a"b' AS k UNION ALL SELECT 'c\\d') d; +--echo # a NULL key +SELECT JSON_OBJECTAGG(k, 1) AS v + FROM (SELECT NULL AS k UNION ALL SELECT 'b') d; + +--echo # +--echo # 6. The result metadata the aggregates report, which decides how the +--echo # bytes are converted on their way to the client. +--echo # + +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 2; +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 2; +SELECT JSON_ARRAY(1) AS v; +SELECT JSON_SET('{}', '$.a', 1) AS v; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_pair_oom.result b/mysql-test/main/func_json_agg_pair_oom.result new file mode 100644 index 0000000000000..3d5187befd3fc --- /dev/null +++ b/mysql-test/main/func_json_agg_pair_oom.result @@ -0,0 +1,74 @@ +# +# A pair that could not be written at all, which is a different +# thing from one that would not fit. +# +# A pair taken off for the length limit leaves an object the brace +# can still go round, and the group is returned short of it. A +# pair the buffer would not take is written in part, and what is +# left is an object no brace can close, so the group is refused +# whole instead. +# +# Nothing reaches that second outcome by asking the server an +# ordinary question: the room a pair takes is asked for before it +# is written, so the writes never run short. Each of the three a +# pair is made of is therefore failed on its own here - the +# separator in front of it, the quote that opens its key, and the +# colon that closes it. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (k VARCHAR(16), v INT); +INSERT INTO t1 VALUES ('a', 1), ('b', 2), ('c', 3); +# +# 1. What the group answers with nothing wrong. More than one +# row, so the separator has a pair in front of it to be +# written after. +# +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; +whole +{"a":1, "b":2, "c":3} +# +# 2. The separator in front of a pair +# +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_separator FROM t1; +no_separator +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 3. The quote that opens the key +# +SET SESSION debug_dbug = '+d,json_objectagg_key_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_key_quote FROM t1; +no_key_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 4. The colon that closes it +# +SET SESSION debug_dbug = '+d,json_objectagg_colon_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_colon FROM t1; +no_colon +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 5. One row, where there is no separator to write, so that the +# first of the three is shown not to fire where the group has +# nothing in front of the pair +# +CREATE TABLE t2 (k VARCHAR(16), v INT); +INSERT INTO t2 VALUES ('a', 1); +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS one_pair FROM t2; +one_pair +{"a":1} +SET SESSION debug_dbug = DEFAULT; +DROP TABLE t2; +# +# 6. The group once the room is there again, which is what says +# the refusals above are about the writing rather than about +# the rows +# +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; +whole +{"a":1, "b":2, "c":3} +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_pair_oom.test b/mysql-test/main/func_json_agg_pair_oom.test new file mode 100644 index 0000000000000..8cb6396a507a8 --- /dev/null +++ b/mysql-test/main/func_json_agg_pair_oom.test @@ -0,0 +1,73 @@ +--source include/have_debug.inc + +--echo # +--echo # A pair that could not be written at all, which is a different +--echo # thing from one that would not fit. +--echo # +--echo # A pair taken off for the length limit leaves an object the brace +--echo # can still go round, and the group is returned short of it. A +--echo # pair the buffer would not take is written in part, and what is +--echo # left is an object no brace can close, so the group is refused +--echo # whole instead. +--echo # +--echo # Nothing reaches that second outcome by asking the server an +--echo # ordinary question: the room a pair takes is asked for before it +--echo # is written, so the writes never run short. Each of the three a +--echo # pair is made of is therefore failed on its own here - the +--echo # separator in front of it, the quote that opens its key, and the +--echo # colon that closes it. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE t1 (k VARCHAR(16), v INT); +INSERT INTO t1 VALUES ('a', 1), ('b', 2), ('c', 3); + +--echo # +--echo # 1. What the group answers with nothing wrong. More than one +--echo # row, so the separator has a pair in front of it to be +--echo # written after. +--echo # +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; + +--echo # +--echo # 2. The separator in front of a pair +--echo # +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_separator FROM t1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 3. The quote that opens the key +--echo # +SET SESSION debug_dbug = '+d,json_objectagg_key_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_key_quote FROM t1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 4. The colon that closes it +--echo # +SET SESSION debug_dbug = '+d,json_objectagg_colon_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS no_colon FROM t1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 5. One row, where there is no separator to write, so that the +--echo # first of the three is shown not to fire where the group has +--echo # nothing in front of the pair +--echo # +CREATE TABLE t2 (k VARCHAR(16), v INT); +INSERT INTO t2 VALUES ('a', 1); +SET SESSION debug_dbug = '+d,json_objectagg_separator_out_of_memory'; +SELECT JSON_OBJECTAGG(k, v) AS one_pair FROM t2; +SET SESSION debug_dbug = DEFAULT; +DROP TABLE t2; + +--echo # +--echo # 6. The group once the room is there again, which is what says +--echo # the refusals above are about the writing rather than about +--echo # the rows +--echo # +SELECT JSON_OBJECTAGG(k, v) AS whole FROM t1; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_reread.result b/mysql-test/main/func_json_agg_reread.result new file mode 100644 index 0000000000000..37bdf79392a36 --- /dev/null +++ b/mysql-test/main/func_json_agg_reread.result @@ -0,0 +1,123 @@ +# +# Asking for the result of a JSON aggregate more than once. +# +# A group is built up across three calls: clear() at its start, add() +# once per row, and val_str() when the result is wanted. The text +# accumulates in a buffer belonging to the item, there being nowhere +# else to keep it between rows, so anything val_str() writes into that +# buffer stays written. +# +# Nothing says the result is wanted only once. HAVING on the alias is +# the shortest statement that wants it twice; the number of times +# follows the number of conditions. +# +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1, 'x'), (1, 'y'), (2, 'z'), (2, 'w'); +# +# 1. Asked once. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +v +{"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1; +v +["x","y","z","w"] +# +# 2. Asked twice. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%'; +v +{"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%'; +v +["x","y","z","w"] +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS valid FROM t1 HAVING valid IS NOT NULL; +valid +1 +# +# 3. Asked three times. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '{%'; +v +{"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '[%'; +v +["x","y","z","w"] +# +# 4. Every group, not only the first. +# +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; +g v +1 ["x","y"] +2 ["z","w"] +# +# 5. A group of one row, where no separator is ever written. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; +v +{"x":"x"} +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; +v +["x"] +# +# 6. A copy of the aggregate, which WITH ROLLUP makes without fixing +# the item again. +# +# The condition is what makes this section a test of anything: a +# rollup level asked for once is closed once whether the flag is +# kept or not, and every other section here reaches the second +# asking through HAVING. The extra level is produced by a copy of +# the item, so it is that copy's own flag being reset per group +# that has to hold. +# +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%'; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +NULL {"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%'; +g v +1 ["x","y"] +2 ["z","w"] +NULL ["x","y","z","w"] +# two conditions, so each level is asked for a third time +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%' AND v LIKE '{%'; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +NULL {"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP +HAVING v LIKE '%' AND v LIKE '[%'; +g v +1 ["x","y"] +2 ["z","w"] +NULL ["x","y","z","w"] +# and the levels without a condition, which must not move either +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP; +g v +1 {"x":"x", "y":"y"} +2 {"z":"z", "w":"w"} +NULL {"x":"x", "y":"y", "z":"z", "w":"w"} +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP; +g v +1 ["x","y"] +2 ["z","w"] +NULL ["x","y","z","w"] +# +# 7. No rows, which is a NULL and stays one however often it is asked +# for. +# +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; +v +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; +v +DROP TABLE t1; diff --git a/mysql-test/main/func_json_agg_reread.test b/mysql-test/main/func_json_agg_reread.test new file mode 100644 index 0000000000000..ac50057eea6d9 --- /dev/null +++ b/mysql-test/main/func_json_agg_reread.test @@ -0,0 +1,84 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Asking for the result of a JSON aggregate more than once. +--echo # +--echo # A group is built up across three calls: clear() at its start, add() +--echo # once per row, and val_str() when the result is wanted. The text +--echo # accumulates in a buffer belonging to the item, there being nowhere +--echo # else to keep it between rows, so anything val_str() writes into that +--echo # buffer stays written. +--echo # +--echo # Nothing says the result is wanted only once. HAVING on the alias is +--echo # the shortest statement that wants it twice; the number of times +--echo # follows the number of conditions. +--echo # + +CREATE TABLE t1 (g INT, a VARCHAR(10)); +INSERT INTO t1 VALUES (1, 'x'), (1, 'y'), (2, 'z'), (2, 'w'); + +--echo # +--echo # 1. Asked once. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1; +SELECT JSON_ARRAYAGG(a) AS v FROM t1; + +--echo # +--echo # 2. Asked twice. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%'; +SELECT JSON_VALID(JSON_OBJECTAGG(a, a)) AS valid FROM t1 HAVING valid IS NOT NULL; + +--echo # +--echo # 3. Asked three times. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '{%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 HAVING v LIKE '%' AND v LIKE '[%'; + +--echo # +--echo # 4. Every group, not only the first. +--echo # +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g HAVING v LIKE '%' + ORDER BY g; + +--echo # +--echo # 5. A group of one row, where no separator is ever written. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE a = 'x' HAVING v LIKE '%'; + +--echo # +--echo # 6. A copy of the aggregate, which WITH ROLLUP makes without fixing +--echo # the item again. +--echo # +--echo # The condition is what makes this section a test of anything: a +--echo # rollup level asked for once is closed once whether the flag is +--echo # kept or not, and every other section here reaches the second +--echo # asking through HAVING. The extra level is produced by a copy of +--echo # the item, so it is that copy's own flag being reset per group +--echo # that has to hold. +--echo # +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%'; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%'; +--echo # two conditions, so each level is asked for a third time +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%' AND v LIKE '{%'; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP + HAVING v LIKE '%' AND v LIKE '[%'; +--echo # and the levels without a condition, which must not move either +SELECT g, JSON_OBJECTAGG(a, a) AS v FROM t1 GROUP BY g WITH ROLLUP; +SELECT g, JSON_ARRAYAGG(a) AS v FROM t1 GROUP BY g WITH ROLLUP; + +--echo # +--echo # 7. No rows, which is a NULL and stays one however often it is asked +--echo # for. +--echo # +SELECT JSON_OBJECTAGG(a, a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; +SELECT JSON_ARRAYAGG(a) AS v FROM t1 WHERE g = 99 HAVING v LIKE '%'; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_aliasing.result b/mysql-test/main/func_json_aliasing.result new file mode 100644 index 0000000000000..739cd743553f7 --- /dev/null +++ b/mysql-test/main/func_json_aliasing.result @@ -0,0 +1,719 @@ +CREATE TABLE t1 (id INT, js VARCHAR(64), v VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"A":1,"B":2}', 'X'), (2, '{"A":3,"B":4}', 'Y'); +# +# 1. Document and value both built into the buffer offered them +# +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v)) AS ins FROM t1; +ins +{"A": 1, "B": 2, "c": "XX"} +{"A": 3, "B": 4, "c": "YY"} +SELECT JSON_INSERT(LOWER(js), '$.c', LOWER(v)) AS ins_low FROM t1; +ins_low +{"a": 1, "b": 2, "c": "x"} +{"a": 3, "b": 4, "c": "y"} +SELECT JSON_INSERT(REVERSE(REVERSE(js)), '$.c', UPPER(v)) AS ins_rev FROM t1; +ins_rev +{"A": 1, "B": 2, "c": "X"} +{"A": 3, "B": 4, "c": "Y"} +SELECT JSON_SET(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS st FROM t1; +st +{"A": "XX", "B": 2} +{"A": "YY", "B": 4} +SELECT JSON_REPLACE(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS rep FROM t1; +rep +{"A": "XX", "B": 2} +{"A": "YY", "B": 4} +SELECT JSON_REMOVE(CONCAT(js, ''), '$.A') AS rem FROM t1; +rem +{"B": 2} +{"B": 4} +SELECT JSON_ARRAY_APPEND(CONCAT('[', js, ']'), '$', CONCAT(v, v)) AS app +FROM t1; +app +[{"A": 1, "B": 2}, "XX"] +[{"A": 3, "B": 4}, "YY"] +SELECT JSON_ARRAY_INSERT(CONCAT('[', js, ']'), '$[0]', CONCAT(v, v)) AS ari +FROM t1; +ari +["XX", {"A": 1, "B": 2}] +["YY", {"A": 3, "B": 4}] +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mrg FROM t1; +mrg +{"A": 1, "B": 2, "c": "X"} +{"A": 3, "B": 4, "c": "Y"} +SELECT JSON_MERGE_PATCH(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mpt +FROM t1; +mpt +{"A": 1, "B": 2, "c": "X"} +{"A": 3, "B": 4, "c": "Y"} +# +# 2. More than one path, so the document read from on the second +# pass is the one the first pass wrote +# +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), +'$.d', CONCAT(v, v, v)) AS two_paths FROM t1; +two_paths +{"A": 1, "B": 2, "c": "XX", "d": "XXX"} +{"A": 3, "B": 4, "c": "YY", "d": "YYY"} +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), +'$.zz.deeper', CONCAT(v, v), +'$.d', CONCAT(v, v, v)) AS three_paths FROM t1; +three_paths +{"A": 1, "B": 2, "c": "XX", "d": "XXX"} +{"A": 3, "B": 4, "c": "YY", "d": "YYY"} +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}'), +CONCAT('{"d":"', v, '"}')) AS mrg3 FROM t1; +mrg3 +{"A": 1, "B": 2, "c": "X", "d": "X"} +{"A": 3, "B": 4, "c": "Y", "d": "Y"} +# +# 3. A value that grows a great deal, so a buffer holding it has +# to be found somewhere else +# +# These are the only cases here big enough to make a buffer +# move, and so the only ones that reach what this file is +# about. How long the answer is cannot be what they ask: +# copying out of a buffer that has moved brings back the same +# number of bytes it would have brought back intact, so a +# length is exactly the measure that survives the fault. They +# ask instead whether the answer still reads as a document, +# whether the member nobody touched came through, and what +# stands at both ends of the value that grew. +# +SELECT LENGTH(g) AS grown, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail +FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', REPEAT(v, 5000)) AS g +FROM t1) AS x; +grown parses kept_a kept_b value_head value_tail +5025 1 1 2 XXXXXXXX XXXXXXXX +5025 1 3 4 YYYYYYYY YYYYYYYY +SELECT LENGTH(g) AS mgrown, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail +FROM (SELECT JSON_MERGE(CONCAT(js, ''), +CONCAT('{"c":"', REPEAT(v, 5000), '"}')) AS g +FROM t1) AS x; +mgrown parses kept_a kept_b value_head value_tail +5025 1 1 2 XXXXXXXX XXXXXXXX +5025 1 3 4 YYYYYYYY YYYYYYYY +# +# 4. A user variable holding the document, written to by the +# expression that works out the value +# +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 30))) AS uvar; +uvar +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +SET @d= '{"a":1111111111}'; +SELECT LENGTH(g) AS uvar_big, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.a') AS kept_a, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail +FROM (SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 100000))) AS g) AS x; +uvar_big parses kept_a value_head value_tail +100026 1 1111111111 zzzzzzzz zzzzzzzz +SET @d= '[1111111111]'; +SELECT JSON_ARRAY_APPEND(@d, '$', (@d := REPEAT('z', 30))) AS uvar_app; +uvar_app +[1111111111, "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"] +SET @d= '{"a":1111111111}'; +SELECT JSON_MERGE(@d, (@d := '{"b":2}')) AS uvar_mrg; +uvar_mrg +{"a": 1111111111, "b": 2} +# +# 5. A routine's own variable holding the document, with the +# value worked out by calling something +# +CREATE FUNCTION f_grow(n INT) RETURNS TEXT +BEGIN +RETURN REPEAT('z', n); +END| +CREATE PROCEDURE p_edit() +BEGIN +DECLARE d TEXT DEFAULT '{"a":1111111111}'; +SELECT JSON_INSERT(d, '$.b', f_grow(30)) AS sp_ins; +SELECT LENGTH(g) AS sp_ins_big, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.a') AS kept_a, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail +FROM (SELECT JSON_INSERT(d, '$.b', f_grow(50000)) AS g) AS x; +SELECT JSON_INSERT(CONCAT(d, ''), '$.b', f_grow(30)) AS sp_ins_concat; +SELECT JSON_MERGE(d, CONCAT('{"b":"', f_grow(30), '"}')) AS sp_mrg; +SELECT JSON_REMOVE(CONCAT(d, ''), '$.a') AS sp_rem; +END| +CALL p_edit(); +sp_ins +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +sp_ins_big parses kept_a value_head value_tail +50026 1 1111111111 zzzzzzzz zzzzzzzz +sp_ins_concat +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +sp_mrg +{"a": 1111111111, "b": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +sp_rem +{} +DROP PROCEDURE p_edit; +DROP FUNCTION f_grow; +# +# 6. A column read straight out of the row, which is the one +# producer that returns the row itself +# +SELECT JSON_INSERT(js, '$.c', (SELECT MAX(v) FROM t1)) AS col_sub FROM t1; +col_sub +{"A": 1, "B": 2, "c": "Y"} +{"A": 3, "B": 4, "c": "Y"} +SELECT JSON_INSERT(js, '$.c', (SELECT COUNT(*) FROM t1)) AS col_cnt FROM t1; +col_cnt +{"A": 1, "B": 2, "c": 2} +{"A": 3, "B": 4, "c": 2} +# +# 7. A value that is itself a document +# +# Everything above passes a plain string, which is copied +# a character at a time with quotes put round it. A value +# that is already a document goes in another way: its bytes +# are noted down where they stand, and only then is the +# composing written into the buffer they may have to share. +# That is the one place in this work where a note of where +# something was outlives an append, so it is the one that +# belongs here. +# +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', +JSON_EXTRACT(CONCAT('{"q":[1,', v, ']}'), '$.q')) +AS typed_ins FROM t1; +typed_ins +{"A": 1, "B": 2, "c": null} +{"A": 3, "B": 4, "c": null} +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', +JSON_OBJECT('k', CONCAT(v, v))) AS typed_built FROM t1; +typed_built +{"A": 1, "B": 2, "c": {"k": "XX"}} +{"A": 3, "B": 4, "c": {"k": "YY"}} +SELECT JSON_MERGE(CONCAT(js, ''), +JSON_OBJECT('c', CONCAT(v, v))) AS typed_mrg FROM t1; +typed_mrg +{"A": 1, "B": 2, "c": "XX"} +{"A": 3, "B": 4, "c": "YY"} +# +# The same where the value is long enough to move a buffer +# while it is being written out. +# +SELECT LENGTH(g) AS typed_big, JSON_VALID(g) AS parses, +JSON_EXTRACT(g, '$.A') AS kept_a, +JSON_LENGTH(JSON_EXTRACT(g, '$.c')) AS value_members, +LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[0]')), 8) AS value_head, +RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[1]')), 8) AS value_tail +FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', +JSON_ARRAY(REPEAT(v, 5000), REPEAT(v, 5000))) AS g +FROM t1) AS x; +typed_big parses kept_a value_members value_head value_tail +10031 1 1 2 XXXXXXXX XXXXXXXX +10031 1 3 2 YYYYYYYY YYYYYYYY +# +# And where the document being edited is a user variable that +# the value expression writes to, with the value a document. +# +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', +JSON_ARRAY((@d := REPEAT('z', 30)), 2)) AS typed_uvar; +typed_uvar +{"a": 1111111111, "b": ["zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", 2]} +DROP TABLE t1; +# +# 8. The document is a routine's variable, and working out +# another argument writes to that same variable +# +# Section 5 calls something to work out the value, but nothing +# it calls can reach the document: a routine's own variables +# are its own, and what it calls cannot see them. A package +# body's variables are shared by everything in the package, so +# there the expression working out another argument can assign +# to the variable the document is being read from, and storing +# a longer value there gives the variable a new buffer and +# lets go of the old one. +# +# The document is worked out before any of the other +# arguments, so the answer each of these owes is the one built +# from the value the variable held then, not from the value it +# holds by the time the answer is put together. +# +# Every argument is worked out just as late as every other, so +# this is not about values: a path reaches it, so does a +# string to search for, so does a width to indent by. The +# cases below are grouped by which argument does the writing. +# +SET @save_sql_mode= @@sql_mode; +SET sql_mode='ORACLE'; +CREATE PACKAGE pkg AS +PROCEDURE p; +PROCEDURE p1; +PROCEDURE p2; +PROCEDURE p3; +PROCEDURE p4; +END; +$$ +CREATE PACKAGE BODY pkg AS +d TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +/* +Section 10 has a variable of its own for each of its cases, and +writes each of them away exactly once. Both halves of that are +needed. A buffer is let go only where the value being stored will +not fit in it, so a variable that has already been written away +once is holding a buffer big enough for the next time and every +case after the first would reach nothing; and the rows these cases +select show the variable, so what is written away has to be put +back or a row would show one thing or another according to when it +was read. Putting it back settles nothing the cases are about - +the buffer the document was read from has gone by then. +*/ +e1 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +e2 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +e3 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +e4 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; +/* +The variable section 11 uses. Its value comes from a function so +that the value is attested - a string written out by hand is +not - and from the function that writes it the plain way, because +a document not formatted that way is read back rather than handed +over and nothing is then said about how deep it is. The value is +deep, so that writing a shallow one over it makes the difference +the section is about. +*/ +g TEXT := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); +/* +The second JSON_UNQUOTE variable of section 9, and the character +set on it is the whole reason it exists. JSON_UNQUOTE passes its +argument straight on only where the argument is already in the set +the answer is declared to be in, and that set is utf8mb4 whatever +the argument was; a variable left to take the set the connection +is using is therefore converted into a buffer belonging to the +caller instead, and no view of it goes anywhere. Every other +variable here is left as it comes, so that the ordinary case is +what the rest of the file reads. +*/ +u TEXT CHARACTER SET utf8mb4 := '{"a":{"k":"zz"},"arr":[7,8]}'; +PROCEDURE reset AS +BEGIN +d := '{"a":{"k":"zz"},"arr":[7,8]}'; +END; +PROCEDURE grew AS +BEGIN +d := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +END; +FUNCTION grow RETURN INT AS +BEGIN +grew(); +RETURN 7; +END; +FUNCTION grow_path RETURN TEXT AS +BEGIN +grew(); +RETURN '$.a'; +END; +FUNCTION grow_scalar_path RETURN TEXT AS +BEGIN +grew(); +RETURN '$.arr[0]'; +END; +FUNCTION grow_needle RETURN TEXT AS +BEGIN +grew(); +RETURN 'zz'; +END; +PROCEDURE reset_u AS +BEGIN +u := '{"a":{"k":"zz"},"arr":[7,8]}'; +END; +FUNCTION grow_u RETURN INT AS +BEGIN +u := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +RETURN 7; +END; +FUNCTION grow1 RETURN INT AS +BEGIN +e1 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e1 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN 7; +END; +FUNCTION grow2_path RETURN TEXT AS +BEGIN +e2 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e2 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN '$.a'; +END; +FUNCTION grow3 RETURN INT AS +BEGIN +e3 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e3 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN 7; +END; +FUNCTION grow4 RETURN INT AS +BEGIN +e4 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; +e4 := '{"a":{"k":"zz"},"arr":[7,8]}'; +RETURN 7; +END; +PROCEDURE reset_g AS +BEGIN +g := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); +END; +FUNCTION flat RETURN INT AS +BEGIN +g := JSON_LOOSE('{"z":1}'); +RETURN 7; +END; +FUNCTION flat_path_rem RETURN TEXT AS +BEGIN +g := JSON_LOOSE('{"z":1}'); +RETURN '$.a.b.c[1]'; +END; +FUNCTION flat_path_qry RETURN TEXT AS +BEGIN +g := JSON_LOOSE('{"z":1}'); +RETURN '$.a'; +END; +PROCEDURE p AS +BEGIN +reset(); SELECT JSON_INSERT(d, '$.b', grow()) AS pkg_ins; +reset(); SELECT JSON_SET(d, '$.a', grow()) AS pkg_set; +reset(); SELECT JSON_REPLACE(d, '$.a', grow()) AS pkg_rep; +reset(); SELECT JSON_ARRAY_APPEND(d, '$.arr', grow()) AS pkg_app; +reset(); SELECT JSON_ARRAY_INSERT(d, '$.arr[0]', grow()) AS pkg_ari; +reset(); SELECT JSON_CONTAINS(d, CONCAT('', grow()), '$.arr') AS pkg_has; +reset(); SELECT JSON_REMOVE(d, grow_path()) AS pkg_rem; +reset(); SELECT JSON_EXISTS(d, grow_path()) AS pkg_exi; +reset(); SELECT JSON_EXTRACT(d, grow_path()) AS pkg_ext; +reset(); SELECT JSON_QUERY(d, grow_path()) AS pkg_qry; +reset(); SELECT JSON_VALUE(d, grow_scalar_path()) AS pkg_val; +reset(); SELECT JSON_CONTAINS_PATH(d, 'one', grow_path()) AS pkg_cpa; +reset(); SELECT JSON_LENGTH(d, grow_path()) AS pkg_len; +reset(); SELECT JSON_KEYS(d, grow_path()) AS pkg_key; +reset(); SELECT JSON_MERGE(d, JSON_OBJECT('b', grow())) AS pkg_mrg; +reset(); SELECT JSON_MERGE_PATCH(d, JSON_OBJECT('b', grow())) AS pkg_mpt; +reset(); SELECT JSON_EQUALS(d, JSON_OBJECT('b', grow())) AS pkg_eqs; +reset(); SELECT JSON_OVERLAPS(d, JSON_OBJECT('arr', +JSON_ARRAY(7, grow() + 1))) AS pkg_ovl; +reset(); SELECT JSON_SEARCH(d, 'one', grow_needle()) AS pkg_sea; +reset(); SELECT JSON_DETAILED(d, grow()) AS pkg_det; +END; +PROCEDURE p1 AS +BEGIN +reset(); SELECT JSON_VALID(d) AS pkg1_vld, JSON_TYPE(d) AS pkg1_typ, +JSON_DEPTH(d) AS pkg1_dep, JSON_LENGTH(d) AS pkg1_len; +reset(); SELECT JSON_KEYS(d) AS pkg1_key; +reset(); SELECT JSON_COMPACT(d) AS pkg1_cmp; +reset(); SELECT JSON_LOOSE(d) AS pkg1_lse; +reset(); SELECT JSON_DETAILED(d) AS pkg1_det; +reset(); SELECT JSON_UNQUOTE(d) AS pkg1_unq; +reset(); SELECT JSON_INSERT(JSON_KEYS(d), '$[2]', grow()) AS pkg1_key_kept; +reset(); SELECT JSON_INSERT(JSON_COMPACT(d), '$.b', grow()) AS pkg1_cmp_kept; +reset(); SELECT JSON_INSERT(JSON_LOOSE(d), '$.b', grow()) AS pkg1_lse_kept; +reset(); SELECT JSON_INSERT(JSON_DETAILED(d), '$.b', grow()) AS pkg1_det_kept; +reset(); SELECT JSON_INSERT(JSON_UNQUOTE(d), '$.b', grow()) AS pkg1_unq_kept; +reset_u(); SELECT JSON_INSERT(JSON_UNQUOTE(u), '$.b', grow_u()) +AS pkg1_unq4_kept; +END; +PROCEDURE p2 AS +BEGIN +SELECT id, e1 AS a FROM t2 +ORDER BY JSON_INSERT(a, '$.b', grow1() * id) DESC; +SELECT id, e2 AS a FROM t2 +ORDER BY JSON_REMOVE(a, grow2_path()) DESC, id; +SELECT id, e3 AS a FROM t2 +ORDER BY JSON_INSERT(JSON_KEYS(a), '$[2]', grow3() * id) DESC; +SELECT id, e4 AS a, COUNT(*) FROM t2 GROUP BY id, a +ORDER BY JSON_INSERT(a, '$.b', grow4() * id) DESC; +END; +PROCEDURE p3 AS +BEGIN +reset_g(); SELECT JSON_SET(g, '$.z', flat()) AS dep_set; +reset_g(); SELECT JSON_ARRAY_APPEND(g, '$.arr', flat()) AS dep_app; +reset_g(); SELECT JSON_ARRAY_INSERT(g, '$.arr[0]', flat()) AS dep_ari; +reset_g(); SELECT JSON_REMOVE(g, flat_path_rem()) AS dep_rem; +reset_g(); SELECT JSON_QUERY(g, flat_path_qry()) AS dep_qry; +reset_g(); SELECT JSON_MERGE(g, JSON_OBJECT('m', flat())) AS dep_mrg; +reset_g(); SELECT JSON_MERGE_PATCH(g, JSON_OBJECT('m', flat())) AS dep_mpt; +END; +PROCEDURE p4 AS +BEGIN +reset(); SELECT jt.v AS pkg4_where FROM +JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt +WHERE grow() = 7; +reset(); SELECT jt.v AS pkg4_list, grow() AS g FROM +JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt; +END; +END; +$$ +CALL pkg.p(); +pkg_ins +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg_set +{"a": 7, "arr": [7, 8]} +pkg_rep +{"a": 7, "arr": [7, 8]} +pkg_app +{"a": {"k": "zz"}, "arr": [7, 8, 7]} +pkg_ari +{"a": {"k": "zz"}, "arr": [7, 7, 8]} +pkg_has +1 +pkg_rem +{"arr": [7, 8]} +pkg_exi +1 +pkg_ext +{"k": "zz"} +pkg_qry +{"k":"zz"} +pkg_val +7 +pkg_cpa +1 +pkg_len +1 +pkg_key +["k"] +pkg_mrg +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg_mpt +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg_eqs +0 +pkg_ovl +1 +pkg_sea +"$.a.k" +pkg_det +{ + "a": + { + "k": "zz" + }, + "arr": + [ + 7, + 8 + ] +} +# +# 9. The document is a routine's variable, and the call has +# nothing left to work out after reading it +# +# A call given nothing but a document is finished with it +# before anything else can run, so it is answered with a +# view into the variable rather than with bytes of its own. +# The first group reads each of them that way. +# +# That holds only while none of the document comes back out +# of the call, so the second group makes each answer the +# document of something that keeps it across working out an +# argument that writes to the variable. JSON_UNQUOTE is +# among them because it passes its argument straight on when +# there is nothing to unquote, and is read the careful way +# for that reason. +# +# Straight on is not what it does with every argument, and +# the difference decides whether there is anything to be +# careful about. The answer is declared to be in one +# particular character set whatever went in, so an argument +# in some other set is converted into a buffer of the +# caller's own on the way out and the variable is left where +# it stands. Only an argument already in that set is the one +# passed on, and only then is the answer a view of the +# variable. So the group ends with the same case twice over, +# once on a variable that takes the connection's set and once +# on one declared in the answer's. +# +CALL pkg.p1(); +pkg1_vld pkg1_typ pkg1_dep pkg1_len +1 OBJECT 3 2 +pkg1_key +["a", "arr"] +pkg1_cmp +{"a":{"k":"zz"},"arr":[7,8]} +pkg1_lse +{"a": {"k": "zz"}, "arr": [7, 8]} +pkg1_det +{ + "a": + { + "k": "zz" + }, + "arr": + [ + 7, + 8 + ] +} +pkg1_unq +{"a":{"k":"zz"},"arr":[7,8]} +pkg1_key_kept +["a", "arr", 7] +pkg1_cmp_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_lse_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_det_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_unq_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +pkg1_unq4_kept +{"a": {"k": "zz"}, "arr": [7, 8], "b": 7} +# +# 10. The document is reached through a reference standing in +# front of the thing that holds it +# +# An expression in an ORDER BY that names an entry of the +# select list does not get that entry. It gets a reference, +# and a reference passes the value on by asking the entry for +# it - so which of the two ways of asking it uses is the +# whole of what these cases are about. +# +# Asked for a plain string, a routine's variable answers with +# a view of itself, on the footing that whoever wanted a +# string may build over the buffer it was offered and must +# not be allowed to build over the variable. Asked for a +# document it answers with bytes of the caller's own, for the +# reason section 8 gives. A reference that puts the first +# question on behalf of a caller that asked the second lets +# the view out again, and section 8's hazard is back with a +# wrapper in front of it. +# +# A value writes the variable in the first case and a path in +# the second, for the reason section 8 gives: an argument is +# an argument, and which one it is decides nothing. +# +# The third asks for the document the cheap way. A call given +# nothing but a document is answered with a view, section 9's +# subject, and a reference has to pass that formatting on as +# the same formatting rather than quietly making it the other +# one; the answer is then kept across a write to show that +# what came back out is nobody's view of anything. +# +# The last case is the same question answered the other way. +# Once the rows are gathered into a temporary table the +# reference reads that table's column rather than the entry, +# and a column of a row that is being read is nobody's to +# free while it is being read - so there a plain string is +# the right thing to ask for and no copy is owed. +# +CREATE TABLE t2 (id INT); +INSERT INTO t2 VALUES (1), (2); +CALL pkg.p2(); +id a +1 {"a":{"k":"zz"},"arr":[7,8]} +2 {"a":{"k":"zz"},"arr":[7,8]} +id a +1 {"a":{"k":"zz"},"arr":[7,8]} +2 {"a":{"k":"zz"},"arr":[7,8]} +id a +1 {"a":{"k":"zz"},"arr":[7,8]} +2 {"a":{"k":"zz"},"arr":[7,8]} +id a COUNT(*) +1 {"a":{"k":"zz"},"arr":[7,8]} 1 +2 {"a":{"k":"zz"},"arr":[7,8]} 1 +DROP TABLE t2; +# +# 11. How deep the document goes, asked after the other arguments +# have been worked out +# +# A function that edits a document asks it three things. Two +# of them - whether it reads as a document at all, and whether +# it is written the plain way - are asked before anything is +# composed, because the answers are about the document that is +# about to be read. How deep it goes is the same kind of +# question and has to be asked at the same time. +# +# Asked at the far end it is a question about whatever the +# variable holds by then, and by then an argument has run. So +# these write a SHALLOWER document over the variable while the +# deeper one is being read: the answer returned is as deep +# as the document it was composed from, and the depth said +# about it would be the shallow one. +# +# Too small is the one direction a depth must never be wrong +# in. A caller splicing this answer into one of its own adds +# its own levels to what it is told, and a total that comes +# out under the limit on a document that is over it is how a +# document nothing can read back gets attested to. A debug +# build reads the answer and stops. +# +# This cannot be shown from any commit below the one that +# gives a variable's reader bytes of its own: without that, +# writing over the variable takes the document away as well as +# the depth, and what shows is section 8's fault instead. +# +# A value writes the variable in the first three and a path in +# the next two, the path being the argument in between for +# JSON_REMOVE and for JSON_QUERY. +# +# The last two are the pair whose answer is composed out of +# document arguments and nothing else, so there the variable is +# written by a value inside another document argument rather +# than by an argument of the function itself. Merging makes +# the answer a level deeper than the deepest of them and +# patching leaves it where it was, and both take the deepest +# from what the arguments say rather than from a reading, so +# both have to have asked while the arguments were still +# talking about the documents that went in. +# +CALL pkg.p3(); +dep_set +{"a": {"b": {"c": [1, 2]}}, "arr": [9], "z": 7} +dep_app +{"a": {"b": {"c": [1, 2]}}, "arr": [9, 7]} +dep_ari +{"a": {"b": {"c": [1, 2]}}, "arr": [7, 9]} +dep_rem +{"a": {"b": {"c": [1]}}, "arr": [9]} +dep_qry +{"b": {"c": [1, 2]}} +dep_mrg +{"a": {"b": {"c": [1, 2]}}, "arr": [9], "m": 7} +dep_mpt +{"a": {"b": {"c": [1, 2]}}, "arr": [9], "m": 7} +# +# 12. The document is walked a row at a time, with something +# else in the query writing to it between rows +# +# Everything above reads a document, works an answer out of +# it and is finished with it inside the one call. A table +# built out of a document is not: it is told where the +# document stands when the walk begins and reads on from +# there each time a row is asked for, so the note of where it +# stands outlives everything the query does between one row +# and the next. +# +# What the query does between rows is work out the rest of +# the row - the condition deciding whether to keep it, and +# the entries printed for it - and either of those can write +# to the variable the document was read from. So the first +# row is read from a document that is still there and every +# row after it from one that has been let go. +# +# Both places are shown because they are different points in +# working a row out, and the array has more than one element +# so that there is a row left to read after the write. The +# rows owed either way are the elements of the array the +# document held when the walk began. +# +CALL pkg.p4(); +pkg4_where +7 +8 +pkg4_list g +7 7 +8 7 +DROP PACKAGE pkg; +SET sql_mode= @save_sql_mode; +# +# The arm that copes with there being no room for the copy a +# variable is passed as is reached only by a debug build, +# and is in func_json_variable_copy_oom. +# diff --git a/mysql-test/main/func_json_aliasing.test b/mysql-test/main/func_json_aliasing.test new file mode 100644 index 0000000000000..63d99754b4491 --- /dev/null +++ b/mysql-test/main/func_json_aliasing.test @@ -0,0 +1,593 @@ +# +# Working out a value while the document is still being read from. +# +# A function that edits a document walks it to find the place to edit +# and then copies the pieces either side of that place out of it, and +# in between it works out the value to put there. Working out a value +# is running whatever expression a caller wrote, and the document is +# not this function's to begin with - it can be a row, a routine's +# variable, another statement's user variable. Nothing stops such an +# expression writing where the document is. +# +# Whether an argument is worked out INTO the buffer that was offered to +# it is a property of the function that produces it, not of the value: +# CONCAT, LOWER, UPPER and REVERSE build into it, TRIM and SUBSTRING +# return a piece of what they were given, a bare column returns +# the row, and something written out by hand is not worked out at all. +# So a case built from written-out arguments exercises none of this, +# and every case below is built from producers that do write. +# + +CREATE TABLE t1 (id INT, js VARCHAR(64), v VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"A":1,"B":2}', 'X'), (2, '{"A":3,"B":4}', 'Y'); + +--echo # +--echo # 1. Document and value both built into the buffer offered them +--echo # +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v)) AS ins FROM t1; +SELECT JSON_INSERT(LOWER(js), '$.c', LOWER(v)) AS ins_low FROM t1; +SELECT JSON_INSERT(REVERSE(REVERSE(js)), '$.c', UPPER(v)) AS ins_rev FROM t1; +SELECT JSON_SET(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS st FROM t1; +SELECT JSON_REPLACE(CONCAT(js, ''), '$.A', CONCAT(v, v)) AS rep FROM t1; +SELECT JSON_REMOVE(CONCAT(js, ''), '$.A') AS rem FROM t1; +SELECT JSON_ARRAY_APPEND(CONCAT('[', js, ']'), '$', CONCAT(v, v)) AS app + FROM t1; +SELECT JSON_ARRAY_INSERT(CONCAT('[', js, ']'), '$[0]', CONCAT(v, v)) AS ari + FROM t1; +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mrg FROM t1; +SELECT JSON_MERGE_PATCH(CONCAT(js, ''), CONCAT('{"c":"', v, '"}')) AS mpt + FROM t1; + +--echo # +--echo # 2. More than one path, so the document read from on the second +--echo # pass is the one the first pass wrote +--echo # +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), + '$.d', CONCAT(v, v, v)) AS two_paths FROM t1; +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', CONCAT(v, v), + '$.zz.deeper', CONCAT(v, v), + '$.d', CONCAT(v, v, v)) AS three_paths FROM t1; +SELECT JSON_MERGE(CONCAT(js, ''), CONCAT('{"c":"', v, '"}'), + CONCAT('{"d":"', v, '"}')) AS mrg3 FROM t1; + +--echo # +--echo # 3. A value that grows a great deal, so a buffer holding it has +--echo # to be found somewhere else +--echo # +--echo # These are the only cases here big enough to make a buffer +--echo # move, and so the only ones that reach what this file is +--echo # about. How long the answer is cannot be what they ask: +--echo # copying out of a buffer that has moved brings back the same +--echo # number of bytes it would have brought back intact, so a +--echo # length is exactly the measure that survives the fault. They +--echo # ask instead whether the answer still reads as a document, +--echo # whether the member nobody touched came through, and what +--echo # stands at both ends of the value that grew. +--echo # +SELECT LENGTH(g) AS grown, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail + FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', REPEAT(v, 5000)) AS g + FROM t1) AS x; +SELECT LENGTH(g) AS mgrown, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.A') AS kept_a, JSON_EXTRACT(g, '$.B') AS kept_b, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c')), 8) AS value_tail + FROM (SELECT JSON_MERGE(CONCAT(js, ''), + CONCAT('{"c":"', REPEAT(v, 5000), '"}')) AS g + FROM t1) AS x; + +--echo # +--echo # 4. A user variable holding the document, written to by the +--echo # expression that works out the value +--echo # +# Each statement below overwrites the variable it reads, so running it a +# second time to check that it repeats itself would be asking it about a +# different document. +--disable_ps2_protocol +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 30))) AS uvar; +SET @d= '{"a":1111111111}'; +SELECT LENGTH(g) AS uvar_big, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.a') AS kept_a, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail + FROM (SELECT JSON_INSERT(@d, '$.b', (@d := REPEAT('z', 100000))) AS g) AS x; +SET @d= '[1111111111]'; +SELECT JSON_ARRAY_APPEND(@d, '$', (@d := REPEAT('z', 30))) AS uvar_app; +SET @d= '{"a":1111111111}'; +SELECT JSON_MERGE(@d, (@d := '{"b":2}')) AS uvar_mrg; +--enable_ps2_protocol + +--echo # +--echo # 5. A routine's own variable holding the document, with the +--echo # value worked out by calling something +--echo # +--delimiter | +CREATE FUNCTION f_grow(n INT) RETURNS TEXT +BEGIN + RETURN REPEAT('z', n); +END| +CREATE PROCEDURE p_edit() +BEGIN + DECLARE d TEXT DEFAULT '{"a":1111111111}'; + SELECT JSON_INSERT(d, '$.b', f_grow(30)) AS sp_ins; + SELECT LENGTH(g) AS sp_ins_big, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.a') AS kept_a, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.b')), 8) AS value_tail + FROM (SELECT JSON_INSERT(d, '$.b', f_grow(50000)) AS g) AS x; + SELECT JSON_INSERT(CONCAT(d, ''), '$.b', f_grow(30)) AS sp_ins_concat; + SELECT JSON_MERGE(d, CONCAT('{"b":"', f_grow(30), '"}')) AS sp_mrg; + SELECT JSON_REMOVE(CONCAT(d, ''), '$.a') AS sp_rem; +END| +--delimiter ; +CALL p_edit(); +DROP PROCEDURE p_edit; +DROP FUNCTION f_grow; + +--echo # +--echo # 6. A column read straight out of the row, which is the one +--echo # producer that returns the row itself +--echo # +SELECT JSON_INSERT(js, '$.c', (SELECT MAX(v) FROM t1)) AS col_sub FROM t1; +SELECT JSON_INSERT(js, '$.c', (SELECT COUNT(*) FROM t1)) AS col_cnt FROM t1; + +--echo # +--echo # 7. A value that is itself a document +--echo # +--echo # Everything above passes a plain string, which is copied +--echo # a character at a time with quotes put round it. A value +--echo # that is already a document goes in another way: its bytes +--echo # are noted down where they stand, and only then is the +--echo # composing written into the buffer they may have to share. +--echo # That is the one place in this work where a note of where +--echo # something was outlives an append, so it is the one that +--echo # belongs here. +--echo # +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', + JSON_EXTRACT(CONCAT('{"q":[1,', v, ']}'), '$.q')) + AS typed_ins FROM t1; +SELECT JSON_INSERT(CONCAT(js, ''), '$.c', + JSON_OBJECT('k', CONCAT(v, v))) AS typed_built FROM t1; +SELECT JSON_MERGE(CONCAT(js, ''), + JSON_OBJECT('c', CONCAT(v, v))) AS typed_mrg FROM t1; + +--echo # +--echo # The same where the value is long enough to move a buffer +--echo # while it is being written out. +--echo # +SELECT LENGTH(g) AS typed_big, JSON_VALID(g) AS parses, + JSON_EXTRACT(g, '$.A') AS kept_a, + JSON_LENGTH(JSON_EXTRACT(g, '$.c')) AS value_members, + LEFT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[0]')), 8) AS value_head, + RIGHT(JSON_UNQUOTE(JSON_EXTRACT(g, '$.c[1]')), 8) AS value_tail + FROM (SELECT JSON_INSERT(CONCAT(js, ''), '$.c', + JSON_ARRAY(REPEAT(v, 5000), REPEAT(v, 5000))) AS g + FROM t1) AS x; + +--echo # +--echo # And where the document being edited is a user variable that +--echo # the value expression writes to, with the value a document. +--echo # +# This one overwrites the variable it reads as well. +--disable_ps2_protocol +SET @d= '{"a":1111111111}'; +SELECT JSON_INSERT(@d, '$.b', + JSON_ARRAY((@d := REPEAT('z', 30)), 2)) AS typed_uvar; +--enable_ps2_protocol + +DROP TABLE t1; + +--echo # +--echo # 8. The document is a routine's variable, and working out +--echo # another argument writes to that same variable +--echo # +--echo # Section 5 calls something to work out the value, but nothing +--echo # it calls can reach the document: a routine's own variables +--echo # are its own, and what it calls cannot see them. A package +--echo # body's variables are shared by everything in the package, so +--echo # there the expression working out another argument can assign +--echo # to the variable the document is being read from, and storing +--echo # a longer value there gives the variable a new buffer and +--echo # lets go of the old one. +--echo # +--echo # The document is worked out before any of the other +--echo # arguments, so the answer each of these owes is the one built +--echo # from the value the variable held then, not from the value it +--echo # holds by the time the answer is put together. +--echo # +--echo # Every argument is worked out just as late as every other, so +--echo # this is not about values: a path reaches it, so does a +--echo # string to search for, so does a width to indent by. The +--echo # cases below are grouped by which argument does the writing. +--echo # +SET @save_sql_mode= @@sql_mode; +SET sql_mode='ORACLE'; + +--delimiter $$ +CREATE PACKAGE pkg AS + PROCEDURE p; + PROCEDURE p1; + PROCEDURE p2; + PROCEDURE p3; + PROCEDURE p4; +END; +$$ +CREATE PACKAGE BODY pkg AS + d TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + /* + Section 10 has a variable of its own for each of its cases, and + writes each of them away exactly once. Both halves of that are + needed. A buffer is let go only where the value being stored will + not fit in it, so a variable that has already been written away + once is holding a buffer big enough for the next time and every + case after the first would reach nothing; and the rows these cases + select show the variable, so what is written away has to be put + back or a row would show one thing or another according to when it + was read. Putting it back settles nothing the cases are about - + the buffer the document was read from has gone by then. + */ + e1 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + e2 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + e3 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + e4 TEXT := '{"a":{"k":"zz"},"arr":[7,8]}'; + /* + The variable section 11 uses. Its value comes from a function so + that the value is attested - a string written out by hand is + not - and from the function that writes it the plain way, because + a document not formatted that way is read back rather than handed + over and nothing is then said about how deep it is. The value is + deep, so that writing a shallow one over it makes the difference + the section is about. + */ + g TEXT := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); + /* + The second JSON_UNQUOTE variable of section 9, and the character + set on it is the whole reason it exists. JSON_UNQUOTE passes its + argument straight on only where the argument is already in the set + the answer is declared to be in, and that set is utf8mb4 whatever + the argument was; a variable left to take the set the connection + is using is therefore converted into a buffer belonging to the + caller instead, and no view of it goes anywhere. Every other + variable here is left as it comes, so that the ordinary case is + what the rest of the file reads. + */ + u TEXT CHARACTER SET utf8mb4 := '{"a":{"k":"zz"},"arr":[7,8]}'; + + PROCEDURE reset AS + BEGIN + d := '{"a":{"k":"zz"},"arr":[7,8]}'; + END; + + PROCEDURE grew AS + BEGIN + d := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + END; + + FUNCTION grow RETURN INT AS + BEGIN + grew(); + RETURN 7; + END; + + FUNCTION grow_path RETURN TEXT AS + BEGIN + grew(); + RETURN '$.a'; + END; + + FUNCTION grow_scalar_path RETURN TEXT AS + BEGIN + grew(); + RETURN '$.arr[0]'; + END; + + FUNCTION grow_needle RETURN TEXT AS + BEGIN + grew(); + RETURN 'zz'; + END; + + PROCEDURE reset_u AS + BEGIN + u := '{"a":{"k":"zz"},"arr":[7,8]}'; + END; + + FUNCTION grow_u RETURN INT AS + BEGIN + u := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + RETURN 7; + END; + + FUNCTION grow1 RETURN INT AS + BEGIN + e1 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e1 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN 7; + END; + + FUNCTION grow2_path RETURN TEXT AS + BEGIN + e2 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e2 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN '$.a'; + END; + + FUNCTION grow3 RETURN INT AS + BEGIN + e3 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e3 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN 7; + END; + + FUNCTION grow4 RETURN INT AS + BEGIN + e4 := '{"a":{"k":"zz"},"arr":[7,8],"pad":"' || REPEAT('z', 20000) || '"}'; + e4 := '{"a":{"k":"zz"},"arr":[7,8]}'; + RETURN 7; + END; + + PROCEDURE reset_g AS + BEGIN + g := JSON_LOOSE('{"a":{"b":{"c":[1,2]}},"arr":[9]}'); + END; + + FUNCTION flat RETURN INT AS + BEGIN + g := JSON_LOOSE('{"z":1}'); + RETURN 7; + END; + + FUNCTION flat_path_rem RETURN TEXT AS + BEGIN + g := JSON_LOOSE('{"z":1}'); + RETURN '$.a.b.c[1]'; + END; + + FUNCTION flat_path_qry RETURN TEXT AS + BEGIN + g := JSON_LOOSE('{"z":1}'); + RETURN '$.a'; + END; + + PROCEDURE p AS + BEGIN + reset(); SELECT JSON_INSERT(d, '$.b', grow()) AS pkg_ins; + reset(); SELECT JSON_SET(d, '$.a', grow()) AS pkg_set; + reset(); SELECT JSON_REPLACE(d, '$.a', grow()) AS pkg_rep; + reset(); SELECT JSON_ARRAY_APPEND(d, '$.arr', grow()) AS pkg_app; + reset(); SELECT JSON_ARRAY_INSERT(d, '$.arr[0]', grow()) AS pkg_ari; + reset(); SELECT JSON_CONTAINS(d, CONCAT('', grow()), '$.arr') AS pkg_has; + + reset(); SELECT JSON_REMOVE(d, grow_path()) AS pkg_rem; + reset(); SELECT JSON_EXISTS(d, grow_path()) AS pkg_exi; + reset(); SELECT JSON_EXTRACT(d, grow_path()) AS pkg_ext; + reset(); SELECT JSON_QUERY(d, grow_path()) AS pkg_qry; + reset(); SELECT JSON_VALUE(d, grow_scalar_path()) AS pkg_val; + reset(); SELECT JSON_CONTAINS_PATH(d, 'one', grow_path()) AS pkg_cpa; + reset(); SELECT JSON_LENGTH(d, grow_path()) AS pkg_len; + reset(); SELECT JSON_KEYS(d, grow_path()) AS pkg_key; + + reset(); SELECT JSON_MERGE(d, JSON_OBJECT('b', grow())) AS pkg_mrg; + reset(); SELECT JSON_MERGE_PATCH(d, JSON_OBJECT('b', grow())) AS pkg_mpt; + reset(); SELECT JSON_EQUALS(d, JSON_OBJECT('b', grow())) AS pkg_eqs; + reset(); SELECT JSON_OVERLAPS(d, JSON_OBJECT('arr', + JSON_ARRAY(7, grow() + 1))) AS pkg_ovl; + + reset(); SELECT JSON_SEARCH(d, 'one', grow_needle()) AS pkg_sea; + reset(); SELECT JSON_DETAILED(d, grow()) AS pkg_det; + END; + + PROCEDURE p1 AS + BEGIN + reset(); SELECT JSON_VALID(d) AS pkg1_vld, JSON_TYPE(d) AS pkg1_typ, + JSON_DEPTH(d) AS pkg1_dep, JSON_LENGTH(d) AS pkg1_len; + reset(); SELECT JSON_KEYS(d) AS pkg1_key; + reset(); SELECT JSON_COMPACT(d) AS pkg1_cmp; + reset(); SELECT JSON_LOOSE(d) AS pkg1_lse; + reset(); SELECT JSON_DETAILED(d) AS pkg1_det; + reset(); SELECT JSON_UNQUOTE(d) AS pkg1_unq; + + reset(); SELECT JSON_INSERT(JSON_KEYS(d), '$[2]', grow()) AS pkg1_key_kept; + reset(); SELECT JSON_INSERT(JSON_COMPACT(d), '$.b', grow()) AS pkg1_cmp_kept; + reset(); SELECT JSON_INSERT(JSON_LOOSE(d), '$.b', grow()) AS pkg1_lse_kept; + reset(); SELECT JSON_INSERT(JSON_DETAILED(d), '$.b', grow()) AS pkg1_det_kept; + reset(); SELECT JSON_INSERT(JSON_UNQUOTE(d), '$.b', grow()) AS pkg1_unq_kept; + reset_u(); SELECT JSON_INSERT(JSON_UNQUOTE(u), '$.b', grow_u()) + AS pkg1_unq4_kept; + END; + + PROCEDURE p2 AS + BEGIN + SELECT id, e1 AS a FROM t2 + ORDER BY JSON_INSERT(a, '$.b', grow1() * id) DESC; + SELECT id, e2 AS a FROM t2 + ORDER BY JSON_REMOVE(a, grow2_path()) DESC, id; + SELECT id, e3 AS a FROM t2 + ORDER BY JSON_INSERT(JSON_KEYS(a), '$[2]', grow3() * id) DESC; + SELECT id, e4 AS a, COUNT(*) FROM t2 GROUP BY id, a + ORDER BY JSON_INSERT(a, '$.b', grow4() * id) DESC; + END; + + PROCEDURE p3 AS + BEGIN + reset_g(); SELECT JSON_SET(g, '$.z', flat()) AS dep_set; + reset_g(); SELECT JSON_ARRAY_APPEND(g, '$.arr', flat()) AS dep_app; + reset_g(); SELECT JSON_ARRAY_INSERT(g, '$.arr[0]', flat()) AS dep_ari; + reset_g(); SELECT JSON_REMOVE(g, flat_path_rem()) AS dep_rem; + reset_g(); SELECT JSON_QUERY(g, flat_path_qry()) AS dep_qry; + reset_g(); SELECT JSON_MERGE(g, JSON_OBJECT('m', flat())) AS dep_mrg; + reset_g(); SELECT JSON_MERGE_PATCH(g, JSON_OBJECT('m', flat())) AS dep_mpt; + END; + + PROCEDURE p4 AS + BEGIN + reset(); SELECT jt.v AS pkg4_where FROM + JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt + WHERE grow() = 7; + reset(); SELECT jt.v AS pkg4_list, grow() AS g FROM + JSON_TABLE(d, '$.arr[*]' COLUMNS (v INT PATH '$')) AS jt; + END; +END; +$$ +--delimiter ; + +CALL pkg.p(); + +--echo # +--echo # 9. The document is a routine's variable, and the call has +--echo # nothing left to work out after reading it +--echo # +--echo # A call given nothing but a document is finished with it +--echo # before anything else can run, so it is answered with a +--echo # view into the variable rather than with bytes of its own. +--echo # The first group reads each of them that way. +--echo # +--echo # That holds only while none of the document comes back out +--echo # of the call, so the second group makes each answer the +--echo # document of something that keeps it across working out an +--echo # argument that writes to the variable. JSON_UNQUOTE is +--echo # among them because it passes its argument straight on when +--echo # there is nothing to unquote, and is read the careful way +--echo # for that reason. +--echo # +--echo # Straight on is not what it does with every argument, and +--echo # the difference decides whether there is anything to be +--echo # careful about. The answer is declared to be in one +--echo # particular character set whatever went in, so an argument +--echo # in some other set is converted into a buffer of the +--echo # caller's own on the way out and the variable is left where +--echo # it stands. Only an argument already in that set is the one +--echo # passed on, and only then is the answer a view of the +--echo # variable. So the group ends with the same case twice over, +--echo # once on a variable that takes the connection's set and once +--echo # on one declared in the answer's. +--echo # +CALL pkg.p1(); + +--echo # +--echo # 10. The document is reached through a reference standing in +--echo # front of the thing that holds it +--echo # +--echo # An expression in an ORDER BY that names an entry of the +--echo # select list does not get that entry. It gets a reference, +--echo # and a reference passes the value on by asking the entry for +--echo # it - so which of the two ways of asking it uses is the +--echo # whole of what these cases are about. +--echo # +--echo # Asked for a plain string, a routine's variable answers with +--echo # a view of itself, on the footing that whoever wanted a +--echo # string may build over the buffer it was offered and must +--echo # not be allowed to build over the variable. Asked for a +--echo # document it answers with bytes of the caller's own, for the +--echo # reason section 8 gives. A reference that puts the first +--echo # question on behalf of a caller that asked the second lets +--echo # the view out again, and section 8's hazard is back with a +--echo # wrapper in front of it. +--echo # +--echo # A value writes the variable in the first case and a path in +--echo # the second, for the reason section 8 gives: an argument is +--echo # an argument, and which one it is decides nothing. +--echo # +--echo # The third asks for the document the cheap way. A call given +--echo # nothing but a document is answered with a view, section 9's +--echo # subject, and a reference has to pass that formatting on as +--echo # the same formatting rather than quietly making it the other +--echo # one; the answer is then kept across a write to show that +--echo # what came back out is nobody's view of anything. +--echo # +--echo # The last case is the same question answered the other way. +--echo # Once the rows are gathered into a temporary table the +--echo # reference reads that table's column rather than the entry, +--echo # and a column of a row that is being read is nobody's to +--echo # free while it is being read - so there a plain string is +--echo # the right thing to ask for and no copy is owed. +--echo # +CREATE TABLE t2 (id INT); +INSERT INTO t2 VALUES (1), (2); +CALL pkg.p2(); +DROP TABLE t2; + +--echo # +--echo # 11. How deep the document goes, asked after the other arguments +--echo # have been worked out +--echo # +--echo # A function that edits a document asks it three things. Two +--echo # of them - whether it reads as a document at all, and whether +--echo # it is written the plain way - are asked before anything is +--echo # composed, because the answers are about the document that is +--echo # about to be read. How deep it goes is the same kind of +--echo # question and has to be asked at the same time. +--echo # +--echo # Asked at the far end it is a question about whatever the +--echo # variable holds by then, and by then an argument has run. So +--echo # these write a SHALLOWER document over the variable while the +--echo # deeper one is being read: the answer returned is as deep +--echo # as the document it was composed from, and the depth said +--echo # about it would be the shallow one. +--echo # +--echo # Too small is the one direction a depth must never be wrong +--echo # in. A caller splicing this answer into one of its own adds +--echo # its own levels to what it is told, and a total that comes +--echo # out under the limit on a document that is over it is how a +--echo # document nothing can read back gets attested to. A debug +--echo # build reads the answer and stops. +--echo # +--echo # This cannot be shown from any commit below the one that +--echo # gives a variable's reader bytes of its own: without that, +--echo # writing over the variable takes the document away as well as +--echo # the depth, and what shows is section 8's fault instead. +--echo # +--echo # A value writes the variable in the first three and a path in +--echo # the next two, the path being the argument in between for +--echo # JSON_REMOVE and for JSON_QUERY. +--echo # +--echo # The last two are the pair whose answer is composed out of +--echo # document arguments and nothing else, so there the variable is +--echo # written by a value inside another document argument rather +--echo # than by an argument of the function itself. Merging makes +--echo # the answer a level deeper than the deepest of them and +--echo # patching leaves it where it was, and both take the deepest +--echo # from what the arguments say rather than from a reading, so +--echo # both have to have asked while the arguments were still +--echo # talking about the documents that went in. +--echo # +CALL pkg.p3(); + +--echo # +--echo # 12. The document is walked a row at a time, with something +--echo # else in the query writing to it between rows +--echo # +--echo # Everything above reads a document, works an answer out of +--echo # it and is finished with it inside the one call. A table +--echo # built out of a document is not: it is told where the +--echo # document stands when the walk begins and reads on from +--echo # there each time a row is asked for, so the note of where it +--echo # stands outlives everything the query does between one row +--echo # and the next. +--echo # +--echo # What the query does between rows is work out the rest of +--echo # the row - the condition deciding whether to keep it, and +--echo # the entries printed for it - and either of those can write +--echo # to the variable the document was read from. So the first +--echo # row is read from a document that is still there and every +--echo # row after it from one that has been let go. +--echo # +--echo # Both places are shown because they are different points in +--echo # working a row out, and the array has more than one element +--echo # so that there is a row left to read after the write. The +--echo # rows owed either way are the elements of the array the +--echo # document held when the walk began. +--echo # +CALL pkg.p4(); + +DROP PACKAGE pkg; +SET sql_mode= @save_sql_mode; + +--echo # +--echo # The arm that copes with there being no room for the copy a +--echo # variable is passed as is reached only by a debug build, +--echo # and is in func_json_variable_copy_oom. +--echo # diff --git a/mysql-test/main/func_json_charset.result b/mysql-test/main/func_json_charset.result new file mode 100644 index 0000000000000..572e98f29e900 --- /dev/null +++ b/mysql-test/main/func_json_charset.result @@ -0,0 +1,574 @@ +# +# Behavioral baseline: character sets. +# +# The same bytes are a different document in a different character +# set, and in some character sets the bytes that JSON uses as +# punctuation are ordinary letters. This test records the character +# set and collation every JSON function reports for its result, what +# each function makes of the same bytes under different labels, and +# what happens when arguments of different character sets meet in one +# call. +# +SET NAMES utf8mb4; +# +# 1. The character set and collation of every result. +# +SELECT CHARSET(JSON_ARRAY(1)) AS cs, COLLATION(JSON_ARRAY(1)) AS co; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_OBJECT('a',1)) AS cs, COLLATION(JSON_OBJECT('a',1)) AS co; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_SET('{}','$.a',1)) AS cs, COLLATION(JSON_SET('{}','$.a',1)) AS co; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_INSERT('{}','$.a',1)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_REPLACE('{"a":1}','$.a',2)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_REMOVE('{"a":1}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_ARRAY_APPEND('[1]','$',2)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_ARRAY_INSERT('[1]','$[0]',2)) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_MERGE('[1]','[2]')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_MERGE_PATCH('{"a":1}','{"b":2}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_EXTRACT('{"a":1}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_QUERY('{"a":{"b":1}}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_VALUE('{"a":1}','$.a')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_KEYS('{"a":1}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_SEARCH('["x"]','one','x')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_COMPACT('{}')) AS cs, CHARSET(JSON_LOOSE('{}')) AS cs2; +cs cs2 +utf8mb4 utf8mb4 +SELECT CHARSET(JSON_DETAILED('{}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_NORMALIZE('{}')) AS cs; +cs +utf8mb4 +SELECT CHARSET(JSON_TYPE('{}')) AS cs; +cs +utf8mb3 +# JSON_QUOTE and JSON_UNQUOTE, whose collations differ from the rest +SELECT CHARSET(JSON_QUOTE('a')) AS cs, COLLATION(JSON_QUOTE('a')) AS co; +cs co +utf8mb4 utf8mb4_bin +SELECT CHARSET(JSON_UNQUOTE('"a"')) AS cs, COLLATION(JSON_UNQUOTE('"a"')) AS co; +cs co +utf8mb4 utf8mb4_bin +SELECT COLLATION(CONCAT(JSON_UNQUOTE('"a"'), '')) AS co; +co +utf8mb4_bin +SELECT JSON_UNQUOTE('"A"') = 'a' AS case_sensitive_compare; +case_sensitive_compare +0 +SELECT JSON_VALUE('{"a":"A"}','$.a') = 'a' AS case_sensitive_compare; +case_sensitive_compare +1 +# the aggregates +SELECT CHARSET(JSON_ARRAYAGG(x)) AS cs, COLLATION(JSON_ARRAYAGG(x)) AS co +FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +cs co +utf8mb4 utf8mb4_general_ci +SELECT CHARSET(JSON_OBJECTAGG(x,x)) AS cs FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +cs +utf8mb4 +# +# 2. The same bytes under different labels. +# +# 0xFF is one character in latin1 and not a character at all in utf8mb4 +SELECT JSON_VALID(_latin1 X'227B7DFF22') AS as_latin1; +as_latin1 +1 +SELECT JSON_VALID(_binary X'227B7DFF22') AS as_binary; +as_binary +1 +SELECT HEX(JSON_UNQUOTE(_latin1 X'22FF22')) AS unq_latin1; +unq_latin1 +C3BF +SELECT HEX(JSON_UNQUOTE(_binary X'22FF22')) AS unq_binary; +unq_binary +C3BF +# the byte 0x5C is a backslash in latin1 and utf8mb4 +SELECT JSON_VALID(_latin1 X'225C6E22') AS as_latin1; +as_latin1 +1 +SELECT JSON_VALID(_utf8mb4 X'225C6E22') AS as_utf8mb4; +as_utf8mb4 +1 +# a two-byte character that ends in a byte JSON treats as punctuation +SELECT JSON_VALID(CONVERT(_latin1 X'227B7D22' USING utf8mb4)) AS v; +v +1 +# +# 3. A character set in which the JSON punctuation bytes are letters. +# In swe7 the bytes 5B 5C 5D 7B 7D are national characters, so the +# same bytes that encode an object elsewhere encode a word here. +# +SELECT JSON_VALID(_swe7 X'7B7D') AS braces_as_swe7; +braces_as_swe7 +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID(_swe7 X'5B5D') AS brackets_as_swe7; +brackets_as_swe7 +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID(_binary X'7B7D') AS braces_as_binary; +braces_as_binary +1 +SELECT HEX(CONVERT(_swe7 X'7B7D' USING utf8mb4)) AS swe7_to_utf8mb4; +swe7_to_utf8mb4 +C3A4C3A5 +SELECT HEX(JSON_ARRAY(_swe7 X'7B7D')) AS embedded; +embedded +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +SELECT HEX(JSON_QUOTE(_swe7 X'7B7D')) AS quoted; +quoted +22C3A4C3A522 +SELECT JSON_VALID(JSON_ARRAY(_swe7 X'7B7D')) AS still_valid; +still_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +# a document written in swe7 itself +SELECT HEX(JSON_COMPACT(CONVERT('{"a":1}' USING swe7))) AS as_swe7; +as_swe7 +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_compact' at position 1 +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS as_swe7; +as_swe7 +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 1 +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS cs; +cs +swe7 +# a swe7 value spliced into a utf8mb4 document +SELECT HEX(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS spliced; +spliced +7B2261223A20312C202262223A2022C3A4C3A5227D +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS ok; +ok +1 +# +# 4. Wide character sets, where a character is not a byte. +# +SELECT HEX(CONVERT('{"a":1}' USING ucs2)) AS doc_ucs2; +doc_ucs2 +007B002200610022003A0031007D +SELECT JSON_VALID(CONVERT('{"a":1}' USING ucs2)) AS v; +v +1 +SELECT JSON_DEPTH(CONVERT('{"a":{"b":1}}' USING ucs2)) AS d; +d +3 +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS compacted; +compacted +007B002200610022003A0031007D +SELECT CHARSET(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS cs; +cs +ucs2 +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS setv; +setv +007B002200610022003A00200031002C0020002200620022003A00200032007D +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS cs; +cs +ucs2 +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), '$.a') AS v; +v +NULL +SELECT HEX(JSON_ARRAY(CONVERT('x' USING ucs2))) AS arr; +arr +005B002200780022005D +SELECT CHARSET(JSON_ARRAY(CONVERT('x' USING ucs2))) AS cs; +cs +ucs2 +# utf16 and utf32 +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf16)) AS v16; +v16 +1 +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf32)) AS v32; +v32 +1 +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING utf16))) AS c16; +c16 +007B002200610022003A0031007D +# +# 5. Arguments of different character sets meeting in one call. +# +SELECT CHARSET(JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4))) AS cs; +cs +utf8mb4 +SELECT JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4)) AS v; +v +["a", "b"] +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', +CONVERT('x' USING utf8mb4))) AS cs; +cs +latin1 +SELECT JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', +CONVERT('x' USING utf8mb4)) AS v; +v +{"a": 1, "b": "x"} +# a ucs2 document with a utf8mb4 value spliced into it +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +CONVERT('x' USING utf8mb4))) AS cs; +cs +ucs2 +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +CONVERT('x' USING utf8mb4))) AS v; +v +007B002200610022003A00200031002C0020002200620022003A0020002200780022007D +# a utf8mb4 document with a ucs2 value spliced into it +SELECT CHARSET(JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2))) AS cs; +cs +utf8mb4 +SELECT JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2)) AS v; +v +{"a": 1, "b": "x"} +# a path argument in another character set than the document +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING latin1)) AS v; +v +1 +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING ucs2)) AS v; +v +NULL +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), CONVERT('$.a' USING latin1)) AS v; +v +NULL +# 5a. A value that is itself a document, spliced into a document of +# another character set. The fragment is written in the character +# set of the answer, and what differs between the functions is how +# that set is chosen: a mutator takes it from its document argument +# and converts the fragment into it, while a constructor aggregates +# over all of its arguments, so the fragment itself can decide the +# set and its bytes then go across unchanged. +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS typed_into_ucs2; +typed_into_ucs2 +007B002200610022003A00200031002C0020002200620022003A0020007B002200780022003A00200032007D007D +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS ok; +ok +1 +SELECT HEX(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS latin1_typed; +latin1_typed +7B2261223A20312C202262223A207B2278223A2022C3A9227D7D +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS ok; +ok +1 +# the constructors take the same fragment without complaint +SELECT HEX(JSON_ARRAY( +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS embedded; +embedded +5B7B2278223A22E9227D5D +SELECT JSON_VALID(JSON_ARRAY( +JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS still_valid; +still_valid +1 +SELECT CHARSET(JSON_ARRAY(JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) AS cs; +cs +latin1 +SELECT HEX(JSON_OBJECT('k', JSON_COMPACT(CONVERT('{"x":2}' USING ucs2)))) AS obj_embedded; +obj_embedded +007B0022006B0022003A0020007B002200780022003A0032007D007D +# +# 6. Binary arguments, which are not text and are copied through. +# +CREATE TABLE t1 (b BLOB, v VARBINARY(10), c VARCHAR(10) CHARACTER SET latin1); +INSERT INTO t1 VALUES (X'FF', X'FF', X'FF'), (X'C3', X'C3', X'C3'); +SELECT CHARSET(JSON_ARRAY(b)) AS cs, COLLATION(JSON_ARRAY(b)) AS co FROM t1 LIMIT 1; +cs co +binary binary +SELECT HEX(JSON_ARRAY(b)) AS h, JSON_VALID(JSON_ARRAY(b)) AS ok FROM t1; +h ok +5B22FF225D 1 +5B22C3225D 1 +SELECT HEX(JSON_ARRAY(v)) AS h, JSON_VALID(JSON_ARRAY(v)) AS ok FROM t1; +h ok +5B22FF225D 1 +5B22C3225D 1 +SELECT HEX(JSON_ARRAY(c)) AS h, JSON_VALID(JSON_ARRAY(c)) AS ok FROM t1; +h ok +5B22FF225D 1 +5B22C3225D 1 +SELECT CHARSET(JSON_ARRAY(c)) AS cs FROM t1 LIMIT 1; +cs +latin1 +SELECT HEX(JSON_OBJECT('k', b)) AS h FROM t1; +h +7B226B223A2022FF227D +7B226B223A2022C3227D +SELECT HEX(JSON_SET('{}', '$.a', b)) AS h FROM t1; +h +7B2261223A2022C3BF227D +7B2261223A2022C383227D +SELECT CHARSET(JSON_SET('{}', '$.a', b)) AS cs FROM t1 LIMIT 1; +cs +utf8mb4 +SELECT HEX(JSON_QUOTE(b)) AS h FROM t1; +h +22C3BF22 +22C38322 +# the raw bytes survive a round trip through a document +SELECT HEX(JSON_VALUE(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +h +FF +C3 +SELECT HEX(JSON_EXTRACT(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +h +22FF22 +22C322 +DROP TABLE t1; +# 6a. A binary column holding a whole document, with bytes that are +# not valid utf8mb4 sitting in a value, in a key, and in the +# structural skeleton. Nothing converts these, so what the +# parser accepts is decided by the binary label alone. +CREATE TABLE t2 (b VARBINARY(60)); +INSERT INTO t2 VALUES +(X'7B2261223A22FF227D'), (X'7B22FF61223A317D'), (X'7B2261223A31FF7D'), +(X'7B2261223A22C3227D'), (X'7B2261223A22C3A9227D'); +SELECT HEX(b) AS stored, JSON_VALID(b) AS v FROM t2 ORDER BY stored; +stored v +7B2261223A22C3227D 1 +7B2261223A22C3A9227D 1 +7B2261223A22FF227D 1 +7B2261223A31FF7D 0 +7B22FF61223A317D 1 +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 6 +SELECT HEX(b) AS stored, HEX(JSON_EXTRACT(b, '$.a')) AS ext FROM t2 ORDER BY stored; +stored ext +7B2261223A22C3227D 22C322 +7B2261223A22C3A9227D 22C3A922 +7B2261223A22FF227D 22FF22 +7B2261223A31FF7D NULL +7B22FF61223A317D NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_extract' at position 6 +SELECT HEX(b) AS stored, HEX(JSON_VALUE(b, '$.a')) AS val FROM t2 ORDER BY stored; +stored val +7B2261223A22C3227D C3 +7B2261223A22C3A9227D C3A9 +7B2261223A22FF227D FF +7B2261223A31FF7D NULL +7B22FF61223A317D NULL +SELECT HEX(b) AS stored, HEX(JSON_KEYS(b)) AS ks FROM t2 ORDER BY stored; +stored ks +7B2261223A22C3227D 5B2261225D +7B2261223A22C3A9227D 5B2261225D +7B2261223A22FF227D 5B2261225D +7B2261223A31FF7D NULL +7B22FF61223A317D 5B22FF61225D +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_keys' at position 6 +SELECT HEX(b) AS stored, HEX(JSON_SET(b, '$.z', 1)) AS mutated FROM t2 ORDER BY stored; +stored mutated +7B2261223A22C3227D 7B2261223A2022C3222C20227A223A20317D +7B2261223A22C3A9227D 7B2261223A2022C3A9222C20227A223A20317D +7B2261223A22FF227D 7B2261223A2022FF222C20227A223A20317D +7B2261223A31FF7D NULL +7B22FF61223A317D 7B22FF61223A20312C20227A223A20317D +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_set' at position 6 +SELECT CHARSET(JSON_EXTRACT(b, '$.a')) AS cs_ext, +CHARSET(JSON_SET(b, '$.z', 1)) AS cs_set FROM t2 LIMIT 1; +cs_ext cs_set +binary binary +DROP TABLE t2; +# +# 6b. CONVERT and CAST over a value that is already a document. +# CONVERT wraps the value and the wrapper is seen through, so the +# result is still treated as a document; CAST does not. +# +CREATE TABLE tc (j JSON); +INSERT INTO tc VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +SELECT HEX(JSON_ARRAY(CONVERT(j USING latin1))) AS h, +CHARSET(JSON_ARRAY(CONVERT(j USING latin1))) AS cs FROM tc; +h cs +5B7B2261223A20312C202262223A2022E9227D5D latin1 +SELECT JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tc; +ok +1 +# a mutator takes the same fragment and writes it back out in the +# character set of its document argument, which is where the answer's +# character set comes from +SELECT HEX(JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1))) AS h FROM tc; +h +7B2278223A20312C202279223A207B2261223A20312C202262223A2022C3A9227D7D +SELECT HEX(JSON_ARRAY(CONVERT(j USING binary))) AS h FROM tc; +h +5B7B2261223A20312C202262223A2022C3A9227D5D +# CAST produces a plain string, so it is quoted rather than embedded +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR))) AS h FROM tc; +h +5B227B5C22615C223A20312C205C22625C223A205C22C3A95C227D225D +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR CHARACTER SET latin1))) AS h FROM tc; +h +5B227B5C22615C223A20312C205C22625C223A205C22E95C227D225D +SELECT HEX(JSON_ARRAY(CONVERT(JSON_OBJECT('a', _utf8mb4 X'C3A9') USING latin1))) AS h; +h +5B7B2261223A2022E9227D5D +SELECT HEX(JSON_SET('{}', '$.a', CONVERT(JSON_OBJECT('k','v') USING latin1))) AS h; +h +7B2261223A207B226B223A202276227D7D +DROP TABLE tc; +# the same wrapper over a column whose stored bytes are not a document +CREATE TABLE tp (j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY(CONVERT(j USING latin1)) AS v, +JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tp; +v ok +NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1)) AS v FROM tp; +v +NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 20 +SELECT JSON_ARRAY(CAST(j AS CHAR)) AS v FROM tp; +v +["{\"a\":1,"] +DROP TABLE tp; +# +# 7. The metadata the client is told about, which decides whether the +# bytes are converted on the way out. +# +SELECT JSON_ARRAY(1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 40 3 Y 0 39 45 +v +[1] +SELECT JSON_OBJECT('a',1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 72 8 Y 0 39 45 +v +{"a": 1} +SELECT JSON_SET('{}','$.a',1) AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 84 8 Y 0 39 45 +v +{"a": 1} +SELECT JSON_EXTRACT('{"a":1}','$.a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 56 1 Y 0 39 45 +v +1 +SELECT JSON_UNQUOTE('"a"') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 12 1 Y 128 39 45 +v +a +SELECT JSON_QUOTE('a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 32 3 Y 128 39 45 +v +"a" +SELECT JSON_KEYS('{"a":1}') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 28 5 Y 0 39 45 +v +["a"] +SELECT JSON_COMPACT('{}') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 8 2 Y 0 0 45 +v +{} +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 16777224 5 Y 0 0 45 +v +[1,2] +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 252 (format=json) 16777224 14 Y 0 0 45 +v +{"1":1, "2":2} +# +# 8. A result delivered in a character set other than the one it was +# computed in. +# +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAY('a') AS v; +v +["a"] +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +["a","b"] +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +{"a":"a", "b":"b"} +SELECT JSON_SET('{}','$.a','b') AS v; +v +{"a": "b"} +SET SESSION character_set_results = DEFAULT; +# The values above are all ASCII, so a conversion on the way out is +# indistinguishable from no conversion. swe7 has no code point for +# the JSON punctuation, so a result that IS converted arrives with +# those characters replaced, and a result that is passed +# unconverted arrives intact. That makes the difference visible. +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('a') AS v; +v +?"a"? +SELECT JSON_OBJECT('a',1) AS v; +v +?"a": 1? +SELECT JSON_SET('{}','$.a','b') AS v; +v +?"a": "b"? +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +v +?1, 2? +SELECT JSON_KEYS('{"a":1}') AS v; +v +?"a"? +SELECT JSON_COMPACT('{"a": 1}') AS v; +v +?"a":1? +SELECT JSON_UNQUOTE('"[a]"') AS v; +v +?a? +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +?"a","b"? +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +v +?"a":"a", "b":"b"? +SET SESSION character_set_results = DEFAULT; +SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_charset.test b/mysql-test/main/func_json_charset.test new file mode 100644 index 0000000000000..321addb6f7540 --- /dev/null +++ b/mysql-test/main/func_json_charset.test @@ -0,0 +1,274 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: character sets. +--echo # +--echo # The same bytes are a different document in a different character +--echo # set, and in some character sets the bytes that JSON uses as +--echo # punctuation are ordinary letters. This test records the character +--echo # set and collation every JSON function reports for its result, what +--echo # each function makes of the same bytes under different labels, and +--echo # what happens when arguments of different character sets meet in one +--echo # call. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. The character set and collation of every result. +--echo # + +SELECT CHARSET(JSON_ARRAY(1)) AS cs, COLLATION(JSON_ARRAY(1)) AS co; +SELECT CHARSET(JSON_OBJECT('a',1)) AS cs, COLLATION(JSON_OBJECT('a',1)) AS co; +SELECT CHARSET(JSON_SET('{}','$.a',1)) AS cs, COLLATION(JSON_SET('{}','$.a',1)) AS co; +SELECT CHARSET(JSON_INSERT('{}','$.a',1)) AS cs; +SELECT CHARSET(JSON_REPLACE('{"a":1}','$.a',2)) AS cs; +SELECT CHARSET(JSON_REMOVE('{"a":1}','$.a')) AS cs; +SELECT CHARSET(JSON_ARRAY_APPEND('[1]','$',2)) AS cs; +SELECT CHARSET(JSON_ARRAY_INSERT('[1]','$[0]',2)) AS cs; +SELECT CHARSET(JSON_MERGE('[1]','[2]')) AS cs; +SELECT CHARSET(JSON_MERGE_PATCH('{"a":1}','{"b":2}')) AS cs; +SELECT CHARSET(JSON_EXTRACT('{"a":1}','$.a')) AS cs; +SELECT CHARSET(JSON_QUERY('{"a":{"b":1}}','$.a')) AS cs; +SELECT CHARSET(JSON_VALUE('{"a":1}','$.a')) AS cs; +SELECT CHARSET(JSON_KEYS('{"a":1}')) AS cs; +SELECT CHARSET(JSON_SEARCH('["x"]','one','x')) AS cs; +SELECT CHARSET(JSON_COMPACT('{}')) AS cs, CHARSET(JSON_LOOSE('{}')) AS cs2; +SELECT CHARSET(JSON_DETAILED('{}')) AS cs; +SELECT CHARSET(JSON_NORMALIZE('{}')) AS cs; +SELECT CHARSET(JSON_TYPE('{}')) AS cs; +--echo # JSON_QUOTE and JSON_UNQUOTE, whose collations differ from the rest +SELECT CHARSET(JSON_QUOTE('a')) AS cs, COLLATION(JSON_QUOTE('a')) AS co; +SELECT CHARSET(JSON_UNQUOTE('"a"')) AS cs, COLLATION(JSON_UNQUOTE('"a"')) AS co; +SELECT COLLATION(CONCAT(JSON_UNQUOTE('"a"'), '')) AS co; +SELECT JSON_UNQUOTE('"A"') = 'a' AS case_sensitive_compare; +SELECT JSON_VALUE('{"a":"A"}','$.a') = 'a' AS case_sensitive_compare; +--echo # the aggregates +SELECT CHARSET(JSON_ARRAYAGG(x)) AS cs, COLLATION(JSON_ARRAYAGG(x)) AS co + FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +SELECT CHARSET(JSON_OBJECTAGG(x,x)) AS cs FROM (SELECT 1 AS x UNION ALL SELECT 2) d; + +--echo # +--echo # 2. The same bytes under different labels. +--echo # + +--echo # 0xFF is one character in latin1 and not a character at all in utf8mb4 +SELECT JSON_VALID(_latin1 X'227B7DFF22') AS as_latin1; +SELECT JSON_VALID(_binary X'227B7DFF22') AS as_binary; +SELECT HEX(JSON_UNQUOTE(_latin1 X'22FF22')) AS unq_latin1; +SELECT HEX(JSON_UNQUOTE(_binary X'22FF22')) AS unq_binary; +--echo # the byte 0x5C is a backslash in latin1 and utf8mb4 +SELECT JSON_VALID(_latin1 X'225C6E22') AS as_latin1; +SELECT JSON_VALID(_utf8mb4 X'225C6E22') AS as_utf8mb4; +--echo # a two-byte character that ends in a byte JSON treats as punctuation +SELECT JSON_VALID(CONVERT(_latin1 X'227B7D22' USING utf8mb4)) AS v; + +--echo # +--echo # 3. A character set in which the JSON punctuation bytes are letters. +--echo # In swe7 the bytes 5B 5C 5D 7B 7D are national characters, so the +--echo # same bytes that encode an object elsewhere encode a word here. +--echo # + +SELECT JSON_VALID(_swe7 X'7B7D') AS braces_as_swe7; +SELECT JSON_VALID(_swe7 X'5B5D') AS brackets_as_swe7; +SELECT JSON_VALID(_binary X'7B7D') AS braces_as_binary; +SELECT HEX(CONVERT(_swe7 X'7B7D' USING utf8mb4)) AS swe7_to_utf8mb4; +SELECT HEX(JSON_ARRAY(_swe7 X'7B7D')) AS embedded; +SELECT HEX(JSON_QUOTE(_swe7 X'7B7D')) AS quoted; +SELECT JSON_VALID(JSON_ARRAY(_swe7 X'7B7D')) AS still_valid; +--echo # a document written in swe7 itself +SELECT HEX(JSON_COMPACT(CONVERT('{"a":1}' USING swe7))) AS as_swe7; +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS as_swe7; +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING swe7), '$.b', 2)) AS cs; +--echo # a swe7 value spliced into a utf8mb4 document +SELECT HEX(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS spliced; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', CONVERT(_swe7 X'7B7D' USING swe7))) AS ok; + +--echo # +--echo # 4. Wide character sets, where a character is not a byte. +--echo # + +SELECT HEX(CONVERT('{"a":1}' USING ucs2)) AS doc_ucs2; +SELECT JSON_VALID(CONVERT('{"a":1}' USING ucs2)) AS v; +SELECT JSON_DEPTH(CONVERT('{"a":{"b":1}}' USING ucs2)) AS d; +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS compacted; +SELECT CHARSET(JSON_COMPACT(CONVERT('{"a": 1}' USING ucs2))) AS cs; +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS setv; +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', 2)) AS cs; +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), '$.a') AS v; +SELECT HEX(JSON_ARRAY(CONVERT('x' USING ucs2))) AS arr; +SELECT CHARSET(JSON_ARRAY(CONVERT('x' USING ucs2))) AS cs; +--echo # utf16 and utf32 +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf16)) AS v16; +SELECT JSON_VALID(CONVERT('{"a":1}' USING utf32)) AS v32; +SELECT HEX(JSON_COMPACT(CONVERT('{"a": 1}' USING utf16))) AS c16; + +--echo # +--echo # 5. Arguments of different character sets meeting in one call. +--echo # + +SELECT CHARSET(JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4))) AS cs; +SELECT JSON_ARRAY(CONVERT('a' USING latin1), CONVERT('b' USING utf8mb4)) AS v; +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', + CONVERT('x' USING utf8mb4))) AS cs; +SELECT JSON_SET(CONVERT('{"a":1}' USING latin1), '$.b', + CONVERT('x' USING utf8mb4)) AS v; +--echo # a ucs2 document with a utf8mb4 value spliced into it +SELECT CHARSET(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + CONVERT('x' USING utf8mb4))) AS cs; +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + CONVERT('x' USING utf8mb4))) AS v; +--echo # a utf8mb4 document with a ucs2 value spliced into it +SELECT CHARSET(JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2))) AS cs; +SELECT JSON_SET('{"a":1}', '$.b', CONVERT('x' USING ucs2)) AS v; +--echo # a path argument in another character set than the document +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING latin1)) AS v; +SELECT JSON_EXTRACT('{"a":1}', CONVERT('$.a' USING ucs2)) AS v; +SELECT JSON_EXTRACT(CONVERT('{"a":1}' USING ucs2), CONVERT('$.a' USING latin1)) AS v; + +--echo # 5a. A value that is itself a document, spliced into a document of +--echo # another character set. The fragment is written in the character +--echo # set of the answer, and what differs between the functions is how +--echo # that set is chosen: a mutator takes it from its document argument +--echo # and converts the fragment into it, while a constructor aggregates +--echo # over all of its arguments, so the fragment itself can decide the +--echo # set and its bytes then go across unchanged. +SELECT HEX(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS typed_into_ucs2; +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS ok; +SELECT HEX(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS latin1_typed; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS ok; +--echo # the constructors take the same fragment without complaint +SELECT HEX(JSON_ARRAY( + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS embedded; +SELECT JSON_VALID(JSON_ARRAY( + JSON_COMPACT(CONVERT(CONCAT('{"x":"', X'E9', '"}') USING latin1)))) AS still_valid; +SELECT CHARSET(JSON_ARRAY(JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) AS cs; +SELECT HEX(JSON_OBJECT('k', JSON_COMPACT(CONVERT('{"x":2}' USING ucs2)))) AS obj_embedded; + +--echo # +--echo # 6. Binary arguments, which are not text and are copied through. +--echo # + +CREATE TABLE t1 (b BLOB, v VARBINARY(10), c VARCHAR(10) CHARACTER SET latin1); +INSERT INTO t1 VALUES (X'FF', X'FF', X'FF'), (X'C3', X'C3', X'C3'); +SELECT CHARSET(JSON_ARRAY(b)) AS cs, COLLATION(JSON_ARRAY(b)) AS co FROM t1 LIMIT 1; +SELECT HEX(JSON_ARRAY(b)) AS h, JSON_VALID(JSON_ARRAY(b)) AS ok FROM t1; +SELECT HEX(JSON_ARRAY(v)) AS h, JSON_VALID(JSON_ARRAY(v)) AS ok FROM t1; +SELECT HEX(JSON_ARRAY(c)) AS h, JSON_VALID(JSON_ARRAY(c)) AS ok FROM t1; +SELECT CHARSET(JSON_ARRAY(c)) AS cs FROM t1 LIMIT 1; +SELECT HEX(JSON_OBJECT('k', b)) AS h FROM t1; +SELECT HEX(JSON_SET('{}', '$.a', b)) AS h FROM t1; +SELECT CHARSET(JSON_SET('{}', '$.a', b)) AS cs FROM t1 LIMIT 1; +SELECT HEX(JSON_QUOTE(b)) AS h FROM t1; +--echo # the raw bytes survive a round trip through a document +SELECT HEX(JSON_VALUE(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +SELECT HEX(JSON_EXTRACT(JSON_OBJECT('k', b), '$.k')) AS h FROM t1; +DROP TABLE t1; + +--echo # 6a. A binary column holding a whole document, with bytes that are +--echo # not valid utf8mb4 sitting in a value, in a key, and in the +--echo # structural skeleton. Nothing converts these, so what the +--echo # parser accepts is decided by the binary label alone. +CREATE TABLE t2 (b VARBINARY(60)); +INSERT INTO t2 VALUES + (X'7B2261223A22FF227D'), (X'7B22FF61223A317D'), (X'7B2261223A31FF7D'), + (X'7B2261223A22C3227D'), (X'7B2261223A22C3A9227D'); +SELECT HEX(b) AS stored, JSON_VALID(b) AS v FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_EXTRACT(b, '$.a')) AS ext FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_VALUE(b, '$.a')) AS val FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_KEYS(b)) AS ks FROM t2 ORDER BY stored; +SELECT HEX(b) AS stored, HEX(JSON_SET(b, '$.z', 1)) AS mutated FROM t2 ORDER BY stored; +SELECT CHARSET(JSON_EXTRACT(b, '$.a')) AS cs_ext, + CHARSET(JSON_SET(b, '$.z', 1)) AS cs_set FROM t2 LIMIT 1; +DROP TABLE t2; + +--echo # +--echo # 6b. CONVERT and CAST over a value that is already a document. +--echo # CONVERT wraps the value and the wrapper is seen through, so the +--echo # result is still treated as a document; CAST does not. +--echo # +CREATE TABLE tc (j JSON); +INSERT INTO tc VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +SELECT HEX(JSON_ARRAY(CONVERT(j USING latin1))) AS h, + CHARSET(JSON_ARRAY(CONVERT(j USING latin1))) AS cs FROM tc; +SELECT JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tc; +--echo # a mutator takes the same fragment and writes it back out in the +--echo # character set of its document argument, which is where the answer's +--echo # character set comes from +SELECT HEX(JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1))) AS h FROM tc; +SELECT HEX(JSON_ARRAY(CONVERT(j USING binary))) AS h FROM tc; +--echo # CAST produces a plain string, so it is quoted rather than embedded +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR))) AS h FROM tc; +SELECT HEX(JSON_ARRAY(CAST(j AS CHAR CHARACTER SET latin1))) AS h FROM tc; +SELECT HEX(JSON_ARRAY(CONVERT(JSON_OBJECT('a', _utf8mb4 X'C3A9') USING latin1))) AS h; +SELECT HEX(JSON_SET('{}', '$.a', CONVERT(JSON_OBJECT('k','v') USING latin1))) AS h; +DROP TABLE tc; +--echo # the same wrapper over a column whose stored bytes are not a document +CREATE TABLE tp (j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY(CONVERT(j USING latin1)) AS v, + JSON_VALID(JSON_ARRAY(CONVERT(j USING latin1))) AS ok FROM tp; +SELECT JSON_SET('{"x":1}', '$.y', CONVERT(j USING latin1)) AS v FROM tp; +SELECT JSON_ARRAY(CAST(j AS CHAR)) AS v FROM tp; +DROP TABLE tp; + +--echo # +--echo # 7. The metadata the client is told about, which decides whether the +--echo # bytes are converted on the way out. +--echo # + +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_ARRAY(1) AS v; +SELECT JSON_OBJECT('a',1) AS v; +SELECT JSON_SET('{}','$.a',1) AS v; +SELECT JSON_EXTRACT('{"a":1}','$.a') AS v; +SELECT JSON_UNQUOTE('"a"') AS v; +SELECT JSON_QUOTE('a') AS v; +SELECT JSON_KEYS('{"a":1}') AS v; +SELECT JSON_COMPACT('{}') AS v; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 1 AS x UNION ALL SELECT 2) d; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +--echo # +--echo # 8. A result delivered in a character set other than the one it was +--echo # computed in. +--echo # + +SET SESSION character_set_results = latin1; +SELECT JSON_ARRAY('a') AS v; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SELECT JSON_SET('{}','$.a','b') AS v; +SET SESSION character_set_results = DEFAULT; + +--echo # The values above are all ASCII, so a conversion on the way out is +--echo # indistinguishable from no conversion. swe7 has no code point for +--echo # the JSON punctuation, so a result that IS converted arrives with +--echo # those characters replaced, and a result that is passed +--echo # unconverted arrives intact. That makes the difference visible. +SET SESSION character_set_results = swe7; +SELECT JSON_ARRAY('a') AS v; +SELECT JSON_OBJECT('a',1) AS v; +SELECT JSON_SET('{}','$.a','b') AS v; +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +SELECT JSON_KEYS('{"a":1}') AS v; +SELECT JSON_COMPACT('{"a": 1}') AS v; +SELECT JSON_UNQUOTE('"[a]"') AS v; +SELECT JSON_ARRAYAGG(x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SELECT JSON_OBJECTAGG(x,x) AS v FROM (SELECT 'a' AS x UNION ALL SELECT 'b') d; +SET SESSION character_set_results = DEFAULT; +SET NAMES utf8mb4; diff --git a/mysql-test/main/func_json_check_scan_count.result b/mysql-test/main/func_json_check_scan_count.result new file mode 100644 index 0000000000000..bc6dbbbba4423 --- /dev/null +++ b/mysql-test/main/func_json_check_scan_count.result @@ -0,0 +1,338 @@ +# +# How many times a column's check constraint reads what is stored. +# +# A JSON column's check reads every value written into it to find +# out whether it is a document. Where the value came out of a JSON +# function, that has been answered already, and the reading finds +# out nothing. +# +# No query can see the difference - a value that is a document is +# stored either way - so nothing but Json_scans says whether the +# reading happened. The counts below are the record of it. A count +# that goes up is a reading that came back. +# +# A debug build reads values back twice over to check what was +# claimed about them - once for what the JSON functions claim, and +# once, wherever a check is left unrun, for what the column was +# marked with. Neither reading is counted: they are the debug +# build's work and not the server's, and counting them would move +# these numbers for reasons no query is responsible for. A count +# of 0 below therefore means the check did not read, not that the +# reading went uncounted. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, j VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +# +# ONE row, from a function that builds a document out of scalars. +# Building it reads nothing - there is nothing there to read - so +# every reading counted here is the check reading the value back. +# +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# TWO such rows. The check reads once per row, so this is twice +# the count above and nothing else. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)), (3, JSON_ARRAY(1, 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# TWO rows written out as literals. Nothing has read them, so the +# check is the only thing that can find out whether they are +# documents, and its reading is not work that can be taken away. +# Two rows, one reading each. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (4, '{"a":1}'), (5, '[1,2]'); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO rows, each put through JSON_COMPACT on the way in - so TWO +# calls, not one. Writing a document out compactly means reading +# it, and that reading is the function's own and stays. Each row +# therefore costs one reading for the function, plus whatever the +# check still costs. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (6, JSON_COMPACT('{"a":1}')), (7, JSON_COMPACT('[1,2]')); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO rows updated, so TWO calls to JSON_SET. Editing a document +# that came out of a column costs the function TWO readings of its +# own - one to find the place to edit and one to read its result +# back, the column having no say in how it was formatted. Four +# readings for the two rows, plus whatever the check still costs. +# +DELETE FROM t1; +INSERT INTO t1 VALUES (8, JSON_OBJECT('a', 1)), (9, JSON_OBJECT('b', 2)); +FLUSH STATUS; +UPDATE t1 SET j = JSON_SET(j, '$.c', 3); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# TWO rows read out of another table, one JSON_COMPACT call each, +# counting the same way as the two-call case above. +# +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 SELECT id, JSON_COMPACT(j) FROM t2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# ONE row, written twice over, and read once per writing: what the +# first writing attested does not carry over to the second. +# Both values are built out of scalars, so neither costs a reading +# of its own and the count is the two checks alone. +# +DELETE FROM t1; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# ONE row into a column with no check at all, built out of scalars. +# Nothing reads it anywhere, and nothing is counted. +# +FLUSH STATUS; +INSERT INTO t2 VALUES (3, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +DROP TABLE t2; +# +# TWO rows through a trigger that edits the value on the way in. +# The trigger's JSON_INSERT is handed NEW.j, which is a column and +# says nothing about itself, so it costs the same TWO readings per +# row as the update above - four for the two rows - plus whatever +# the check still costs. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +DROP TABLE t1; +# +# ONE row, into a column whose check reads a DIFFERENT column. +# What was written into the one says nothing about the other, so +# the check keeps reading: one reading, and it stays. +# +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +DROP TABLE t1; +# +# TWO rows into a column whose check asks something else as well. +# Both values are built out of scalars, so the count is the checks +# alone - one per row, and both stay. +# +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 40)); +Warnings: +Warning 4271 CHECK constraint of column 'j' asks more than JSON_VALID() of it; the column is not a JSON column +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# TWO rows into a table with a table-level check, which is not a +# column's check whatever it asks. One reading per row, and both +# stay. +# +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# The same function asked directly, outside any check. +# +# A column's check IS json_valid(), so the reading left unrun +# above is the one json_valid() itself makes, and it is left +# unrun for the same reason wherever the value has answered. +# TWO values built out of scalars, neither of them read. +# +FLUSH STATUS; +SELECT JSON_VALID(JSON_OBJECT('a', 1)) AS built_object, +JSON_VALID(JSON_ARRAY(1, 2)) AS built_array; +built_object built_array +1 1 +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# The same two values written out, which nothing has answered +# for. One reading each, and both stay. +# +FLUSH STATUS; +SELECT JSON_VALID('{"a": 1}') AS written_object, +JSON_VALID('[1, 2]') AS written_array; +written_object written_array +1 1 +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO values that are not documents at all. Each is read, and +# each says so. +# +FLUSH STATUS; +SELECT JSON_VALID('not a document') AS text, +JSON_VALID('[1, 2') AS unfinished; +text unfinished +0 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# TWO rows into a COMPRESSED column whose check asks nothing but +# whether it holds a document. A compressed store does not put +# down the characters it was handed: it puts down an image of them +# behind a header of its own. Nothing the value answered can be +# carried by bytes like that, so the check reads for itself - one +# reading per row, and both stay. +# +# The two values are built out of scalars and cost no reading of +# their own. They differ in length on purpose: the short one is +# put down as it stands behind the header and the long one is +# packed, and neither is the text that was passed. +# +CREATE TABLE t1 (j VARCHAR(1024) COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# The same two rows into a LONGTEXT COMPRESSED column, which is a +# different class of field storing the same way. Both compressed +# classes answer the same question about themselves, so neither +# has to be named for either to be refused. +# +CREATE TABLE t1 (j LONGTEXT COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t1; +# +# THE CONTROL: the same two values into the same two column types +# with the compression taken off. These put down what they were +# handed, so the check is left unrun and nothing is counted - which +# is what makes the counts above a fact about compressing rather +# than about the values, the widths or the types. +# +CREATE TABLE t1 (j VARCHAR(1024) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (j LONGTEXT CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +INSERT INTO t2 VALUES (JSON_OBJECT('a', 1)), +(JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +DROP TABLE t2; +# +# TWO rows whose checked column is written by a trigger and by +# nothing else. An assignment in a trigger is a store like any +# other and attests to what it put down, so the check is left +# unrun. The value is built out of scalars and reads nothing of +# its own, so nothing at all is counted. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +# +# The same trigger over a table where no column carries a check. +# Nothing here can ever be asked whether it holds a document, so +# the assignment does not stop to attest to what it stored - the +# restraint the other two store sites already showed. A debug +# build stops the server where a mark is set that nothing can +# read, which is what says the restraint is being shown. +# +CREATE TABLE t1 (id INT, j VARCHAR(64)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +# +# A row cut in two, where each piece is written over the row image +# the piece before it left behind. +# +# An update over part of a period writes back, as rows of their +# own, the pieces the update did not cover. Each piece is built by +# putting the row back as it stood and writing one end of the +# period over it, and the check runs on what that came to - so the +# answer a trigger gave for the piece before is an answer about +# bytes that are no longer there. +# +# THREE readings: one for the update itself, whose SET does not +# touch the checked column, and one for each of the two pieces. A +# piece written over an answer that was left standing would show up +# as a count of two. +# +CREATE TABLE t1 (id INT, s DATE, e DATE, j VARCHAR(64) CHECK (JSON_VALID(j)), +PERIOD FOR p(s, e)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_OBJECT('seen', NEW.id); +INSERT INTO t1 (id, s, e) VALUES (1, '2000-01-01', '2010-01-01'); +FLUSH STATUS; +UPDATE t1 FOR PORTION OF p FROM '2002-01-01' TO '2003-01-01' SET id = 2; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +SELECT id, s, e, j FROM t1 ORDER BY s; +id s e j +1 2000-01-01 2002-01-01 {"seen": 1} +2 2002-01-01 2003-01-01 {"seen": 1} +1 2003-01-01 2010-01-01 {"seen": 1} +DROP TABLE t1; diff --git a/mysql-test/main/func_json_check_scan_count.test b/mysql-test/main/func_json_check_scan_count.test new file mode 100644 index 0000000000000..6184f3ef23f06 --- /dev/null +++ b/mysql-test/main/func_json_check_scan_count.test @@ -0,0 +1,315 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a column's check constraint reads what is stored. +--echo # +--echo # A JSON column's check reads every value written into it to find +--echo # out whether it is a document. Where the value came out of a JSON +--echo # function, that has been answered already, and the reading finds +--echo # out nothing. +--echo # +--echo # No query can see the difference - a value that is a document is +--echo # stored either way - so nothing but Json_scans says whether the +--echo # reading happened. The counts below are the record of it. A count +--echo # that goes up is a reading that came back. +--echo # +--echo # A debug build reads values back twice over to check what was +--echo # claimed about them - once for what the JSON functions claim, and +--echo # once, wherever a check is left unrun, for what the column was +--echo # marked with. Neither reading is counted: they are the debug +--echo # build's work and not the server's, and counting them would move +--echo # these numbers for reasons no query is responsible for. A count +--echo # of 0 below therefore means the check did not read, not that the +--echo # reading went uncounted. +--echo # + +SET NAMES utf8mb4; + +# Everything here is counted rather than read, so a statement that is run a +# second time to check that it repeats itself would count twice. +--disable_ps2_protocol +# A check constraint is looked at while the column definition is validated, +# which for a prepared CREATE TABLE is prepare time. +--enable_prepare_warnings + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, j VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +--echo # +--echo # ONE row, from a function that builds a document out of scalars. +--echo # Building it reads nothing - there is nothing there to read - so +--echo # every reading counted here is the check reading the value back. +--echo # +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO such rows. The check reads once per row, so this is twice +--echo # the count above and nothing else. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)), (3, JSON_ARRAY(1, 2)); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows written out as literals. Nothing has read them, so the +--echo # check is the only thing that can find out whether they are +--echo # documents, and its reading is not work that can be taken away. +--echo # Two rows, one reading each. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (4, '{"a":1}'), (5, '[1,2]'); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows, each put through JSON_COMPACT on the way in - so TWO +--echo # calls, not one. Writing a document out compactly means reading +--echo # it, and that reading is the function's own and stays. Each row +--echo # therefore costs one reading for the function, plus whatever the +--echo # check still costs. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 VALUES (6, JSON_COMPACT('{"a":1}')), (7, JSON_COMPACT('[1,2]')); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows updated, so TWO calls to JSON_SET. Editing a document +--echo # that came out of a column costs the function TWO readings of its +--echo # own - one to find the place to edit and one to read its result +--echo # back, the column having no say in how it was formatted. Four +--echo # readings for the two rows, plus whatever the check still costs. +--echo # +DELETE FROM t1; +INSERT INTO t1 VALUES (8, JSON_OBJECT('a', 1)), (9, JSON_OBJECT('b', 2)); +FLUSH STATUS; +UPDATE t1 SET j = JSON_SET(j, '$.c', 3); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows read out of another table, one JSON_COMPACT call each, +--echo # counting the same way as the two-call case above. +--echo # +DELETE FROM t1; +FLUSH STATUS; +INSERT INTO t1 SELECT id, JSON_COMPACT(j) FROM t2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # ONE row, written twice over, and read once per writing: what the +--echo # first writing attested does not carry over to the second. +--echo # Both values are built out of scalars, so neither costs a reading +--echo # of its own and the count is the two checks alone. +--echo # +DELETE FROM t1; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # ONE row into a column with no check at all, built out of scalars. +--echo # Nothing reads it anywhere, and nothing is counted. +--echo # +FLUSH STATUS; +INSERT INTO t2 VALUES (3, JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # TWO rows through a trigger that edits the value on the way in. +--echo # The trigger's JSON_INSERT is handed NEW.j, which is a column and +--echo # says nothing about itself, so it costs the same TWO readings per +--echo # row as the update above - four for the two rows - plus whatever +--echo # the check still costs. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # ONE row, into a column whose check reads a DIFFERENT column. +--echo # What was written into the one says nothing about the other, so +--echo # the check keeps reading: one reading, and it stays. +--echo # +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # TWO rows into a column whose check asks something else as well. +--echo # Both values are built out of scalars, so the count is the checks +--echo # alone - one per row, and both stay. +--echo # +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 40)); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # TWO rows into a table with a table-level check, which is not a +--echo # column's check whatever it asks. One reading per row, and both +--echo # stay. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)), (2, JSON_OBJECT('b', 2)); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # The same function asked directly, outside any check. +--echo # +--echo # A column's check IS json_valid(), so the reading left unrun +--echo # above is the one json_valid() itself makes, and it is left +--echo # unrun for the same reason wherever the value has answered. +--echo # TWO values built out of scalars, neither of them read. +--echo # +FLUSH STATUS; +SELECT JSON_VALID(JSON_OBJECT('a', 1)) AS built_object, + JSON_VALID(JSON_ARRAY(1, 2)) AS built_array; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same two values written out, which nothing has answered +--echo # for. One reading each, and both stay. +--echo # +FLUSH STATUS; +SELECT JSON_VALID('{"a": 1}') AS written_object, + JSON_VALID('[1, 2]') AS written_array; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO values that are not documents at all. Each is read, and +--echo # each says so. +--echo # +FLUSH STATUS; +SELECT JSON_VALID('not a document') AS text, + JSON_VALID('[1, 2') AS unfinished; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # TWO rows into a COMPRESSED column whose check asks nothing but +--echo # whether it holds a document. A compressed store does not put +--echo # down the characters it was handed: it puts down an image of them +--echo # behind a header of its own. Nothing the value answered can be +--echo # carried by bytes like that, so the check reads for itself - one +--echo # reading per row, and both stay. +--echo # +--echo # The two values are built out of scalars and cost no reading of +--echo # their own. They differ in length on purpose: the short one is +--echo # put down as it stands behind the header and the long one is +--echo # packed, and neither is the text that was passed. +--echo # +CREATE TABLE t1 (j VARCHAR(1024) COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # The same two rows into a LONGTEXT COMPRESSED column, which is a +--echo # different class of field storing the same way. Both compressed +--echo # classes answer the same question about themselves, so neither +--echo # has to be named for either to be refused. +--echo # +CREATE TABLE t1 (j LONGTEXT COMPRESSED CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # THE CONTROL: the same two values into the same two column types +--echo # with the compression taken off. These put down what they were +--echo # handed, so the check is left unrun and nothing is counted - which +--echo # is what makes the counts above a fact about compressing rather +--echo # than about the values, the widths or the types. +--echo # +CREATE TABLE t1 (j VARCHAR(1024) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (j LONGTEXT CHECK (JSON_VALID(j))); +FLUSH STATUS; +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +INSERT INTO t2 VALUES (JSON_OBJECT('a', 1)), + (JSON_OBJECT('a', REPEAT('x', 200))); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # TWO rows whose checked column is written by a trigger and by +--echo # nothing else. An assignment in a trigger is a store like any +--echo # other and attests to what it put down, so the check is left +--echo # unrun. The value is built out of scalars and reads nothing of +--echo # its own, so nothing at all is counted. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # The same trigger over a table where no column carries a check. +--echo # Nothing here can ever be asked whether it holds a document, so +--echo # the assignment does not stop to attest to what it stored - the +--echo # restraint the other two store sites already showed. A debug +--echo # build stops the server where a mark is set that nothing can +--echo # read, which is what says the restraint is being shown. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_OBJECT('seen', NEW.id); +FLUSH STATUS; +INSERT INTO t1 (id) VALUES (1), (2); +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t1; + +--echo # +--echo # A row cut in two, where each piece is written over the row image +--echo # the piece before it left behind. +--echo # +--echo # An update over part of a period writes back, as rows of their +--echo # own, the pieces the update did not cover. Each piece is built by +--echo # putting the row back as it stood and writing one end of the +--echo # period over it, and the check runs on what that came to - so the +--echo # answer a trigger gave for the piece before is an answer about +--echo # bytes that are no longer there. +--echo # +--echo # THREE readings: one for the update itself, whose SET does not +--echo # touch the checked column, and one for each of the two pieces. A +--echo # piece written over an answer that was left standing would show up +--echo # as a count of two. +--echo # +CREATE TABLE t1 (id INT, s DATE, e DATE, j VARCHAR(64) CHECK (JSON_VALID(j)), + PERIOD FOR p(s, e)); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_OBJECT('seen', NEW.id); +INSERT INTO t1 (id, s, e) VALUES (1, '2000-01-01', '2010-01-01'); +FLUSH STATUS; +UPDATE t1 FOR PORTION OF p FROM '2002-01-01' TO '2003-01-01' SET id = 2; +SHOW STATUS LIKE 'Json_scans'; +SELECT id, s, e, j FROM t1 ORDER BY s; +DROP TABLE t1; + +--disable_prepare_warnings +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_check_skip.result b/mysql-test/main/func_json_check_skip.result new file mode 100644 index 0000000000000..05c90d9c2c721 --- /dev/null +++ b/mysql-test/main/func_json_check_skip.result @@ -0,0 +1,339 @@ +# +# What a JSON column's check constraint keeps out. +# +# A column declared JSON carries a check constraint that reads every +# value stored into it and refuses the ones that are not documents. +# Where the value came from something that has already attested +# it, that reading finds out nothing that was not already known, and +# can be left undone. +# +# Everything below is a case where it CANNOT be left undone, and the +# answers here are what the column does about each of them. Nothing +# in this file is supposed to change: it says what is refused, and a +# value that stops being refused is a value that got in. +# +SET NAMES utf8mb4; +# +# 1. A document that came from a JSON function goes in and stays put. +# +CREATE TABLE t1 (id INT, j JSON); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1, 'b', 2)), +(2, JSON_ARRAY(1, 2, 3)); +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "b": 2} +2 [1, 2, 3] +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; +id still_a_document +1 1 +2 1 +# the same through every other way a row is written +INSERT INTO t1 SELECT id + 10, JSON_INSERT(j, '$.c', 3) FROM t1 ORDER BY id; +UPDATE t1 SET j = JSON_SET(j, '$.d', 4) WHERE id = 1; +REPLACE INTO t1 VALUES (2, JSON_ARRAY(9, 8)); +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; +id still_a_document +1 1 +2 1 +2 1 +11 1 +12 1 +DROP TABLE t1; +# +# 2. Text that is not a document is refused, whoever wrote it. +# +CREATE TABLE t1 (id INT, j JSON); +INSERT INTO t1 VALUES (1, 'not a document'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, '{"a":1'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, CONCAT('{"a":', '1')); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +UPDATE t1 SET j = 'not a document'; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1} +DROP TABLE t1; +# +# 3. A document that does not fit is cut, and what is left of it is +# read like anything else. The value that arrives came from a JSON +# function and was a document when it set out; what the column holds +# is not, and that is what the check has to see. +# +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2, 3, 4567)); +ERROR 23000: CONSTRAINT `t1.v` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# short enough to arrive whole +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT v FROM t1; +v +[1, 2] +DROP TABLE t1; +# a cut that happens to leave a document behind is accepted, as it +# always has been - the text after it is what went missing +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES ('[1,2,3,45],6789'); +Warnings: +Warning 1265 Data truncated for column 'v' at row 1 +SELECT v FROM t1; +v +[1,2,3,45] +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; +# +# 4. A character set that cannot encode a document does not get one. +# The brackets are not in swe7 at all, so they arrive as question +# marks and what is stored is not a document. +# +SET @@sql_mode = ''; +CREATE TABLE t1 (v VARCHAR(64) CHARACTER SET swe7 CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +ERROR 23000: CONSTRAINT `t1.v` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; +# +# 5. A row that is written twice is read twice. +# +# ON DUPLICATE KEY UPDATE writes the new row, reads it, then puts +# the old row's columns back and writes the update over the top. +# What the first writing attested is gone by then, and the +# second writing has to attest to itself. +# +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) +ON DUPLICATE KEY UPDATE j = 'not a document'; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1} +# and the update that IS a document goes through +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"c": 3} +# the insert half refused, with a second row present to prove the +# statement reached it +INSERT INTO t1 VALUES (1, JSON_OBJECT('d', 4)), (2, 'not a document') +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('e', 5); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"e": 5} +DROP TABLE t1; +# +# 6. A check that reads a DIFFERENT column is not this column's +# check, however much it looks like one. +# +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +# a is given a document, b is not; the check reads b and refuses +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), 'not a document'); +ERROR 23000: CONSTRAINT `t1.a` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# b a document, a not - accepted, the check having nothing to say +# about a +INSERT INTO t1 VALUES ('not a document', JSON_OBJECT('b', 2)); +SELECT a, b FROM t1; +a b +not a document {"b": 2} +DROP TABLE t1; +# +# 7. A check that asks anything besides whether the value is a +# document keeps asking it. +# +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 12)); +Warnings: +Warning 4271 CHECK constraint of column 'j' asks more than JSON_VALID() of it; the column is not a JSON column +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (JSON_OBJECT('aaaaaaaaaa', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT j FROM t1; +j +{"a": 1} +DROP TABLE t1; +CREATE TABLE t1 (id INT, j VARCHAR(64), +CONSTRAINT ck CHECK (id > 0 AND JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (0, JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `ck` failed for `test`.`t1` +INSERT INTO t1 VALUES (1, 'not a document'); +ERROR 23000: CONSTRAINT `ck` failed for `test`.`t1` +SELECT id, j FROM t1; +id j +1 {"a": 1} +DROP TABLE t1; +# a table-level check that asks only about the document is still a +# table-level check +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (2, 'not a document'); +ERROR 23000: CONSTRAINT `ck` failed for `test`.`t1` +SELECT id, j FROM t1; +id j +1 {"a": 1} +DROP TABLE t1; +# +# 8. A trigger writing the column is a writer like any other. +# +CREATE TABLE t1 (id INT, j JSON); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "seen": 1} +DROP TRIGGER tr1; +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +SET NEW.j = 'not a document'; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "seen": 1} +DROP TRIGGER tr1; +# written twice in one trigger, the last writing being the one that +# counts +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +BEGIN +SET NEW.j = JSON_OBJECT('first', 1); +SET NEW.j = 'not a document'; +END| +INSERT INTO t1 VALUES (3, JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "seen": 1} +DROP TRIGGER tr1; +DROP TABLE t1; +# +# 9. Switching the checks off leaves bytes behind that the column +# would not have taken, and switching them back on does not go +# looking for them. What it does do is refuse the next bad value. +# +CREATE TABLE t1 (id INT, j JSON); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 'not a document'); +SET @@check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +id j reads_as_a_document +1 not a document 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES (3, 'also not a document'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +id reads_as_a_document +1 0 +2 1 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# a copying ALTER reads every row again and finds the one that was +# let through +ALTER TABLE t1 FORCE, ALGORITHM=COPY; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +# and ignoring drops it rather than keeping it +ALTER IGNORE TABLE t1 FORCE, ALGORITHM=COPY; +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +id reads_as_a_document +2 1 +DROP TABLE t1; +# +# 10. A versioned table writes a history row that is not read, and a +# current row that is. +# +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))) +WITH SYSTEM VERSIONING; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +UPDATE t1 SET j = JSON_OBJECT('b', 2) WHERE id = 1; +UPDATE t1 SET j = 'not a document' WHERE id = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"b": 2} +SELECT j, JSON_VALID(j) AS reads_as_a_document +FROM t1 FOR SYSTEM_TIME ALL ORDER BY j; +j reads_as_a_document +{"a": 1} 1 +{"b": 2} 1 +DROP TABLE t1; +# +# 11. Every other way of writing rows. +# +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); +# INSERT ... SELECT stops at the row that is not a document, the +# rows before it having already been written +INSERT INTO t1 SELECT id, txt FROM t2 ORDER BY id; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +DELETE FROM t1; +INSERT INTO t1 SELECT id, JSON_COMPACT(txt) FROM t2 WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +# a plain UPDATE +UPDATE t1 SET j = 'not a document' WHERE id = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1 SET j = JSON_SET(j, '$.b', 2) WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a": 1, "b": 2} +# an update across two tables, both of them present in t1 this time +INSERT INTO t1 VALUES (2, JSON_OBJECT('x', 9)); +UPDATE t1, t2 SET t1.j = t2.txt WHERE t1.id = t2.id AND t2.id = 2; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1, t2 SET t1.j = 'not a document' WHERE t1.id = t2.id; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1, t2 SET t1.j = JSON_COMPACT(t2.txt) WHERE t1.id = t2.id AND t2.id = 1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +2 {"x": 9} +DROP TABLE t1, t2; +# +# 12. Rows read out of a file are text and are read like text. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); +SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt'; +Warnings: +Warning 1287 ' INTO FROM...' instead +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt' INTO TABLE t1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +1 +# the same file with only the document in it +DELETE FROM t2 WHERE id = 2; +SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt'; +Warnings: +Warning 1287 ' INTO FROM...' instead +LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt' INTO TABLE t1; +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1} +1 {"a":1} +DROP TABLE t1, t2; diff --git a/mysql-test/main/func_json_check_skip.test b/mysql-test/main/func_json_check_skip.test new file mode 100644 index 0000000000000..7e03cab350433 --- /dev/null +++ b/mysql-test/main/func_json_check_skip.test @@ -0,0 +1,308 @@ +--echo # +--echo # What a JSON column's check constraint keeps out. +--echo # +--echo # A column declared JSON carries a check constraint that reads every +--echo # value stored into it and refuses the ones that are not documents. +--echo # Where the value came from something that has already attested +--echo # it, that reading finds out nothing that was not already known, and +--echo # can be left undone. +--echo # +--echo # Everything below is a case where it CANNOT be left undone, and the +--echo # answers here are what the column does about each of them. Nothing +--echo # in this file is supposed to change: it says what is refused, and a +--echo # value that stops being refused is a value that got in. +--echo # + +SET NAMES utf8mb4; + +# Some of the statements below are looked at once when they are prepared and +# warn there rather than when they run, so the warnings only reach the client +# when prepare warnings are asked for. +--enable_prepare_warnings + +--echo # +--echo # 1. A document that came from a JSON function goes in and stays put. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1, 'b', 2)), + (2, JSON_ARRAY(1, 2, 3)); +SELECT id, j FROM t1 ORDER BY id; +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; + +--echo # the same through every other way a row is written +INSERT INTO t1 SELECT id + 10, JSON_INSERT(j, '$.c', 3) FROM t1 ORDER BY id; +UPDATE t1 SET j = JSON_SET(j, '$.d', 4) WHERE id = 1; +REPLACE INTO t1 VALUES (2, JSON_ARRAY(9, 8)); +SELECT id, JSON_VALID(j) AS still_a_document FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 2. Text that is not a document is refused, whoever wrote it. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, 'not a document'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, '{"a":1'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, CONCAT('{"a":', '1')); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = 'not a document'; +SELECT id, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 3. A document that does not fit is cut, and what is left of it is +--echo # read like anything else. The value that arrives came from a JSON +--echo # function and was a document when it set out; what the column holds +--echo # is not, and that is what the check has to see. +--echo # + +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; + +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2, 3, 4567)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # short enough to arrive whole +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT v FROM t1; +DROP TABLE t1; + +--echo # a cut that happens to leave a document behind is accepted, as it +--echo # always has been - the text after it is what went missing +CREATE TABLE t1 (v VARCHAR(10) CHECK (JSON_VALID(v))); +INSERT INTO t1 VALUES ('[1,2,3,45],6789'); +SELECT v FROM t1; +DROP TABLE t1; + +SET @@sql_mode = @sql_mode_saved; + +--echo # +--echo # 4. A character set that cannot encode a document does not get one. +--echo # The brackets are not in swe7 at all, so they arrive as question +--echo # marks and what is stored is not a document. +--echo # + +SET @@sql_mode = ''; +CREATE TABLE t1 (v VARCHAR(64) CHARACTER SET swe7 CHECK (JSON_VALID(v))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; + +--echo # +--echo # 5. A row that is written twice is read twice. +--echo # +--echo # ON DUPLICATE KEY UPDATE writes the new row, reads it, then puts +--echo # the old row's columns back and writes the update over the top. +--echo # What the first writing attested is gone by then, and the +--echo # second writing has to attest to itself. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) + ON DUPLICATE KEY UPDATE j = 'not a document'; +SELECT id, j FROM t1 ORDER BY id; + +--echo # and the update that IS a document goes through +INSERT INTO t1 VALUES (1, JSON_OBJECT('b', 2)) + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('c', 3); +SELECT id, j FROM t1 ORDER BY id; + +--echo # the insert half refused, with a second row present to prove the +--echo # statement reached it +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, JSON_OBJECT('d', 4)), (2, 'not a document') + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('e', 5); +SELECT id, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 6. A check that reads a DIFFERENT column is not this column's +--echo # check, however much it looks like one. +--echo # + +CREATE TABLE t1 (a VARCHAR(64) CHECK (JSON_VALID(b)), b VARCHAR(64)); +--echo # a is given a document, b is not; the check reads b and refuses +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), 'not a document'); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # b a document, a not - accepted, the check having nothing to say +--echo # about a +INSERT INTO t1 VALUES ('not a document', JSON_OBJECT('b', 2)); +SELECT a, b FROM t1; +DROP TABLE t1; + +--echo # +--echo # 7. A check that asks anything besides whether the value is a +--echo # document keeps asking it. +--echo # + +CREATE TABLE t1 (j VARCHAR(64) CHECK (JSON_VALID(j) AND LENGTH(j) < 12)); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT('aaaaaaaaaa', 1)); +SELECT j FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 (id INT, j VARCHAR(64), + CONSTRAINT ck CHECK (id > 0 AND JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (0, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, 'not a document'); +SELECT id, j FROM t1; +DROP TABLE t1; + +--echo # a table-level check that asks only about the document is still a +--echo # table-level check +CREATE TABLE t1 (id INT, j VARCHAR(64), CONSTRAINT ck CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (2, 'not a document'); +SELECT id, j FROM t1; +DROP TABLE t1; + +--echo # +--echo # 8. A trigger writing the column is a writer like any other. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.seen', 1); +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +DROP TRIGGER tr1; + +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW + SET NEW.j = 'not a document'; +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +DROP TRIGGER tr1; + +--echo # written twice in one trigger, the last writing being the one that +--echo # counts +DELIMITER |; +CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW +BEGIN + SET NEW.j = JSON_OBJECT('first', 1); + SET NEW.j = 'not a document'; +END| +DELIMITER ;| +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, JSON_OBJECT('a', 1)); +SELECT id, j FROM t1 ORDER BY id; +DROP TRIGGER tr1; +DROP TABLE t1; + +--echo # +--echo # 9. Switching the checks off leaves bytes behind that the column +--echo # would not have taken, and switching them back on does not go +--echo # looking for them. What it does do is refuse the next bad value. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 'not a document'); +SET @@check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +INSERT INTO t1 VALUES (2, JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, 'also not a document'); +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; + +--echo # a copying ALTER reads every row again and finds the one that was +--echo # let through +--error ER_CONSTRAINT_FAILED +ALTER TABLE t1 FORCE, ALGORITHM=COPY; +--echo # and ignoring drops it rather than keeping it +ALTER IGNORE TABLE t1 FORCE, ALGORITHM=COPY; +SELECT id, JSON_VALID(j) AS reads_as_a_document FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 10. A versioned table writes a history row that is not read, and a +--echo # current row that is. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))) + WITH SYSTEM VERSIONING; +INSERT INTO t1 VALUES (1, JSON_OBJECT('a', 1)); +UPDATE t1 SET j = JSON_OBJECT('b', 2) WHERE id = 1; +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = 'not a document' WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +SELECT j, JSON_VALID(j) AS reads_as_a_document + FROM t1 FOR SYSTEM_TIME ALL ORDER BY j; +DROP TABLE t1; + +--echo # +--echo # 11. Every other way of writing rows. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT PRIMARY KEY, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); + +--echo # INSERT ... SELECT stops at the row that is not a document, the +--echo # rows before it having already been written +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 SELECT id, txt FROM t2 ORDER BY id; +SELECT id, j FROM t1 ORDER BY id; +DELETE FROM t1; +INSERT INTO t1 SELECT id, JSON_COMPACT(txt) FROM t2 WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; + +--echo # a plain UPDATE +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = 'not a document' WHERE id = 1; +UPDATE t1 SET j = JSON_SET(j, '$.b', 2) WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; + +--echo # an update across two tables, both of them present in t1 this time +INSERT INTO t1 VALUES (2, JSON_OBJECT('x', 9)); +--error ER_CONSTRAINT_FAILED +UPDATE t1, t2 SET t1.j = t2.txt WHERE t1.id = t2.id AND t2.id = 2; +--error ER_CONSTRAINT_FAILED +UPDATE t1, t2 SET t1.j = 'not a document' WHERE t1.id = t2.id; +UPDATE t1, t2 SET t1.j = JSON_COMPACT(t2.txt) WHERE t1.id = t2.id AND t2.id = 1; +SELECT id, j FROM t1 ORDER BY id; +DROP TABLE t1, t2; + +--echo # +--echo # 12. Rows read out of a file are text and are read like text. +--echo # + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +CREATE TABLE t2 (id INT, txt VARCHAR(64)); +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, 'not a document'); +--let $DATA = $MYSQLTEST_VARDIR/tmp/func_json_check_skip.txt +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE '$DATA' +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--error ER_CONSTRAINT_FAILED +--eval LOAD DATA INFILE '$DATA' INTO TABLE t1 +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--remove_file $DATA + +--echo # the same file with only the document in it +DELETE FROM t2 WHERE id = 2; +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval SELECT id, txt FROM t2 ORDER BY id INTO OUTFILE '$DATA' +--replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--eval LOAD DATA INFILE '$DATA' INTO TABLE t1 +SELECT id, j FROM t1 ORDER BY id; +--remove_file $DATA +DROP TABLE t1, t2; + +--disable_prepare_warnings diff --git a/mysql-test/main/func_json_check_store.result b/mysql-test/main/func_json_check_store.result new file mode 100644 index 0000000000000..7bac511755bf7 --- /dev/null +++ b/mysql-test/main/func_json_check_store.result @@ -0,0 +1,345 @@ +# +# What a check constraint has to read for itself, whatever wrote the +# value. +# +# A value that arrived from something that attested to it is only +# the value the check will read if TWO things hold: the store put +# those same characters in the row and nothing has written over them +# since. Every case below is one where one of the two fails, and +# the check has to read the row for itself. +# +# These answers are what a released server gives. A value getting +# in here that reads back as not a document is a value that got past +# the constraint declaring it must be one. +# +SET NAMES utf8mb4; +# +# 1. Stores that do not put down the characters they were given. +# +# BINARY pads with NUL, and the padding is part of what is stored +CREATE TABLE t1 (b BINARY(20) CHECK (JSON_VALID(b))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +ERROR 23000: CONSTRAINT `t1.b` failed for `test`.`t1` +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +ERROR 23000: CONSTRAINT `t1.b` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# CHAR pads too, with spaces, which are not read back +CREATE TABLE t1 (c CHAR(20) CHECK (JSON_VALID(c))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +INSERT INTO t1 VALUES ('not a document'); +ERROR 23000: CONSTRAINT `t1.c` failed for `test`.`t1` +SELECT c, JSON_VALID(c) AS reads_as_a_document FROM t1; +c reads_as_a_document +{"a": 1} 1 +DROP TABLE t1; +# ENUM keeps the member it matched, not the text it was given +CREATE TABLE t1 (e ENUM('True','banana') CHECK (JSON_VALID(e))); +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +ERROR 23000: CONSTRAINT `t1.e` failed for `test`.`t1` +INSERT INTO t1 VALUES (JSON_COMPACT('2')); +ERROR 23000: CONSTRAINT `t1.e` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# SET the same +CREATE TABLE t1 (s SET('True','1') CHECK (JSON_VALID(s))); +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +ERROR 23000: CONSTRAINT `t1.s` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# a column that DOES put down what it was given takes the document +CREATE TABLE t1 (v VARCHAR(64) CHECK (JSON_VALID(v)), +t LONGTEXT CHECK (JSON_VALID(t))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_ARRAY(1, 2)); +SELECT v, t FROM t1; +v t +{"a": 1} [1, 2] +DROP TABLE t1; +# +# 2. A stored generated column is written twice: once from what was +# given, and again from its own expression, which is what stays. +# +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; +CREATE TABLE t1 (a VARCHAR(20), j LONGTEXT AS (a) STORED CHECK (JSON_VALID(j))); +INSERT INTO t1 (a, j) VALUES ('not a document', JSON_OBJECT('k', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# and the value the expression does produce is taken +INSERT INTO t1 (a, j) VALUES ('{"k":1}', JSON_OBJECT('k', 1)); +Warnings: +Warning 1906 The value specified for generated column 'j' in table 't1' has been ignored +SELECT a, j FROM t1; +a j +{"k":1} {"k":1} +# the same on update +UPDATE t1 SET a = 'not a document'; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT a, j FROM t1; +a j +{"k":1} {"k":1} +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; +# +# 3. A row whose writing ends before the check is reached leaves +# nothing behind for the NEXT row to be read with. +# +# an update where one row comes out exactly as it went in +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('k', 1)); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (2, 0, 'not a document'); +SET @@check_constraint_checks = ON; +CREATE TRIGGER t1t BEFORE UPDATE ON t1 FOR EACH ROW +BEGIN +IF OLD.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +UPDATE t1 SET b = b + (id = 2); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, b, j FROM t1 ORDER BY id; +id b j +1 1 {"k": 1} +2 0 not a document +DROP TABLE t1; +# an insert through a view whose own check refuses the row first +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, +j VARCHAR(64) DEFAULT 'not a document' + CHECK (JSON_VALID(j))); +CREATE VIEW v1 AS SELECT id, b, j FROM t1 WHERE b > 0 WITH CHECK OPTION; +CREATE TRIGGER t1t BEFORE INSERT ON t1 FOR EACH ROW +BEGIN +IF NEW.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +INSERT IGNORE INTO v1 (id, b) VALUES (1, 0), (2, 1); +Warnings: +Warning 1369 CHECK OPTION failed `test`.`v1` +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, b, j FROM t1 ORDER BY id; +id b j +DROP VIEW v1; +DROP TABLE t1; +# +# 4. A row written twice over, where the SECOND writing does not +# touch the column at all. +# +# The first writing puts a document there and attests to it; the +# old row's bytes are then put back, and the update clause leaves +# the column alone. What the check reads is the old bytes, which +# nothing has attested. +# +CREATE TABLE t1 (id INT PRIMARY KEY, a INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 0, 'not a document'); +SET @@check_constraint_checks = ON; +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('x', 1)) +ON DUPLICATE KEY UPDATE a = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, a, j FROM t1 ORDER BY id; +id a j +1 0 not a document +# and where the update clause DOES write the column, it is read the +# ordinary way +INSERT INTO t1 VALUES (1, 2, JSON_OBJECT('x', 1)) +ON DUPLICATE KEY UPDATE j = JSON_OBJECT('y', 2); +SELECT id, a, j FROM t1 ORDER BY id; +id a j +1 0 {"y": 2} +DROP TABLE t1; +# +# 5. Stores that put down every byte they were given and call them +# by another name. +# +# A binary column takes the bytes of a document written in some +# other character set exactly as they stand, and a column in some +# other set takes the bytes of a binary one the same way. Neither +# writes the characters again, so what the row holds is not the +# characters that were attested, and the check has to read it. +# +# a document written in a wide set, into a binary column +CREATE TABLE t1 (j BLOB CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (CONVERT(JSON_OBJECT('a', 1) USING ucs2)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (CONVERT(JSON_ARRAY(1, 2) USING utf16)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +# the same documents in a set the column can take go in +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +j +{"a": 1} +[1, 2] +DROP TABLE t1; +# a binary document, into a column written in a wide set +CREATE TABLE t1 (j TEXT CHARACTER SET ucs2 CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (JSON_ARRAY(_binary'ab')); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (JSON_OBJECT(_binary'k', 1)); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) AS rows_that_got_in FROM t1; +rows_that_got_in +0 +DROP TABLE t1; +# a store that does write the characters again, in the column's +# own set, keeps the document +CREATE TABLE t1 (j TEXT CHARACTER SET latin1 CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +j +{"a": 1} +[1, 2] +DROP TABLE t1; +# +# 6. A row split in two by an update over part of its period. Such +# an update writes the pieces of the old row that the period does +# not cover as new rows, and it writes them by putting the saved +# copy of the row back into the buffer the checks read. Anything +# said about what stood there before is about the value the update +# wrote, not the one just restored, so it goes with the row it was +# about. +# +# A trigger is what makes the difference visible: it writes the +# document column while the piece is being made, so the check has +# something to catch. +# +CREATE TABLE t1 (id INT, +j VARCHAR(64) CHECK (JSON_VALID(j)), +s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +INSERT INTO t1 (id, j, s, e) +VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), +(2, '[1,2]', '1999-01-01', '2018-12-12'); +UPDATE t1 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t1; +id j ty s e +1 {"a":1} OBJECT 1999-01-01 2000-01-01 +1 {"a":1} OBJECT 2018-01-01 2018-12-12 +2 [1,2] ARRAY 1999-01-01 2000-01-01 +2 [1,2] ARRAY 2018-01-01 2018-12-12 +6 {"a":1} OBJECT 2000-01-01 2018-01-01 +7 [1,2] ARRAY 2000-01-01 2018-01-01 +# the same split with the pieces given documents of their own +CREATE TABLE t2 (id INT, +j VARCHAR(64) CHECK (JSON_VALID(j)), +s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t2_bi BEFORE INSERT ON t2 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t2 (id, j, s, e) +VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), +(2, '{"b":2}', '1999-01-01', '2018-12-12'); +UPDATE t2 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t2; +id j ty s e +1 {"a": 1, "piece": 1} OBJECT 1999-01-01 2000-01-01 +1 {"a": 1, "piece": 1} OBJECT 2018-01-01 2018-12-12 +2 {"b": 2, "piece": 2} OBJECT 1999-01-01 2000-01-01 +2 {"b": 2, "piece": 2} OBJECT 2018-01-01 2018-12-12 +6 {"a": 1, "piece": 1} OBJECT 2000-01-01 2018-01-01 +7 {"b": 2, "piece": 2} OBJECT 2000-01-01 2018-01-01 +# the row the update itself writes is checked as any other row is, +# and it is written after the pieces have been, so what the pieces +# left behind must not be believed of it: a portion set to +# something that is not a document is refused +CREATE TABLE t3 (id INT, +j VARCHAR(64) CHECK (JSON_VALID(j)), +s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t3_bi BEFORE INSERT ON t3 FOR EACH ROW +SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t3 (id, j, s, e) +VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), +(2, '{"b":2}', '1999-01-01', '2018-12-12'); +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= 'not a document'; +ERROR 23000: CONSTRAINT `t3.j` failed for `test`.`t3` +SELECT id, j, s, e FROM t3; +id j s e +1 {"a": 1, "piece": 1} 1999-01-01 2018-12-12 +2 {"b": 2, "piece": 2} 1999-01-01 2018-12-12 +# and one set to a document goes in +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= JSON_OBJECT('portion', id); +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t3; +id j ty s e +1 {"a": 1, "piece": 1} OBJECT 1999-01-01 2000-01-01 +1 {"a": 1, "piece": 1} OBJECT 2018-01-01 2018-12-12 +1 {"portion": 1} OBJECT 2000-01-01 2018-01-01 +2 {"b": 2, "piece": 2} OBJECT 1999-01-01 2000-01-01 +2 {"b": 2, "piece": 2} OBJECT 2018-01-01 2018-12-12 +2 {"portion": 2} OBJECT 2000-01-01 2018-01-01 +DROP TABLE t3, t2, t1; +# +# 7. A value that came out SQL NULL, put into a column that cannot +# hold one. +# +# A column of a temporary table is attested to once, when the +# table is built out of something that attests to every value it +# will make. That answer is about the column and stands over +# every row of it, a row whose value came out NULL among them. +# What such a row puts in the destination is not the NULL but +# whatever the destination takes in place of one, and nothing has +# attested that. +# +CREATE TABLE src (d LONGTEXT) CHARSET utf8mb4; +INSERT INTO src VALUES ('{"a":1}'), ('{"b":2}'); +# grouping builds the temporary table, and the path finds nothing +CREATE TABLE t1 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +SET @save_sql_mode= @@sql_mode; +SET SESSION sql_mode= ''; +INSERT INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') FROM src GROUP BY d; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SET SESSION sql_mode= @save_sql_mode; +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +# the same insert told to carry on past what it cannot store +INSERT IGNORE INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') +FROM src GROUP BY d; +Warnings: +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT COUNT(*) FROM t1; +COUNT(*) +0 +# a union builds one too, and the rows where the path does find a +# value go in while the ones where it does not are refused +CREATE TABLE t2 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT IGNORE INTO t2 (j) +SELECT JSON_EXTRACT(d, '$.nosuch') FROM src UNION ALL +SELECT JSON_EXTRACT(d, '$.a') FROM src; +Warnings: +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t2.j` failed for `test`.`t2` +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t2.j` failed for `test`.`t2` +Warning 1048 Column 'j' cannot be null +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `t2.j` failed for `test`.`t2` +SELECT j, JSON_TYPE(j) AS ty FROM t2; +j ty +1 INTEGER +# a column that CAN hold a NULL holds the NULL itself, which is +# not a document and is not asked to be one +CREATE TABLE t3 (j LONGTEXT CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t3 (j) SELECT JSON_EXTRACT(d, '$.a') FROM src GROUP BY d; +SELECT j, j IS NULL AS was_null FROM t3; +j was_null +1 0 +NULL 1 +DROP TABLE t3, t2, t1, src; diff --git a/mysql-test/main/func_json_check_store.test b/mysql-test/main/func_json_check_store.test new file mode 100644 index 0000000000000..7ca4a500faccb --- /dev/null +++ b/mysql-test/main/func_json_check_store.test @@ -0,0 +1,297 @@ +--echo # +--echo # What a check constraint has to read for itself, whatever wrote the +--echo # value. +--echo # +--echo # A value that arrived from something that attested to it is only +--echo # the value the check will read if TWO things hold: the store put +--echo # those same characters in the row and nothing has written over them +--echo # since. Every case below is one where one of the two fails, and +--echo # the check has to read the row for itself. +--echo # +--echo # These answers are what a released server gives. A value getting +--echo # in here that reads back as not a document is a value that got past +--echo # the constraint declaring it must be one. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Stores that do not put down the characters they were given. +--echo # + +--echo # BINARY pads with NUL, and the padding is part of what is stored +CREATE TABLE t1 (b BINARY(20) CHECK (JSON_VALID(b))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(1, 2)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # CHAR pads too, with spaces, which are not read back +CREATE TABLE t1 (c CHAR(20) CHECK (JSON_VALID(c))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES ('not a document'); +SELECT c, JSON_VALID(c) AS reads_as_a_document FROM t1; +DROP TABLE t1; + +--echo # ENUM keeps the member it matched, not the text it was given +CREATE TABLE t1 (e ENUM('True','banana') CHECK (JSON_VALID(e))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_COMPACT('2')); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # SET the same +CREATE TABLE t1 (s SET('True','1') CHECK (JSON_VALID(s))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_COMPACT('true')); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # a column that DOES put down what it was given takes the document +CREATE TABLE t1 (v VARCHAR(64) CHECK (JSON_VALID(v)), + t LONGTEXT CHECK (JSON_VALID(t))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1), JSON_ARRAY(1, 2)); +SELECT v, t FROM t1; +DROP TABLE t1; + +--echo # +--echo # 2. A stored generated column is written twice: once from what was +--echo # given, and again from its own expression, which is what stays. +--echo # + +SET @sql_mode_saved = @@sql_mode; +SET @@sql_mode = ''; +CREATE TABLE t1 (a VARCHAR(20), j LONGTEXT AS (a) STORED CHECK (JSON_VALID(j))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 (a, j) VALUES ('not a document', JSON_OBJECT('k', 1)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # and the value the expression does produce is taken +INSERT INTO t1 (a, j) VALUES ('{"k":1}', JSON_OBJECT('k', 1)); +SELECT a, j FROM t1; +--echo # the same on update +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET a = 'not a document'; +SELECT a, j FROM t1; +DROP TABLE t1; +SET @@sql_mode = @sql_mode_saved; + +--echo # +--echo # 3. A row whose writing ends before the check is reached leaves +--echo # nothing behind for the NEXT row to be read with. +--echo # + +--echo # an update where one row comes out exactly as it went in +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('k', 1)); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (2, 0, 'not a document'); +SET @@check_constraint_checks = ON; +DELIMITER |; +CREATE TRIGGER t1t BEFORE UPDATE ON t1 FOR EACH ROW +BEGIN + IF OLD.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +DELIMITER ;| +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET b = b + (id = 2); +SELECT id, b, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # an insert through a view whose own check refuses the row first +CREATE TABLE t1 (id INT PRIMARY KEY, b INT, + j VARCHAR(64) DEFAULT 'not a document' + CHECK (JSON_VALID(j))); +CREATE VIEW v1 AS SELECT id, b, j FROM t1 WHERE b > 0 WITH CHECK OPTION; +DELIMITER |; +CREATE TRIGGER t1t BEFORE INSERT ON t1 FOR EACH ROW +BEGIN + IF NEW.id = 1 THEN SET NEW.j = JSON_OBJECT('k', 1); END IF; +END| +DELIMITER ;| +INSERT IGNORE INTO v1 (id, b) VALUES (1, 0), (2, 1); +SELECT id, b, j FROM t1 ORDER BY id; +DROP VIEW v1; +DROP TABLE t1; + +--echo # +--echo # 4. A row written twice over, where the SECOND writing does not +--echo # touch the column at all. +--echo # +--echo # The first writing puts a document there and attests to it; the +--echo # old row's bytes are then put back, and the update clause leaves +--echo # the column alone. What the check reads is the old bytes, which +--echo # nothing has attested. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, a INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +SET @@check_constraint_checks = OFF; +INSERT INTO t1 VALUES (1, 0, 'not a document'); +SET @@check_constraint_checks = ON; +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (1, 1, JSON_OBJECT('x', 1)) + ON DUPLICATE KEY UPDATE a = 1; +SELECT id, a, j FROM t1 ORDER BY id; +--echo # and where the update clause DOES write the column, it is read the +--echo # ordinary way +INSERT INTO t1 VALUES (1, 2, JSON_OBJECT('x', 1)) + ON DUPLICATE KEY UPDATE j = JSON_OBJECT('y', 2); +SELECT id, a, j FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 5. Stores that put down every byte they were given and call them +--echo # by another name. +--echo # +--echo # A binary column takes the bytes of a document written in some +--echo # other character set exactly as they stand, and a column in some +--echo # other set takes the bytes of a binary one the same way. Neither +--echo # writes the characters again, so what the row holds is not the +--echo # characters that were attested, and the check has to read it. +--echo # + +--echo # a document written in a wide set, into a binary column +CREATE TABLE t1 (j BLOB CHECK (JSON_VALID(j))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (CONVERT(JSON_OBJECT('a', 1) USING ucs2)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (CONVERT(JSON_ARRAY(1, 2) USING utf16)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +--echo # the same documents in a set the column can take go in +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +DROP TABLE t1; + +--echo # a binary document, into a column written in a wide set +CREATE TABLE t1 (j TEXT CHARACTER SET ucs2 CHECK (JSON_VALID(j))); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_ARRAY(_binary'ab')); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (JSON_OBJECT(_binary'k', 1)); +SELECT COUNT(*) AS rows_that_got_in FROM t1; +DROP TABLE t1; + +--echo # a store that does write the characters again, in the column's +--echo # own set, keeps the document +CREATE TABLE t1 (j TEXT CHARACTER SET latin1 CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (JSON_OBJECT('a', 1)), (JSON_ARRAY(1, 2)); +SELECT j FROM t1; +DROP TABLE t1; + +--echo # +--echo # 6. A row split in two by an update over part of its period. Such +--echo # an update writes the pieces of the old row that the period does +--echo # not cover as new rows, and it writes them by putting the saved +--echo # copy of the row back into the buffer the checks read. Anything +--echo # said about what stood there before is about the value the update +--echo # wrote, not the one just restored, so it goes with the row it was +--echo # about. +--echo # +--echo # A trigger is what makes the difference visible: it writes the +--echo # document column while the piece is being made, so the check has +--echo # something to catch. +--echo # + +CREATE TABLE t1 (id INT, + j VARCHAR(64) CHECK (JSON_VALID(j)), + s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +INSERT INTO t1 (id, j, s, e) + VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), + (2, '[1,2]', '1999-01-01', '2018-12-12'); + +UPDATE t1 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +--sorted_result +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t1; + +--echo # the same split with the pieces given documents of their own +CREATE TABLE t2 (id INT, + j VARCHAR(64) CHECK (JSON_VALID(j)), + s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t2_bi BEFORE INSERT ON t2 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t2 (id, j, s, e) + VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), + (2, '{"b":2}', '1999-01-01', '2018-12-12'); +UPDATE t2 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET id= id + 5; +--sorted_result +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t2; + +--echo # the row the update itself writes is checked as any other row is, +--echo # and it is written after the pieces have been, so what the pieces +--echo # left behind must not be believed of it: a portion set to +--echo # something that is not a document is refused +CREATE TABLE t3 (id INT, + j VARCHAR(64) CHECK (JSON_VALID(j)), + s DATE, e DATE, PERIOD FOR apptime(s,e)) CHARSET utf8mb4; +CREATE TRIGGER t3_bi BEFORE INSERT ON t3 FOR EACH ROW + SET NEW.j = JSON_INSERT(NEW.j, '$.piece', NEW.id); +INSERT INTO t3 (id, j, s, e) + VALUES (1, '{"a":1}', '1999-01-01', '2018-12-12'), + (2, '{"b":2}', '1999-01-01', '2018-12-12'); +--error ER_CONSTRAINT_FAILED +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= 'not a document'; +--sorted_result +SELECT id, j, s, e FROM t3; + +--echo # and one set to a document goes in +UPDATE t3 FOR PORTION OF apptime FROM '2000-01-01' TO '2018-01-01' + SET j= JSON_OBJECT('portion', id); +--sorted_result +SELECT id, j, JSON_TYPE(j) AS ty, s, e FROM t3; + +DROP TABLE t3, t2, t1; + +--echo # +--echo # 7. A value that came out SQL NULL, put into a column that cannot +--echo # hold one. +--echo # +--echo # A column of a temporary table is attested to once, when the +--echo # table is built out of something that attests to every value it +--echo # will make. That answer is about the column and stands over +--echo # every row of it, a row whose value came out NULL among them. +--echo # What such a row puts in the destination is not the NULL but +--echo # whatever the destination takes in place of one, and nothing has +--echo # attested that. +--echo # + +CREATE TABLE src (d LONGTEXT) CHARSET utf8mb4; +INSERT INTO src VALUES ('{"a":1}'), ('{"b":2}'); + +--echo # grouping builds the temporary table, and the path finds nothing +CREATE TABLE t1 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +SET @save_sql_mode= @@sql_mode; +SET SESSION sql_mode= ''; +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') FROM src GROUP BY d; +SET SESSION sql_mode= @save_sql_mode; +SELECT COUNT(*) FROM t1; + +--echo # the same insert told to carry on past what it cannot store +INSERT IGNORE INTO t1 (j) SELECT JSON_EXTRACT(d, '$.nosuch') + FROM src GROUP BY d; +SELECT COUNT(*) FROM t1; + +--echo # a union builds one too, and the rows where the path does find a +--echo # value go in while the ones where it does not are refused +CREATE TABLE t2 (j LONGTEXT NOT NULL CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT IGNORE INTO t2 (j) + SELECT JSON_EXTRACT(d, '$.nosuch') FROM src UNION ALL + SELECT JSON_EXTRACT(d, '$.a') FROM src; +--sorted_result +SELECT j, JSON_TYPE(j) AS ty FROM t2; + +--echo # a column that CAN hold a NULL holds the NULL itself, which is +--echo # not a document and is not asked to be one +CREATE TABLE t3 (j LONGTEXT CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t3 (j) SELECT JSON_EXTRACT(d, '$.a') FROM src GROUP BY d; +--sorted_result +SELECT j, j IS NULL AS was_null FROM t3; + +DROP TABLE t3, t2, t1, src; diff --git a/mysql-test/main/func_json_columns.result b/mysql-test/main/func_json_columns.result new file mode 100644 index 0000000000000..07cd62dba7f5f --- /dev/null +++ b/mysql-test/main/func_json_columns.result @@ -0,0 +1,999 @@ +# +# Behavioral baseline: JSON values that come out of a table. +# +# A JSON column carries a check constraint that is supposed to keep +# invalid documents out, but the check can be switched off for the +# duration of a statement, after which the column holds bytes that no +# longer satisfy it. This test records what every JSON function does +# with such a column, what the check constraints look like, and how a +# user-created temporary table compares with a base table. +# +SET NAMES utf8mb4; +# +# 1. What declaring a column JSON actually creates. +# +CREATE TABLE t1 (id INT, j JSON); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT column_name, data_type, column_type +FROM information_schema.columns +WHERE table_schema = 'test' AND table_name = 't1' + ORDER BY ordinal_position; +column_name data_type column_type +id int int(11) +j longtext longtext +SELECT constraint_name, check_clause +FROM information_schema.check_constraints +WHERE constraint_schema = 'test' AND table_name = 't1'; +constraint_name check_clause +j json_valid(`j`) +# a column that is only LONGTEXT does not get the constraint +CREATE TABLE t2 (id INT, j LONGTEXT); +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT COUNT(*) AS constraints +FROM information_schema.check_constraints +WHERE constraint_schema = 'test' AND table_name = 't2'; +constraints +0 +# a constraint written out by hand, worded exactly like the automatic one +CREATE TABLE t3 (id INT, j LONGTEXT CHECK (json_valid(j))); +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# a constraint that mentions the column inside a larger expression +CREATE TABLE t4 (id INT, j LONGTEXT CHECK (json_valid(j) AND id > 0)); +Warnings: +Warning 4271 CHECK constraint of column 'j' asks more than JSON_VALID() of it; the column is not a JSON column +SHOW CREATE TABLE t4; +Table Create Table +t4 CREATE TABLE `t4` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL CHECK (json_valid(`j`) and `id` > 0) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# a table level constraint over two columns +CREATE TABLE t5 (id INT, j LONGTEXT, k LONGTEXT, +CONSTRAINT both_valid CHECK (json_valid(j) AND json_valid(k))); +SHOW CREATE TABLE t5; +Table Create Table +t5 CREATE TABLE `t5` ( + `id` int(11) DEFAULT NULL, + `j` longtext DEFAULT NULL, + `k` longtext DEFAULT NULL, + CONSTRAINT `both_valid` CHECK (json_valid(`j`) and json_valid(`k`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# +# 2. The constraint doing its job. +# +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'); +INSERT INTO t1 VALUES (2, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t1 VALUES (3, '{"a":1,'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +INSERT INTO t1 VALUES (3, 'not json at all'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +UPDATE t1 SET j = '{' WHERE id = 1; +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1,"b":[1,2]} +2 {"a": 1, "b": [1, 2]} +# the hand written constraint behaves the same way +INSERT INTO t3 VALUES (1, '{'); +ERROR 23000: CONSTRAINT `t3.j` failed for `test`.`t3` +INSERT INTO t3 VALUES (1, '{"a":1}'); +SELECT id, j FROM t3; +id j +1 {"a":1} +# the constraint shapes that are NOT a bare check of one column. Each +# of these still has to run in full, so what they accept and reject is +# recorded here rather than left to the CREATE statement alone. +# constraint on the column combined with a test of another column +INSERT INTO t4 VALUES (1, '{'); +ERROR 23000: CONSTRAINT `t4.j` failed for `test`.`t4` +INSERT INTO t4 VALUES (0, '{"a":1}'); +ERROR 23000: CONSTRAINT `t4.j` failed for `test`.`t4` +INSERT INTO t4 VALUES (1, '{"a":1}'); +SELECT id, j FROM t4; +id j +1 {"a":1} +# table level constraint over two columns +INSERT INTO t5 VALUES (1, '{', '{"a":1}'); +ERROR 23000: CONSTRAINT `both_valid` failed for `test`.`t5` +INSERT INTO t5 VALUES (1, '{"a":1}', '{'); +ERROR 23000: CONSTRAINT `both_valid` failed for `test`.`t5` +INSERT INTO t5 VALUES (1, '{"a":1}', '{"b":2}'); +SELECT id, j, k FROM t5; +id j k +1 {"a":1} {"b":2} +# an OR, where the other side can let an invalid document through +CREATE TABLE t5b (id INT, j LONGTEXT CHECK (json_valid(j) OR id > 0)); +Warnings: +Warning 4270 CHECK constraint of column 'j' can pass without JSON_VALID() holding; the column is not a JSON column +INSERT INTO t5b VALUES (1, '{'); +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +INSERT INTO t5b VALUES (0, '{'); +ERROR 23000: CONSTRAINT `t5b.j` failed for `test`.`t5b` +SELECT id, j, JSON_VALID(j) AS valid FROM t5b; +id j valid +1 { 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# an AND whose parts all have to hold, but where none of them is +# the call: the promise is inside an OR one level down, so the +# constraint can pass without it and the column stays ordinary +CREATE TABLE t5d (id INT, +j LONGTEXT CHECK (LENGTH(j) > 0 AND +(json_valid(j) OR id > 0))); +Warnings: +Warning 4270 CHECK constraint of column 'j' can pass without JSON_VALID() holding; the column is not a JSON column +INSERT INTO t5d VALUES (1, '{'); +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +INSERT INTO t5d VALUES (0, '{'); +ERROR 23000: CONSTRAINT `t5d.j` failed for `test`.`t5d` +INSERT INTO t5d VALUES (0, '{"a":1}'); +SELECT id, j, JSON_VALID(j) AS valid FROM t5d ORDER BY id, j; +id j valid +0 {"a":1} 1 +1 { 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# a constraint attached to one column that tests a different column +CREATE TABLE t5c (a LONGTEXT, b LONGTEXT CHECK (json_valid(a))); +Warnings: +Warning 4269 CHECK constraint of column 'b' calls JSON_VALID() on something other than 'b'; the column is not a JSON column +INSERT INTO t5c VALUES ('{"a":1}', '{'); +INSERT INTO t5c VALUES ('{', '{"a":1}'); +ERROR 23000: CONSTRAINT `t5c.b` failed for `test`.`t5c` +SELECT a, b FROM t5c; +a b +{"a":1} { +# writing any constraint of your own on a JSON column replaces the +# automatic one, so invalid documents go in with the checks left on +CREATE TABLE t5e (j JSON CHECK (LENGTH(j) > 0)); +SHOW CREATE TABLE t5e; +Table Create Table +t5e CREATE TABLE `t5e` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (octet_length(`j`) > 0) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO t5e VALUES ('{"a":1}'); +INSERT INTO t5e VALUES ('{'); +INSERT INTO t5e VALUES (''); +ERROR 23000: CONSTRAINT `t5e.j` failed for `test`.`t5e` +SELECT j, JSON_VALID(j) AS valid FROM t5e ORDER BY j; +j valid +{ 0 +{"a":1} 1 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS still_valid +FROM t5e ORDER BY j; +v still_valid +["{"] 1 +["{\"a\":1}"] 1 +DROP TABLE t5b, t5c, t5d, t5e; +# +# 3. Switching the check off, which is what makes a stored document +# unverifiable afterwards. +# +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,'); +INSERT INTO t1 VALUES (11, 'not json at all'); +INSERT INTO t1 VALUES (12, '{"a":1} trailing'); +INSERT INTO t1 VALUES (13, CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32))); +INSERT INTO t3 VALUES (10, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +# the rows are there and the constraint is back on +SELECT id, j FROM t1 ORDER BY id; +id j +1 {"a":1,"b":[1,2]} +2 {"a": 1, "b": [1, 2]} +10 {"a":1, +11 not json at all +12 {"a":1} trailing +13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT id, JSON_VALID(j) AS valid FROM t1 ORDER BY id; +id valid +1 1 +2 1 +10 0 +11 0 +12 0 +13 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +# a new statement is checked again, the stored rows are not +INSERT INTO t1 VALUES (14, '{'); +ERROR 23000: CONSTRAINT `t1.j` failed for `test`.`t1` +# reading the column back does not check anything +SELECT id, j FROM t1 WHERE id >= 10 ORDER BY id; +id j +10 {"a":1, +11 not json at all +12 {"a":1} trailing +13 [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT id, HEX(j) FROM t1 WHERE id = 10; +id HEX(j) +10 7B2261223A312C +# +# 4. Every function fed the column that no longer satisfies its check. +# +SELECT id, JSON_VALID(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 0 +11 0 +12 0 +13 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT id, JSON_TYPE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_type' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_type' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_type' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_type' at position 32 +SELECT id, JSON_DEPTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_depth' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_depth' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_depth' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +SELECT id, JSON_LENGTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_length' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_length' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_length' at position 32 +SELECT id, JSON_KEYS(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 ["a"] +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 1 +SELECT id, JSON_EXTRACT(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_extract' at position 32 +SELECT id, JSON_QUERY(j, '$') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 {"a":1} +13 NULL +SELECT id, JSON_VALUE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 1 +11 NULL +12 1 +13 NULL +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 16 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 32 +SELECT id, JSON_INSERT(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_insert' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 16 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 32 +SELECT id, JSON_REPLACE(j, '$.a', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_replace' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_replace' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_replace' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 32 +SELECT id, JSON_REMOVE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_remove' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 4 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_remove' at position 32 +SELECT id, JSON_ARRAY_APPEND(j, '$', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array_append' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 14 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +SELECT id, JSON_MERGE(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 {"a": 1, "z": 1} +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 32 +SELECT id, JSON_MERGE_PATCH(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 {"a": 1, "z": 1} +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_patch' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 1 +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_patch' at position 32 +SELECT id, JSON_COMPACT(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_compact' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_compact' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_compact' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_compact' at position 32 +SELECT id, JSON_LOOSE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_loose' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_loose' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_loose' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_loose' at position 32 +SELECT id, JSON_NORMALIZE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_normalize' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_normalize' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_normalize' at position 32 +SELECT id, JSON_EXISTS(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 1 +11 NULL +12 1 +13 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_exists' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_exists' at position 32 +SELECT id, JSON_CONTAINS(j, '1', '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 1 +11 NULL +12 1 +13 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_contains' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains' at position 32 +SELECT id, JSON_SEARCH(j, 'one', '1') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 "$.a" +11 NULL +12 "$.a" +13 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_search' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_search' at position 32 +SELECT id, JSON_EQUALS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_equals' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_equals' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_equals' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_equals' at position 32 +SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 0 +11 0 +12 1 +13 0 +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_overlaps' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +# +# 5. The column as an embedded argument. A JSON column is embedded as +# a document, so whatever it holds is copied into the result as is. +# +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; +id still_valid +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 32 +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS still_valid +FROM t1 WHERE id >= 10 ORDER BY id; +id still_valid +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 32 +SELECT id, JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 32 +SELECT id, JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +13 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 32 +# the same column read as plain text is quoted instead of embedded +SELECT id, JSON_ARRAY(CONCAT(j, '')) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 ["{\"a\":1,"] +11 ["not json at all"] +12 ["{\"a\":1} trailing"] +13 ["[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]"] +SELECT id, JSON_VALID(JSON_ARRAY(CONCAT(j, ''))) AS still_valid +FROM t1 WHERE id >= 10 ORDER BY id; +id still_valid +10 1 +11 1 +12 1 +13 1 +# a LONGTEXT column holding the same bytes is quoted, not embedded +SELECT id, JSON_ARRAY(j) AS v FROM t3 ORDER BY id; +id v +1 [{"a":1}] +10 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t3 ORDER BY id; +id still_valid +1 1 +10 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +# 5a. The same column reached through a scalar subquery, where the +# value is fetched once and held rather than read per row. +CREATE TABLE ts (id INT, j JSON); +INSERT INTO ts VALUES (1, '{"a": 1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ts VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 1)) AS v; +v +[{"a": 1}] +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 1))) AS ok; +ok +1 +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +v_poisoned +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 2))) AS ok_poisoned; +ok_poisoned +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 1)) AS v; +v +{"k": {"a": 1}} +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +v_poisoned +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 1)) AS v; +v +{"d": {"a": 1}} +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +v_poisoned +NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 13 +DROP TABLE ts; +# 5b. The same column reached through a view. A merged view reads the +# table directly; a materialised one copies the value into a +# temporary table first. +CREATE TABLE tv (id INT, j JSON); +INSERT INTO tv VALUES (1, '{"a": 1, "b": [1, 2]}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tv VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +CREATE ALGORITHM=MERGE VIEW v_merge AS SELECT id, j FROM tv; +CREATE ALGORITHM=TEMPTABLE VIEW v_tmp AS SELECT id, j FROM tv; +SHOW CREATE VIEW v_merge; +View Create View character_set_client collation_connection +v_merge CREATE ALGORITHM=MERGE DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v_merge` AS select `tv`.`id` AS `id`,`tv`.`j` AS `j` from `tv` utf8mb4 utf8mb4_general_ci +SHOW CREATE VIEW v_tmp; +View Create View character_set_client collation_connection +v_tmp CREATE ALGORITHM=TEMPTABLE DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v_tmp` AS select `tv`.`id` AS `id`,`tv`.`j` AS `j` from `tv` utf8mb4 utf8mb4_general_ci +SELECT id, j, HEX(j) FROM v_merge ORDER BY id; +id j HEX(j) +1 {"a": 1, "b": [1, 2]} 7B2261223A20312C202262223A205B312C20325D7D +2 {"a":1, 7B2261223A312C +SELECT id, j, HEX(j) FROM v_tmp ORDER BY id; +id j HEX(j) +1 {"a": 1, "b": [1, 2]} 7B2261223A20312C202262223A205B312C20325D7D +2 {"a":1, 7B2261223A312C +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok +FROM v_merge ORDER BY id; +id v ok +1 [{"a": 1, "b": [1, 2]}] 1 +2 NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok +FROM v_tmp ORDER BY id; +id v ok +1 [{"a": 1, "b": [1, 2]}] 1 +2 NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_merge ORDER BY id; +id v +1 {"a": 1, "b": [1, 2], "z": 1} +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_tmp ORDER BY id; +id v +1 {"a": 1, "b": [1, 2], "z": 1} +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT JSON_ARRAYAGG(j) AS v FROM v_merge WHERE id = 1; +v +[{"a": 1, "b": [1, 2]}] +DROP VIEW v_merge, v_tmp; +DROP TABLE tv; +# +# 6. A valid column, for comparison: the same calls with nothing wrong. +# +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id < 10 ORDER BY id; +id v +1 [{"a":1,"b":[1,2]}] +2 [{"a": 1, "b": [1, 2]}] +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id < 10 ORDER BY id; +id v +1 {"a": 1, "b": [1, 2], "z": 1} +2 {"a": 1, "b": [1, 2], "z": 1} +SELECT id, JSON_EXTRACT(j, '$.b') AS v FROM t1 WHERE id < 10 ORDER BY id; +id v +1 [1, 2] +2 [1, 2] +# a column value keeps the formatting it was stored with +SELECT id, HEX(j) FROM t1 WHERE id < 10 ORDER BY id; +id HEX(j) +1 7B2261223A312C2262223A5B312C325D7D +2 7B2261223A20312C202262223A205B312C20325D7D +# 6a. The same document stored in every column type it can live in. +# Retrieval returns the stored bytes unchanged in all of them; +# only the JSON-typed column is embedded rather than quoted. +CREATE TABLE tr (jj JSON, lt LONGTEXT CHECK (JSON_VALID(lt)), vc VARCHAR(64), +vb VARBINARY(64), bl BLOB); +INSERT INTO tr VALUES ('{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', +'{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', +'{"a": 1, "b":[1, 2]}'); +SELECT HEX(jj) AS h_json, HEX(lt) AS h_longtext, HEX(vc) AS h_varchar, +HEX(vb) AS h_varbinary, HEX(bl) AS h_blob FROM tr; +h_json h_longtext h_varchar h_varbinary h_blob +7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D 7B2261223A20312C20202262223A5B312C20325D7D +SELECT jj, lt, vc, vb, bl FROM tr; +jj lt vc vb bl +{"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} {"a": 1, "b":[1, 2]} +SELECT JSON_ARRAY(jj) AS a_json, JSON_ARRAY(vc) AS a_varchar, +JSON_ARRAY(vb) AS a_varbinary, JSON_ARRAY(bl) AS a_blob FROM tr; +a_json a_varchar a_varbinary a_blob +[{"a": 1, "b":[1, 2]}] ["{\"a\": 1, \"b\":[1, 2]}"] ["{\"a\": 1, \"b\":[1, 2]}"] ["{\"a\": 1, \"b\":[1, 2]}"] +DROP TABLE tr; +# +# 7. Storing a JSON function result back into a column, which is where +# the check constraint runs again. +# +CREATE TABLE t6 (j JSON); +INSERT INTO t6 SELECT JSON_SET('{"a":1}', '$.b', 2); +INSERT INTO t6 SELECT JSON_ARRAY(1,2); +INSERT INTO t6 SELECT JSON_OBJECT('a',1); +SELECT j FROM t6; +j +{"a": 1, "b": 2} +[1, 2] +{"a": 1} +# a value the constructor will not build. A released server built +# something 32 deep, which is one too many to read back, and the +# check on the column was what caught it. The constructor now +# answers NULL, and the complaint that says why is a warning, which +# a statement inserting under strict mode makes an error of - so +# the statement stops at the value rather than at the column, and +# says what is wrong with the value. +CREATE TABLE t7 (j JSON); +INSERT INTO t7 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +ERROR HY000: Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT COUNT(*) AS stored FROM t7; +stored +0 +# the same value into a plain LONGTEXT column, which has no check +# on it to reach: the constructor is where it stops either way +CREATE TABLE t8 (j LONGTEXT); +INSERT INTO t8 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +ERROR HY000: Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT COUNT(*) AS stored FROM t8; +stored +0 +# a value that is truncated on the way into the column +CREATE TABLE t9 (j VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t9 SELECT JSON_SET('{"a":1,"b":2}', '$.a', 9); +Warnings: +Warning 1265 Data truncated for column 'j' at row 1 +SELECT j, JSON_VALID(j) AS still_valid FROM t9; +j still_valid +{"a": 9, " 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode=DEFAULT; +DROP TABLE t6, t7, t8, t9; +# 7a. The same truncating store into a column that IS checked. The +# chop happens first and the constraint then judges the chopped +# bytes, which sometimes still parse and sometimes do not. +CREATE TABLE tn (v VARCHAR(10) CHECK (json_valid(v))); +SET @@sql_mode=''; +# chopped into an unparseable prefix: rejected +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +ERROR 23000: CONSTRAINT `tn.v` failed for `test`.`tn` +# chopped on a boundary that still parses: accepted, with a warning +INSERT INTO tn VALUES (CONCAT('[1,2,3,45]', ',6789')); +Warnings: +Warning 1265 Data truncated for column 'v' at row 1 +INSERT INTO tn VALUES ('{"a":1}'); +SELECT v, JSON_VALID(v) AS valid FROM tn ORDER BY v; +v valid +[1,2,3,45] 1 +{"a":1} 1 +SET @@sql_mode=DEFAULT; +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +ERROR 22001: Data too long for column 'v' at row 1 +SELECT COUNT(*) AS stored FROM tn; +stored +2 +DROP TABLE tn; +# 7b. A store that converts the character set on the way in. The +# constraint judges the converted bytes, not the ones the +# function produced. +# swe7 cannot carry the JSON punctuation at all +CREATE TABLE t11 (j LONGTEXT CHARACTER SET swe7 CHECK (json_valid(j))); +SET @@sql_mode=''; +INSERT INTO t11 VALUES ('{"a":1}'); +ERROR 23000: CONSTRAINT `t11.j` failed for `test`.`t11` +SELECT COUNT(*) AS stored FROM t11; +stored +0 +# the same column without a constraint keeps the damaged bytes +CREATE TABLE t12 (j LONGTEXT CHARACTER SET swe7); +INSERT INTO t12 VALUES ('{"a":1}'); +Warnings: +Warning 1366 Incorrect string value: '{"a":1...' for column `test`.`t12`.`j` at row 1 +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t12; +stored v +3F2261223A313F 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SET @@sql_mode=DEFAULT; +INSERT INTO t11 VALUES ('{"a":1}'); +ERROR 22007: Incorrect string value: '{"a":1...' for column `test`.`t11`.`j` at row 1 +# a conversion that loses nothing still changes the bytes +CREATE TABLE t13 (j VARCHAR(100) CHARACTER SET latin1 CHECK (json_valid(j))); +INSERT INTO t13 VALUES (JSON_OBJECT('a', _utf8mb4 X'C3A9')); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t13; +stored v +7B2261223A2022E9227D 1 +CREATE TABLE tu (j JSON); +INSERT INTO tu VALUES ('{"a":1}'); +INSERT INTO t13 SELECT JSON_SET(j, '$.c', 3) FROM tu; +SELECT HEX(j) FROM t13 ORDER BY j; +HEX(j) +7B2261223A2022E9227D +7B2261223A20312C202263223A20337D +DROP TABLE t11, t12, t13, tu; +# 7c. A table rebuilt by ALTER, which checks every existing row again +# rather than the value of one statement. +CREATE TABLE ta (id INT, j JSON); +INSERT INTO ta VALUES (1, '{"a":1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ta VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +ALTER TABLE ta FORCE, ALGORITHM=COPY; +ERROR 23000: CONSTRAINT `ta.j` failed for `test`.`ta` +ALTER IGNORE TABLE ta FORCE, ALGORITHM=COPY; +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Warning 4025 CONSTRAINT `ta.j` failed for `test`.`ta` +SELECT id, j FROM ta ORDER BY id; +id j +1 {"a":1} +DROP TABLE ta; +# ALTER that converts the character set rewrites the stored bytes +CREATE TABLE tb (j JSON); +INSERT INTO tb VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +ALTER TABLE tb CONVERT TO CHARACTER SET latin1; +SHOW CREATE TABLE tb; +Table Create Table +tb CREATE TABLE `tb` ( + `j` longtext DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT HEX(j) AS stored, JSON_VALID(j) AS valid FROM tb; +stored valid +7B2261223A20312C202262223A2022E9227D 1 +SELECT HEX(JSON_ARRAY(j)) AS h FROM tb; +h +5B7B2261223A20312C202262223A2022E9227D5D +DROP TABLE tb; +# 7d. Generated columns, where the JSON function result is computed +# and stored by the server rather than supplied by the statement. +CREATE TABLE tg (j JSON, +c1 LONGTEXT AS (JSON_SET(j, '$.z', 9)) PERSISTENT, +c2 VARCHAR(20) AS (JSON_EXTRACT(j, '$.a')) PERSISTENT, KEY (c2)); +SHOW CREATE TABLE tg; +Table Create Table +tg CREATE TABLE `tg` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)), + `c1` longtext GENERATED ALWAYS AS (json_set(`j`,'$.z',9)) STORED, + `c2` varchar(20) GENERATED ALWAYS AS (json_extract(`j`,'$.a')) STORED, + KEY `c2` (`c2`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO tg (j) VALUES ('{"a": 1, "b": 2}'); +SELECT j, c1, HEX(c1), c2 FROM tg; +j c1 HEX(c1) c2 +{"a": 1, "b": 2} {"a": 1, "b": 2, "z": 9} 7B2261223A20312C202262223A20322C20227A223A20397D 1 +UPDATE tg SET j = '{"a": 7}'; +SELECT j, c1, HEX(c1), c2 FROM tg; +j c1 HEX(c1) c2 +{"a": 7} {"a": 7, "z": 9} 7B2261223A20372C20227A223A20397D 7 +SELECT c2 FROM tg FORCE INDEX (c2) WHERE c2 = '7'; +c2 +7 +DROP TABLE tg; +# +# 8. A temporary table the user created. It is under user control the +# same way a base table is, including the check switch. +# +CREATE TEMPORARY TABLE tmp1 (id INT, j JSON); +SHOW CREATE TABLE tmp1; +Table Create Table +tmp1 CREATE TEMPORARY TABLE `tmp1` ( + `id` int(11) DEFAULT NULL, + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +INSERT INTO tmp1 VALUES (1, '{"a":1}'); +INSERT INTO tmp1 VALUES (2, '{'); +ERROR 23000: CONSTRAINT `tmp1.j` failed for `test`.`tmp1` +SET SESSION check_constraint_checks = OFF; +INSERT INTO tmp1 VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS valid FROM tmp1 ORDER BY id; +id j valid +1 {"a":1} 1 +2 {"a":1, 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT id, JSON_ARRAY(j) AS v FROM tmp1 ORDER BY id; +id v +1 [{"a":1}] +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM tmp1 ORDER BY id; +id still_valid +1 1 +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM tmp1 ORDER BY id; +id v +1 {"a": 1, "z": 1} +2 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +DROP TEMPORARY TABLE tmp1; +# +# 9. Values passing through a temporary table the server made for +# itself: grouping, materialised derived tables, union branches and +# window frames. +# +CREATE TABLE t10 (g INT, j JSON, txt VARCHAR(30)); +INSERT INTO t10 VALUES +(1, '{"a":1}', '{"a":1}'), (1, '{"a":2}', '{"a":2}'), +(2, '{"b":1}', '{"b":1}'); +# grouping over a JSON expression +SELECT g, JSON_ARRAYAGG(JSON_SET(j, '$.z', 1)) AS v FROM t10 GROUP BY g ORDER BY g; +g v +1 [{"a": 1, "z": 1},{"a": 2, "z": 1}] +2 [{"b": 1, "z": 1}] +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j ORDER BY j; +v +{"a": 1, "z": 1} +{"a": 2, "z": 1} +{"b": 1, "z": 1} +SELECT g, COUNT(*) AS c FROM t10 GROUP BY g, JSON_SET(j, '$.z', 1) ORDER BY g, c; +g c +1 1 +1 1 +2 1 +# a materialised derived table +SELECT JSON_VALID(v) AS valid, v +FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d +ORDER BY v; +valid v +1 {"a": 1, "z": 1} +1 {"a": 2, "z": 1} +1 {"b": 1, "z": 1} +SELECT JSON_SET(v, '$.y', 2) AS w +FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d +ORDER BY w; +w +{"a": 1, "z": 1, "y": 2} +{"a": 2, "z": 1, "y": 2} +{"b": 1, "z": 1, "y": 2} +SELECT JSON_ARRAY(v) AS w +FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d +ORDER BY w; +w +[{"a": 1, "z": 1}] +[{"a": 2, "z": 1}] +[{"b": 1, "z": 1}] +# union branches, where one branch is a document and the other is text +SELECT v, JSON_VALID(v) AS valid FROM ( +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 +UNION +SELECT txt FROM t10 WHERE g = 2) u ORDER BY v; +v valid +{"a": 1, "z": 1} 1 +{"a": 2, "z": 1} 1 +{"b":1} 1 +SELECT JSON_ARRAY(v) AS w FROM ( +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 +UNION +SELECT txt FROM t10 WHERE g = 2) u ORDER BY w; +w +["{\"a\": 1, \"z\": 1}"] +["{\"a\": 2, \"z\": 1}"] +["{\"b\":1}"] +# a window frame, which materialises its input +SELECT g, JSON_SET(j, '$.z', 1) AS v, +ROW_NUMBER() OVER (PARTITION BY g ORDER BY j) AS rn +FROM t10 ORDER BY g, rn; +g v rn +1 {"a": 1, "z": 1} 1 +1 {"a": 2, "z": 1} 2 +2 {"b": 1, "z": 1} 1 +SELECT g, JSON_ARRAY(JSON_SET(j, '$.z', 1)) AS v, +COUNT(*) OVER (PARTITION BY g) AS c +FROM t10 ORDER BY g, v; +g v c +1 [{"a": 1, "z": 1}] 2 +1 [{"a": 2, "z": 1}] 2 +2 [{"b": 1, "z": 1}] 1 +# an ORDER BY that has to materialise the value first +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 ORDER BY JSON_EXTRACT(j, '$.a'), v; +v +{"b": 1, "z": 1} +{"a": 1, "z": 1} +{"a": 2, "z": 1} +DROP TABLE t1, t2, t3, t4, t5, t10; diff --git a/mysql-test/main/func_json_columns.test b/mysql-test/main/func_json_columns.test new file mode 100644 index 0000000000000..e1fa8eba4c6e0 --- /dev/null +++ b/mysql-test/main/func_json_columns.test @@ -0,0 +1,445 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: JSON values that come out of a table. +--echo # +--echo # A JSON column carries a check constraint that is supposed to keep +--echo # invalid documents out, but the check can be switched off for the +--echo # duration of a statement, after which the column holds bytes that no +--echo # longer satisfy it. This test records what every JSON function does +--echo # with such a column, what the check constraints look like, and how a +--echo # user-created temporary table compares with a base table. +--echo # + +SET NAMES utf8mb4; + +# A check constraint is looked at while the column definition is validated, +# which for a prepared CREATE TABLE is prepare time, so the warnings the +# constraints below draw only reach the client when prepare warnings are +# asked for. +--enable_prepare_warnings + +--echo # +--echo # 1. What declaring a column JSON actually creates. +--echo # + +CREATE TABLE t1 (id INT, j JSON); +SHOW CREATE TABLE t1; +SELECT column_name, data_type, column_type + FROM information_schema.columns + WHERE table_schema = 'test' AND table_name = 't1' + ORDER BY ordinal_position; +SELECT constraint_name, check_clause + FROM information_schema.check_constraints + WHERE constraint_schema = 'test' AND table_name = 't1'; + +--echo # a column that is only LONGTEXT does not get the constraint +CREATE TABLE t2 (id INT, j LONGTEXT); +SHOW CREATE TABLE t2; +SELECT COUNT(*) AS constraints + FROM information_schema.check_constraints + WHERE constraint_schema = 'test' AND table_name = 't2'; + +--echo # a constraint written out by hand, worded exactly like the automatic one +CREATE TABLE t3 (id INT, j LONGTEXT CHECK (json_valid(j))); +SHOW CREATE TABLE t3; +--echo # a constraint that mentions the column inside a larger expression +CREATE TABLE t4 (id INT, j LONGTEXT CHECK (json_valid(j) AND id > 0)); +SHOW CREATE TABLE t4; +--echo # a table level constraint over two columns +CREATE TABLE t5 (id INT, j LONGTEXT, k LONGTEXT, + CONSTRAINT both_valid CHECK (json_valid(j) AND json_valid(k))); +SHOW CREATE TABLE t5; + +--echo # +--echo # 2. The constraint doing its job. +--echo # + +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'); +INSERT INTO t1 VALUES (2, '{"a": 1, "b": [1, 2]}'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, '{"a":1,'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (3, 'not json at all'); +--error ER_CONSTRAINT_FAILED +UPDATE t1 SET j = '{' WHERE id = 1; +SELECT id, j FROM t1 ORDER BY id; +--echo # the hand written constraint behaves the same way +--error ER_CONSTRAINT_FAILED +INSERT INTO t3 VALUES (1, '{'); +INSERT INTO t3 VALUES (1, '{"a":1}'); +SELECT id, j FROM t3; + +--echo # the constraint shapes that are NOT a bare check of one column. Each +--echo # of these still has to run in full, so what they accept and reject is +--echo # recorded here rather than left to the CREATE statement alone. +--echo # constraint on the column combined with a test of another column +--error ER_CONSTRAINT_FAILED +INSERT INTO t4 VALUES (1, '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t4 VALUES (0, '{"a":1}'); +INSERT INTO t4 VALUES (1, '{"a":1}'); +SELECT id, j FROM t4; +--echo # table level constraint over two columns +--error ER_CONSTRAINT_FAILED +INSERT INTO t5 VALUES (1, '{', '{"a":1}'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5 VALUES (1, '{"a":1}', '{'); +INSERT INTO t5 VALUES (1, '{"a":1}', '{"b":2}'); +SELECT id, j, k FROM t5; +--echo # an OR, where the other side can let an invalid document through +CREATE TABLE t5b (id INT, j LONGTEXT CHECK (json_valid(j) OR id > 0)); +INSERT INTO t5b VALUES (1, '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5b VALUES (0, '{'); +SELECT id, j, JSON_VALID(j) AS valid FROM t5b; +--echo # an AND whose parts all have to hold, but where none of them is +--echo # the call: the promise is inside an OR one level down, so the +--echo # constraint can pass without it and the column stays ordinary +CREATE TABLE t5d (id INT, + j LONGTEXT CHECK (LENGTH(j) > 0 AND + (json_valid(j) OR id > 0))); +INSERT INTO t5d VALUES (1, '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5d VALUES (0, '{'); +INSERT INTO t5d VALUES (0, '{"a":1}'); +SELECT id, j, JSON_VALID(j) AS valid FROM t5d ORDER BY id, j; +--echo # a constraint attached to one column that tests a different column +CREATE TABLE t5c (a LONGTEXT, b LONGTEXT CHECK (json_valid(a))); +INSERT INTO t5c VALUES ('{"a":1}', '{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5c VALUES ('{', '{"a":1}'); +SELECT a, b FROM t5c; +--echo # writing any constraint of your own on a JSON column replaces the +--echo # automatic one, so invalid documents go in with the checks left on +CREATE TABLE t5e (j JSON CHECK (LENGTH(j) > 0)); +SHOW CREATE TABLE t5e; +INSERT INTO t5e VALUES ('{"a":1}'); +INSERT INTO t5e VALUES ('{'); +--error ER_CONSTRAINT_FAILED +INSERT INTO t5e VALUES (''); +SELECT j, JSON_VALID(j) AS valid FROM t5e ORDER BY j; +SELECT JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS still_valid + FROM t5e ORDER BY j; +DROP TABLE t5b, t5c, t5d, t5e; + +--echo # +--echo # 3. Switching the check off, which is what makes a stored document +--echo # unverifiable afterwards. +--echo # + +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,'); +INSERT INTO t1 VALUES (11, 'not json at all'); +INSERT INTO t1 VALUES (12, '{"a":1} trailing'); +INSERT INTO t1 VALUES (13, CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32))); +INSERT INTO t3 VALUES (10, '{"a":1,'); +SET SESSION check_constraint_checks = ON; + +--echo # the rows are there and the constraint is back on +SELECT id, j FROM t1 ORDER BY id; +SELECT id, JSON_VALID(j) AS valid FROM t1 ORDER BY id; +--echo # a new statement is checked again, the stored rows are not +--error ER_CONSTRAINT_FAILED +INSERT INTO t1 VALUES (14, '{'); +--echo # reading the column back does not check anything +SELECT id, j FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, HEX(j) FROM t1 WHERE id = 10; + +--echo # +--echo # 4. Every function fed the column that no longer satisfies its check. +--echo # + +SELECT id, JSON_VALID(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_TYPE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_DEPTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_LENGTH(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_KEYS(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_EXTRACT(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_QUERY(j, '$') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALUE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_INSERT(j, '$.z', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_REPLACE(j, '$.a', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_REMOVE(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_ARRAY_APPEND(j, '$', 1) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_MERGE(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_MERGE_PATCH(j, '{"z":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_COMPACT(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_LOOSE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_NORMALIZE(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_EXISTS(j, '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_CONTAINS(j, '1', '$.a') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_SEARCH(j, 'one', '1') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_EQUALS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_OVERLAPS(j, '{"a":1}') AS v FROM t1 WHERE id >= 10 ORDER BY id; + +--echo # +--echo # 5. The column as an embedded argument. A JSON column is embedded as +--echo # a document, so whatever it holds is copied into the result as is. +--echo # + +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS still_valid + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +SELECT id, JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10 GROUP BY id ORDER BY id; +--echo # the same column read as plain text is quoted instead of embedded +SELECT id, JSON_ARRAY(CONCAT(j, '')) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(CONCAT(j, ''))) AS still_valid + FROM t1 WHERE id >= 10 ORDER BY id; +--echo # a LONGTEXT column holding the same bytes is quoted, not embedded +SELECT id, JSON_ARRAY(j) AS v FROM t3 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM t3 ORDER BY id; + +--echo # 5a. The same column reached through a scalar subquery, where the +--echo # value is fetched once and held rather than read per row. +CREATE TABLE ts (id INT, j JSON); +INSERT INTO ts VALUES (1, '{"a": 1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ts VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 1)) AS v; +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 1))) AS ok; +SELECT JSON_ARRAY((SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +SELECT JSON_VALID(JSON_ARRAY((SELECT j FROM ts WHERE id = 2))) AS ok_poisoned; +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 1)) AS v; +SELECT JSON_OBJECT('k', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 1)) AS v; +SELECT JSON_SET('{}', '$.d', (SELECT j FROM ts WHERE id = 2)) AS v_poisoned; +DROP TABLE ts; + +--echo # 5b. The same column reached through a view. A merged view reads the +--echo # table directly; a materialised one copies the value into a +--echo # temporary table first. +CREATE TABLE tv (id INT, j JSON); +INSERT INTO tv VALUES (1, '{"a": 1, "b": [1, 2]}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tv VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +CREATE ALGORITHM=MERGE VIEW v_merge AS SELECT id, j FROM tv; +CREATE ALGORITHM=TEMPTABLE VIEW v_tmp AS SELECT id, j FROM tv; +SHOW CREATE VIEW v_merge; +SHOW CREATE VIEW v_tmp; +SELECT id, j, HEX(j) FROM v_merge ORDER BY id; +SELECT id, j, HEX(j) FROM v_tmp ORDER BY id; +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok + FROM v_merge ORDER BY id; +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS ok + FROM v_tmp ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_merge ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM v_tmp ORDER BY id; +SELECT JSON_ARRAYAGG(j) AS v FROM v_merge WHERE id = 1; +DROP VIEW v_merge, v_tmp; +DROP TABLE tv; + +--echo # +--echo # 6. A valid column, for comparison: the same calls with nothing wrong. +--echo # + +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id < 10 ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM t1 WHERE id < 10 ORDER BY id; +SELECT id, JSON_EXTRACT(j, '$.b') AS v FROM t1 WHERE id < 10 ORDER BY id; +--echo # a column value keeps the formatting it was stored with +SELECT id, HEX(j) FROM t1 WHERE id < 10 ORDER BY id; + +--echo # 6a. The same document stored in every column type it can live in. +--echo # Retrieval returns the stored bytes unchanged in all of them; +--echo # only the JSON-typed column is embedded rather than quoted. +CREATE TABLE tr (jj JSON, lt LONGTEXT CHECK (JSON_VALID(lt)), vc VARCHAR(64), + vb VARBINARY(64), bl BLOB); +INSERT INTO tr VALUES ('{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', + '{"a": 1, "b":[1, 2]}', '{"a": 1, "b":[1, 2]}', + '{"a": 1, "b":[1, 2]}'); +SELECT HEX(jj) AS h_json, HEX(lt) AS h_longtext, HEX(vc) AS h_varchar, + HEX(vb) AS h_varbinary, HEX(bl) AS h_blob FROM tr; +SELECT jj, lt, vc, vb, bl FROM tr; +SELECT JSON_ARRAY(jj) AS a_json, JSON_ARRAY(vc) AS a_varchar, + JSON_ARRAY(vb) AS a_varbinary, JSON_ARRAY(bl) AS a_blob FROM tr; +DROP TABLE tr; + +--echo # +--echo # 7. Storing a JSON function result back into a column, which is where +--echo # the check constraint runs again. +--echo # + +CREATE TABLE t6 (j JSON); +INSERT INTO t6 SELECT JSON_SET('{"a":1}', '$.b', 2); +INSERT INTO t6 SELECT JSON_ARRAY(1,2); +INSERT INTO t6 SELECT JSON_OBJECT('a',1); +SELECT j FROM t6; +--echo # a value the constructor will not build. A released server built +--echo # something 32 deep, which is one too many to read back, and the +--echo # check on the column was what caught it. The constructor now +--echo # answers NULL, and the complaint that says why is a warning, which +--echo # a statement inserting under strict mode makes an error of - so +--echo # the statement stops at the value rather than at the column, and +--echo # says what is wrong with the value. +CREATE TABLE t7 (j JSON); +--error ER_JSON_DEPTH +INSERT INTO t7 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +SELECT COUNT(*) AS stored FROM t7; +--echo # the same value into a plain LONGTEXT column, which has no check +--echo # on it to reach: the constructor is where it stops either way +CREATE TABLE t8 (j LONGTEXT); +--error ER_JSON_DEPTH +INSERT INTO t8 SELECT JSON_ARRAY(JSON_COMPACT(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)))); +SELECT COUNT(*) AS stored FROM t8; +--echo # a value that is truncated on the way into the column +CREATE TABLE t9 (j VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t9 SELECT JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT j, JSON_VALID(j) AS still_valid FROM t9; +SET @@sql_mode=DEFAULT; +DROP TABLE t6, t7, t8, t9; + +--echo # 7a. The same truncating store into a column that IS checked. The +--echo # chop happens first and the constraint then judges the chopped +--echo # bytes, which sometimes still parse and sometimes do not. +CREATE TABLE tn (v VARCHAR(10) CHECK (json_valid(v))); +SET @@sql_mode=''; +--echo # chopped into an unparseable prefix: rejected +--error ER_CONSTRAINT_FAILED +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +--echo # chopped on a boundary that still parses: accepted, with a warning +INSERT INTO tn VALUES (CONCAT('[1,2,3,45]', ',6789')); +INSERT INTO tn VALUES ('{"a":1}'); +SELECT v, JSON_VALID(v) AS valid FROM tn ORDER BY v; +SET @@sql_mode=DEFAULT; +--error ER_DATA_TOO_LONG +INSERT INTO tn VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT COUNT(*) AS stored FROM tn; +DROP TABLE tn; + +--echo # 7b. A store that converts the character set on the way in. The +--echo # constraint judges the converted bytes, not the ones the +--echo # function produced. +--echo # swe7 cannot carry the JSON punctuation at all +CREATE TABLE t11 (j LONGTEXT CHARACTER SET swe7 CHECK (json_valid(j))); +SET @@sql_mode=''; +--error ER_CONSTRAINT_FAILED +INSERT INTO t11 VALUES ('{"a":1}'); +SELECT COUNT(*) AS stored FROM t11; +--echo # the same column without a constraint keeps the damaged bytes +CREATE TABLE t12 (j LONGTEXT CHARACTER SET swe7); +INSERT INTO t12 VALUES ('{"a":1}'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t12; +SET @@sql_mode=DEFAULT; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +INSERT INTO t11 VALUES ('{"a":1}'); +--echo # a conversion that loses nothing still changes the bytes +CREATE TABLE t13 (j VARCHAR(100) CHARACTER SET latin1 CHECK (json_valid(j))); +INSERT INTO t13 VALUES (JSON_OBJECT('a', _utf8mb4 X'C3A9')); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v FROM t13; +CREATE TABLE tu (j JSON); +INSERT INTO tu VALUES ('{"a":1}'); +INSERT INTO t13 SELECT JSON_SET(j, '$.c', 3) FROM tu; +SELECT HEX(j) FROM t13 ORDER BY j; +DROP TABLE t11, t12, t13, tu; + +--echo # 7c. A table rebuilt by ALTER, which checks every existing row again +--echo # rather than the value of one statement. +CREATE TABLE ta (id INT, j JSON); +INSERT INTO ta VALUES (1, '{"a":1}'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO ta VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +--error ER_CONSTRAINT_FAILED +ALTER TABLE ta FORCE, ALGORITHM=COPY; +ALTER IGNORE TABLE ta FORCE, ALGORITHM=COPY; +SELECT id, j FROM ta ORDER BY id; +DROP TABLE ta; +--echo # ALTER that converts the character set rewrites the stored bytes +CREATE TABLE tb (j JSON); +INSERT INTO tb VALUES (JSON_SET('{"a":1}', '$.b', _utf8mb4 X'C3A9')); +ALTER TABLE tb CONVERT TO CHARACTER SET latin1; +SHOW CREATE TABLE tb; +SELECT HEX(j) AS stored, JSON_VALID(j) AS valid FROM tb; +SELECT HEX(JSON_ARRAY(j)) AS h FROM tb; +DROP TABLE tb; + +--echo # 7d. Generated columns, where the JSON function result is computed +--echo # and stored by the server rather than supplied by the statement. +CREATE TABLE tg (j JSON, + c1 LONGTEXT AS (JSON_SET(j, '$.z', 9)) PERSISTENT, + c2 VARCHAR(20) AS (JSON_EXTRACT(j, '$.a')) PERSISTENT, KEY (c2)); +SHOW CREATE TABLE tg; +INSERT INTO tg (j) VALUES ('{"a": 1, "b": 2}'); +SELECT j, c1, HEX(c1), c2 FROM tg; +UPDATE tg SET j = '{"a": 7}'; +SELECT j, c1, HEX(c1), c2 FROM tg; +SELECT c2 FROM tg FORCE INDEX (c2) WHERE c2 = '7'; +DROP TABLE tg; + +--echo # +--echo # 8. A temporary table the user created. It is under user control the +--echo # same way a base table is, including the check switch. +--echo # + +CREATE TEMPORARY TABLE tmp1 (id INT, j JSON); +SHOW CREATE TABLE tmp1; +INSERT INTO tmp1 VALUES (1, '{"a":1}'); +--error ER_CONSTRAINT_FAILED +INSERT INTO tmp1 VALUES (2, '{'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tmp1 VALUES (2, '{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT id, j, JSON_VALID(j) AS valid FROM tmp1 ORDER BY id; +SELECT id, JSON_ARRAY(j) AS v FROM tmp1 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS still_valid FROM tmp1 ORDER BY id; +SELECT id, JSON_SET(j, '$.z', 1) AS v FROM tmp1 ORDER BY id; +DROP TEMPORARY TABLE tmp1; + +--echo # +--echo # 9. Values passing through a temporary table the server made for +--echo # itself: grouping, materialised derived tables, union branches and +--echo # window frames. +--echo # + +CREATE TABLE t10 (g INT, j JSON, txt VARCHAR(30)); +INSERT INTO t10 VALUES + (1, '{"a":1}', '{"a":1}'), (1, '{"a":2}', '{"a":2}'), + (2, '{"b":1}', '{"b":1}'); + +--echo # grouping over a JSON expression +SELECT g, JSON_ARRAYAGG(JSON_SET(j, '$.z', 1)) AS v FROM t10 GROUP BY g ORDER BY g; +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j ORDER BY j; +SELECT g, COUNT(*) AS c FROM t10 GROUP BY g, JSON_SET(j, '$.z', 1) ORDER BY g, c; + +--echo # a materialised derived table +SELECT JSON_VALID(v) AS valid, v + FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d + ORDER BY v; +SELECT JSON_SET(v, '$.y', 2) AS w + FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d + ORDER BY w; +SELECT JSON_ARRAY(v) AS w + FROM (SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 GROUP BY j) d + ORDER BY w; + +--echo # union branches, where one branch is a document and the other is text +SELECT v, JSON_VALID(v) AS valid FROM ( + SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 + UNION + SELECT txt FROM t10 WHERE g = 2) u ORDER BY v; +SELECT JSON_ARRAY(v) AS w FROM ( + SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 WHERE g = 1 + UNION + SELECT txt FROM t10 WHERE g = 2) u ORDER BY w; + +--echo # a window frame, which materialises its input +SELECT g, JSON_SET(j, '$.z', 1) AS v, + ROW_NUMBER() OVER (PARTITION BY g ORDER BY j) AS rn + FROM t10 ORDER BY g, rn; +SELECT g, JSON_ARRAY(JSON_SET(j, '$.z', 1)) AS v, + COUNT(*) OVER (PARTITION BY g) AS c + FROM t10 ORDER BY g, v; + +--echo # an ORDER BY that has to materialise the value first +SELECT JSON_SET(j, '$.z', 1) AS v FROM t10 ORDER BY JSON_EXTRACT(j, '$.a'), v; + +DROP TABLE t1, t2, t3, t4, t5, t10; + +--disable_prepare_warnings diff --git a/mysql-test/main/func_json_depth.result b/mysql-test/main/func_json_depth.result new file mode 100644 index 0000000000000..eff0354ddeef8 --- /dev/null +++ b/mysql-test/main/func_json_depth.result @@ -0,0 +1,380 @@ +# +# Behavioral baseline: nesting depth. +# +# The scanner keeps a fixed stack, so documents nested past its size +# are rejected. This test records the exact depth at which each +# function changes its answer, and what happens when a document that +# is acceptable on its own is spliced into another document deep +# enough that the combination is not. +# +# +# 1. Where the limit sits, for arrays and for objects. +# +SET @d29 = CONCAT(REPEAT('[', 29), '1', REPEAT(']', 29)); +SET @d30 = CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @d31 = CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d32 = CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32)); +SET @d33 = CONCAT(REPEAT('[', 33), '1', REPEAT(']', 33)); +SELECT JSON_VALID(@d29) AS v29, JSON_VALID(@d30) AS v30, JSON_VALID(@d31) AS v31; +v29 v30 v31 +1 1 1 +SELECT JSON_VALID(@d32) AS v32; +v32 +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_VALID(@d33) AS v33; +v33 +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_DEPTH(@d29) AS d29, JSON_DEPTH(@d30) AS d30, JSON_DEPTH(@d31) AS d31; +d29 d30 d31 +30 31 32 +SELECT JSON_DEPTH(@d32) AS d32; +d32 +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +# the same depths built from objects +SET @o30 = CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @o31 = CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SET @o32 = CONCAT(REPEAT('{"a":', 32), '1', REPEAT('}', 32)); +SELECT JSON_VALID(@o30) AS v30, JSON_VALID(@o31) AS v31, JSON_VALID(@o32) AS v32; +v30 v31 v32 +1 1 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 156 +SELECT JSON_DEPTH(@o31) AS d31; +d31 +32 +SELECT JSON_DEPTH(@o32) AS d32; +d32 +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 156 +# alternating containers reach the same limit +SET @m30 = CONCAT(REPEAT('[{"a":', 15), '1', REPEAT('}]', 15)); +SET @m32 = CONCAT(REPEAT('[{"a":', 16), '1', REPEAT('}]', 16)); +SELECT JSON_VALID(@m30) AS v30, JSON_VALID(@m32) AS v32; +v30 v32 +1 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 92 +SELECT JSON_DEPTH(@m30) AS d30; +d30 +31 +# +# 2. Every function at, and one past, the limit. +# +SELECT JSON_VALID(@d31) AS ok, JSON_VALID(@d32) AS over_limit; +ok over_limit +1 0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_DEPTH(@d31) AS ok; +ok +32 +SELECT JSON_DEPTH(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +SELECT JSON_TYPE(@d31) AS ok; +ok +ARRAY +SELECT JSON_TYPE(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_type' at position 32 +SELECT JSON_LENGTH(@d31) AS ok; +ok +1 +SELECT JSON_LENGTH(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_length' at position 32 +SELECT JSON_COMPACT(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_compact' at position 32 +SELECT JSON_LOOSE(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_loose' at position 32 +SELECT JSON_DETAILED(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_detailed' at position 32 +SELECT JSON_NORMALIZE(@d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_normalize' at position 32 +SELECT JSON_EXTRACT(@d32, '$') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_extract' at position 32 +SELECT JSON_QUERY(@d32, '$') AS over_limit; +over_limit +NULL +SELECT JSON_VALUE(@d32, '$[0]') AS over_limit; +over_limit +NULL +SELECT JSON_KEYS(@o32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_keys' at position 156 +SELECT JSON_SET(@d32, '$[0]', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 32 +SELECT JSON_INSERT(@d32, '$.a', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 32 +SELECT JSON_REPLACE(@d32, '$[0]', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 32 +SELECT JSON_REMOVE(@d32, '$[0]') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_remove' at position 32 +SELECT JSON_ARRAY_APPEND(@d32, '$', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +SELECT JSON_ARRAY_INSERT(@d32, '$[0]', 2) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_insert' at position 35 +SELECT JSON_MERGE(@d32, '[1]') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 32 +SELECT JSON_MERGE('[1]', @d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_merge_preserve' at position 32 +SELECT JSON_MERGE_PATCH(@d32, '{"a":1}') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_patch' at position 32 +SELECT JSON_MERGE_PATCH('{"a":1}', @d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_merge_patch' at position 32 +SELECT JSON_CONTAINS(@d32, '1') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains' at position 32 +SELECT JSON_CONTAINS_PATH(@d32, 'one', '$') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains_path' at position 32 +SELECT JSON_OVERLAPS(@d32, '[1]') AS over_limit; +over_limit +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_EQUALS(@d32, @d32) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_equals' at position 32 +SELECT JSON_SEARCH(@d32, 'one', '1') AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_search' at position 32 +SELECT JSON_EXISTS(@d32, '$') AS over_limit; +over_limit +1 +# +# 3. Path depth is limited independently of document depth. +# +SET @p31 = CONCAT('$', REPEAT('[0]', 31)); +SET @p32 = CONCAT('$', REPEAT('[0]', 32)); +SET @p33 = CONCAT('$', REPEAT('[0]', 33)); +SELECT JSON_EXTRACT(@d31, @p31) AS p31; +p31 +1 +SELECT JSON_EXTRACT(@d31, @p32) AS p32; +p32 +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_extract' at position 96 +SELECT JSON_EXTRACT(@d31, @p33) AS p33; +p33 +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_extract' at position 96 +SELECT JSON_EXISTS(@d31, @p32) AS p32; +p32 +NULL +SELECT JSON_SET('[1]', @p33, 2) AS p33; +p33 +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_set' at position 96 +# +# 4. Composed depth. A document that is acceptable on its own can be +# placed inside another document, and the sum is what the result +# has to carry. These record what happens today when the sum goes +# past the limit. +# +# a JSON-typed argument is embedded as a document, not as a string +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d29))) AS d30; +d30 +31 +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d30))) AS d31; +d31 +32 +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS d32; +d32 +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT(@d31))) AS still_valid; +still_valid +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT LENGTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS produced_bytes; +produced_bytes +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +# the same through an object constructor +SELECT JSON_DEPTH(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS d32; +d32 +32 +SELECT JSON_VALID(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS still_valid; +still_valid +1 +# two levels of constructor +SELECT JSON_VALID(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d30)))) AS still_valid; +still_valid +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d29)))) AS d31; +d31 +32 +# a mutator splicing a deep value into a shallow document +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d29))) AS d30; +d30 +31 +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d30))) AS d31; +d31 +32 +SELECT JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 43 +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31))) AS still_valid; +still_valid +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 43 +SELECT JSON_INSERT('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 43 +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 35 +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_COMPACT(@d31)) AS over_limit; +over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_insert' at position 32 +# a mutator splicing a deep value into an already deep document +SET @d15 = CONCAT(REPEAT('[', 15), '1', REPEAT(']', 15)); +SET @d16 = CONCAT(REPEAT('[', 16), '1', REPEAT(']', 16)); +SET @p15 = CONCAT('$', REPEAT('[0]', 15)); +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS composed; +composed +31 +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS still_valid; +still_valid +1 +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS composed; +composed +32 +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS still_valid; +still_valid +1 +# merging two deep documents +SELECT JSON_DEPTH(JSON_MERGE(@d30, @d30)) AS merged; +merged +31 +SELECT JSON_DEPTH(JSON_MERGE(JSON_ARRAY(JSON_COMPACT(@d29)), '[1]')) AS merged; +merged +31 +# merging two objects, which nest rather than concatenate, so the +# composed depth is the sum and can cross the limit +SELECT JSON_DEPTH(JSON_MERGE(@o30, @o30)) AS composed_at_limit; +composed_at_limit +32 +SELECT JSON_VALID(JSON_MERGE(@o30, @o30)) AS still_valid; +still_valid +1 +SELECT JSON_MERGE(@o31, @o31) AS composed_over_limit; +composed_over_limit +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 156 +SELECT JSON_VALID(JSON_MERGE(@o31, @o31)) AS still_valid; +still_valid +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 156 +# this one stays within the limit and succeeds +SELECT JSON_MERGE_PATCH(CONCAT('{"a":', @d30, '}'), '{"b":1}') AS at_limit_ok; +at_limit_ok +{"a": [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]], "b": 1} +# a deep value spliced as a plain string is escaped, not embedded +SELECT JSON_DEPTH(JSON_ARRAY(CONCAT(@d31, ''))) AS as_string; +as_string +2 +SELECT JSON_VALID(JSON_ARRAY(CONCAT(@d31, ''))) AS still_valid; +still_valid +1 +# +# 5. Depth reached through repeated mutation, one level at a time. +# +SELECT JSON_DEPTH(JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), +'$.a[0]', JSON_ARRAY()), +'$.a[0][0]', JSON_ARRAY())) AS d; +d +4 +SELECT JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), +'$.a[0]', JSON_ARRAY()), +'$.a[0][0]', JSON_ARRAY()) AS v; +v +{"a": [[[]]]} diff --git a/mysql-test/main/func_json_depth.test b/mysql-test/main/func_json_depth.test new file mode 100644 index 0000000000000..9af7f0cebbb3c --- /dev/null +++ b/mysql-test/main/func_json_depth.test @@ -0,0 +1,155 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: nesting depth. +--echo # +--echo # The scanner keeps a fixed stack, so documents nested past its size +--echo # are rejected. This test records the exact depth at which each +--echo # function changes its answer, and what happens when a document that +--echo # is acceptable on its own is spliced into another document deep +--echo # enough that the combination is not. +--echo # + +--echo # +--echo # 1. Where the limit sits, for arrays and for objects. +--echo # + +SET @d29 = CONCAT(REPEAT('[', 29), '1', REPEAT(']', 29)); +SET @d30 = CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @d31 = CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d32 = CONCAT(REPEAT('[', 32), '1', REPEAT(']', 32)); +SET @d33 = CONCAT(REPEAT('[', 33), '1', REPEAT(']', 33)); + +SELECT JSON_VALID(@d29) AS v29, JSON_VALID(@d30) AS v30, JSON_VALID(@d31) AS v31; +SELECT JSON_VALID(@d32) AS v32; +SELECT JSON_VALID(@d33) AS v33; +SELECT JSON_DEPTH(@d29) AS d29, JSON_DEPTH(@d30) AS d30, JSON_DEPTH(@d31) AS d31; +SELECT JSON_DEPTH(@d32) AS d32; + +--echo # the same depths built from objects +SET @o30 = CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @o31 = CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SET @o32 = CONCAT(REPEAT('{"a":', 32), '1', REPEAT('}', 32)); +SELECT JSON_VALID(@o30) AS v30, JSON_VALID(@o31) AS v31, JSON_VALID(@o32) AS v32; +SELECT JSON_DEPTH(@o31) AS d31; +SELECT JSON_DEPTH(@o32) AS d32; + +--echo # alternating containers reach the same limit +SET @m30 = CONCAT(REPEAT('[{"a":', 15), '1', REPEAT('}]', 15)); +SET @m32 = CONCAT(REPEAT('[{"a":', 16), '1', REPEAT('}]', 16)); +SELECT JSON_VALID(@m30) AS v30, JSON_VALID(@m32) AS v32; +SELECT JSON_DEPTH(@m30) AS d30; + +--echo # +--echo # 2. Every function at, and one past, the limit. +--echo # + +SELECT JSON_VALID(@d31) AS ok, JSON_VALID(@d32) AS over_limit; +SELECT JSON_DEPTH(@d31) AS ok; +SELECT JSON_DEPTH(@d32) AS over_limit; +SELECT JSON_TYPE(@d31) AS ok; +SELECT JSON_TYPE(@d32) AS over_limit; +SELECT JSON_LENGTH(@d31) AS ok; +SELECT JSON_LENGTH(@d32) AS over_limit; +SELECT JSON_COMPACT(@d32) AS over_limit; +SELECT JSON_LOOSE(@d32) AS over_limit; +SELECT JSON_DETAILED(@d32) AS over_limit; +SELECT JSON_NORMALIZE(@d32) AS over_limit; +SELECT JSON_EXTRACT(@d32, '$') AS over_limit; +SELECT JSON_QUERY(@d32, '$') AS over_limit; +SELECT JSON_VALUE(@d32, '$[0]') AS over_limit; +SELECT JSON_KEYS(@o32) AS over_limit; +SELECT JSON_SET(@d32, '$[0]', 2) AS over_limit; +SELECT JSON_INSERT(@d32, '$.a', 2) AS over_limit; +SELECT JSON_REPLACE(@d32, '$[0]', 2) AS over_limit; +SELECT JSON_REMOVE(@d32, '$[0]') AS over_limit; +SELECT JSON_ARRAY_APPEND(@d32, '$', 2) AS over_limit; +SELECT JSON_ARRAY_INSERT(@d32, '$[0]', 2) AS over_limit; +SELECT JSON_MERGE(@d32, '[1]') AS over_limit; +SELECT JSON_MERGE('[1]', @d32) AS over_limit; +SELECT JSON_MERGE_PATCH(@d32, '{"a":1}') AS over_limit; +SELECT JSON_MERGE_PATCH('{"a":1}', @d32) AS over_limit; +SELECT JSON_CONTAINS(@d32, '1') AS over_limit; +SELECT JSON_CONTAINS_PATH(@d32, 'one', '$') AS over_limit; +SELECT JSON_OVERLAPS(@d32, '[1]') AS over_limit; +SELECT JSON_EQUALS(@d32, @d32) AS over_limit; +SELECT JSON_SEARCH(@d32, 'one', '1') AS over_limit; +SELECT JSON_EXISTS(@d32, '$') AS over_limit; + +--echo # +--echo # 3. Path depth is limited independently of document depth. +--echo # + +SET @p31 = CONCAT('$', REPEAT('[0]', 31)); +SET @p32 = CONCAT('$', REPEAT('[0]', 32)); +SET @p33 = CONCAT('$', REPEAT('[0]', 33)); +SELECT JSON_EXTRACT(@d31, @p31) AS p31; +SELECT JSON_EXTRACT(@d31, @p32) AS p32; +SELECT JSON_EXTRACT(@d31, @p33) AS p33; +SELECT JSON_EXISTS(@d31, @p32) AS p32; +SELECT JSON_SET('[1]', @p33, 2) AS p33; + +--echo # +--echo # 4. Composed depth. A document that is acceptable on its own can be +--echo # placed inside another document, and the sum is what the result +--echo # has to carry. These record what happens today when the sum goes +--echo # past the limit. +--echo # + +--echo # a JSON-typed argument is embedded as a document, not as a string +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d29))) AS d30; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d30))) AS d31; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS d32; +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT(@d31))) AS still_valid; +SELECT LENGTH(JSON_ARRAY(JSON_COMPACT(@d31))) AS produced_bytes; +--echo # the same through an object constructor +SELECT JSON_DEPTH(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS d32; +SELECT JSON_VALID(JSON_OBJECT('a', JSON_COMPACT(@d30))) AS still_valid; +--echo # two levels of constructor +SELECT JSON_VALID(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d30)))) AS still_valid; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_COMPACT(@d29)))) AS d31; + +--echo # a mutator splicing a deep value into a shallow document +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d29))) AS d30; +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d30))) AS d31; +SELECT JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', JSON_COMPACT(@d31))) AS still_valid; +SELECT JSON_INSERT('{"a":1}', '$.b', JSON_COMPACT(@d31)) AS over_limit; +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_COMPACT(@d31)) AS over_limit; +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_COMPACT(@d31)) AS over_limit; + +--echo # a mutator splicing a deep value into an already deep document +SET @d15 = CONCAT(REPEAT('[', 15), '1', REPEAT(']', 15)); +SET @d16 = CONCAT(REPEAT('[', 16), '1', REPEAT(']', 16)); +SET @p15 = CONCAT('$', REPEAT('[0]', 15)); +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS composed; +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d15))) AS still_valid; +SELECT JSON_DEPTH(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS composed; +SELECT JSON_VALID(JSON_SET(@d16, @p15, JSON_COMPACT(@d16))) AS still_valid; + +--echo # merging two deep documents +SELECT JSON_DEPTH(JSON_MERGE(@d30, @d30)) AS merged; +SELECT JSON_DEPTH(JSON_MERGE(JSON_ARRAY(JSON_COMPACT(@d29)), '[1]')) AS merged; +--echo # merging two objects, which nest rather than concatenate, so the +--echo # composed depth is the sum and can cross the limit +SELECT JSON_DEPTH(JSON_MERGE(@o30, @o30)) AS composed_at_limit; +SELECT JSON_VALID(JSON_MERGE(@o30, @o30)) AS still_valid; +SELECT JSON_MERGE(@o31, @o31) AS composed_over_limit; +SELECT JSON_VALID(JSON_MERGE(@o31, @o31)) AS still_valid; +--echo # this one stays within the limit and succeeds +SELECT JSON_MERGE_PATCH(CONCAT('{"a":', @d30, '}'), '{"b":1}') AS at_limit_ok; + +--echo # a deep value spliced as a plain string is escaped, not embedded +SELECT JSON_DEPTH(JSON_ARRAY(CONCAT(@d31, ''))) AS as_string; +SELECT JSON_VALID(JSON_ARRAY(CONCAT(@d31, ''))) AS still_valid; + +--echo # +--echo # 5. Depth reached through repeated mutation, one level at a time. +--echo # + +SELECT JSON_DEPTH(JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), + '$.a[0]', JSON_ARRAY()), + '$.a[0][0]', JSON_ARRAY())) AS d; +SELECT JSON_SET(JSON_SET(JSON_SET('{}', '$.a', JSON_ARRAY()), + '$.a[0]', JSON_ARRAY()), + '$.a[0][0]', JSON_ARRAY()) AS v; diff --git a/mysql-test/main/func_json_embed.result b/mysql-test/main/func_json_embed.result new file mode 100644 index 0000000000000..03e1d7db78d82 --- /dev/null +++ b/mysql-test/main/func_json_embed.result @@ -0,0 +1,744 @@ +# +# Splicing a value that is typed as JSON into a document. +# +# A value whose type is JSON is spliced into the result as it +# stands rather than quoted as a string. This is not only about +# objects and arrays: a JSON number, string or boolean is spliced +# the same way, which is what keeps a stored "a string" from coming +# back out with its quotes escaped. +# +# Being typed as JSON and being JSON are different things: the type +# comes from a check constraint, and a constraint can be switched +# off for a statement, leaving bytes behind that it would have +# rejected. Nothing between the column and the result reads those +# bytes. +# +# The constructors are the interesting half, because they have no +# closing pass over what they built. Whatever they splice is what +# the caller gets. +# +# A value that does not parse is spliced all the same, byte for +# byte. What the constructor has then built is not a document, so +# it returns NULL instead of it, and a warning names the argument +# that was responsible. A warning and not a note, because it is +# the only word the statement gets about why it was given NULL, +# and because a statement running under strict mode then stops at +# the value rather than finishing without one. A function that +# reads its whole answer back has a second complaint to make about +# the same value, and leaves this one a note. +# +# Non ASCII characters below are written as the bytes that encode +# them, so this file needs no character set of its own: +# C3 A9 is U+00E9, one byte in latin1 (E9) +# E6 BC A2 is U+6F22, which latin1 has no room for +# +SET NAMES utf8mb4; +# +# 1. Which values are spliced as documents and which are quoted. +# +CREATE TABLE t1 (id INT, j JSON, txt LONGTEXT); +INSERT INTO t1 VALUES (1, '{"a": 1}', '{"a": 1}'); +SELECT JSON_ARRAY(j) AS typed, JSON_ARRAY(txt) AS untyped FROM t1; +typed untyped +[{"a": 1}] ["{\"a\": 1}"] +# a function that returns JSON is typed the same way +SELECT JSON_ARRAY(JSON_COMPACT('[1, 2]')) AS from_function; +from_function +[[1, 2]] +SELECT JSON_ARRAY(CONCAT(j, '')) AS through_concat FROM t1; +through_concat +["{\"a\": 1}"] +# +# 2. A column holding bytes its constraint would have rejected. +# +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,', NULL); +INSERT INTO t1 VALUES (11, 'not json at all', NULL); +INSERT INTO t1 VALUES (12, '{"a":1} and then some', NULL); +SET SESSION check_constraint_checks = ON; +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document +FROM t1 WHERE id >= 10 ORDER BY id; +id built_a_document +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 9 +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS built_a_document +FROM t1 WHERE id >= 10 ORDER BY id; +id built_a_document +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 9 +# more than one element, so the value is spliced in the middle of a +# document rather than as the whole of it +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 9 +SELECT id, JSON_OBJECT('before', 1, 'k', j, 'after', 2) AS v +FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 4 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 4 to function 'json_object' at position 9 +# the aggregates, which splice a value per row +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS built_a_document +FROM t1 WHERE id >= 10; +built_a_document +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 9 +SELECT JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +SELECT JSON_VALID(JSON_OBJECTAGG(id, j)) AS built_a_document +FROM t1 WHERE id >= 10; +built_a_document +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_objectagg' at position 9 +# a group mixing a good row with a bad one +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id = 1 OR id = 10; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +# the mutators splice the same way, but read back what they built +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v +FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 21 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_set' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 14 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_set' at position 9 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 22 +SELECT id, JSON_ARRAY_APPEND('[0]', '$', j) AS v +FROM t1 WHERE id >= 10 ORDER BY id; +id v +10 NULL +11 NULL +12 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_array_append' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 12 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_array_append' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 5 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_array_append' at position 9 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 13 +# +# 3. Bytes that are not a document and would not have looked wrong +# afterwards. +# +# An empty value leaves nothing behind at all, so what is built +# round it parses and reports one element fewer than it was given. +# A document like that gives no sign of what happened to it, and +# is exactly what is not returned: the row is refused, and the +# warning says which argument it was. +# +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (20, '', NULL); +INSERT INTO t1 VALUES (21, ' ', NULL); +INSERT INTO t1 VALUES (22, '// nothing here', NULL); +SET SESSION check_constraint_checks = ON; +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS looks_fine, +JSON_LENGTH(JSON_ARRAY(j)) AS elements +FROM t1 WHERE id >= 20 ORDER BY id; +id v looks_fine elements +20 NULL NULL NULL +21 NULL NULL NULL +22 NULL NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +SELECT id, JSON_ARRAY(1, j, 2) AS v, JSON_VALID(JSON_ARRAY(1, j, 2)) AS looks_fine, +JSON_LENGTH(JSON_ARRAY(1, j, 2)) AS elements +FROM t1 WHERE id >= 20 ORDER BY id; +id v looks_fine elements +20 NULL NULL NULL +21 NULL NULL NULL +22 NULL NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_array' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_array' at position 1 +SELECT id, JSON_OBJECT('k', j) AS v, JSON_VALID(JSON_OBJECT('k', j)) AS looks_fine +FROM t1 WHERE id >= 20 ORDER BY id; +id v looks_fine +20 NULL NULL +21 NULL NULL +22 NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_object' at position 1 +SELECT JSON_ARRAYAGG(j) AS v, JSON_VALID(JSON_ARRAYAGG(j)) AS looks_fine +FROM t1 WHERE id >= 20; +v looks_fine +NULL NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +# +# 4. Nesting that only goes over the limit once the value is wrapped. +# +# The scanner takes 31 nested structures. A document of exactly 31 +# is a document; put it inside one more and it is not, so the +# constructor that wrapped it has nothing to return. A mutator +# reads its whole answer back and finds out there, as it always +# did. +# +CREATE TABLE t2 (id INT, j JSON); +INSERT INTO t2 VALUES (31, CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))); +INSERT INTO t2 VALUES (30, CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30))); +SELECT id, JSON_VALID(j) AS on_its_own, JSON_DEPTH(j) AS deep FROM t2 ORDER BY id; +id on_its_own deep +30 1 31 +31 1 32 +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS wrapped_once, +LENGTH(JSON_ARRAY(j)) AS bytes FROM t2 ORDER BY id; +id wrapped_once bytes +30 1 63 +31 NULL NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS wrapped_once FROM t2 ORDER BY id; +id wrapped_once +30 1 +31 NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_object' at position 63 +# without a table in the way +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT( +CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))))) AS wrapped_once; +wrapped_once +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +# a mutator splices at the depth the path reaches, and reads back +SELECT id, JSON_VALID(JSON_SET('{}', '$.k', j)) AS spliced FROM t2 ORDER BY id; +id spliced +30 1 +31 NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 36 +# +# 5. A document written in a different character set from the one it +# is being spliced into. +# +# The constructors agree a character set across their arguments +# first, so the value arrives already converted. The mutators take +# the character set of the document they are editing and convert +# nothing, so the value arrives as it was written. +# +CREATE TABLE t3 (id INT, j JSON); +INSERT INTO t3 VALUES (1, _utf8mb4 0x5B22C3A9225D); +INSERT INTO t3 VALUES (2, _utf8mb4 0x5B22E6BCA2225D); +SELECT id, CHARSET(j) AS column_cs, HEX(j) AS bytes FROM t3 ORDER BY id; +id column_cs bytes +1 utf8mb4 5B22C3A9225D +2 utf8mb4 5B22E6BCA2225D +# spliced by a constructor whose other argument is latin1 +SELECT id, CHARSET(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS result_cs, +HEX(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS bytes +FROM t3 ORDER BY id; +id result_cs bytes +1 utf8mb4 5B2278222C205B22C3A9225D5D +2 utf8mb4 5B2278222C205B22E6BCA2225D5D +# spliced by a mutator into a latin1 document +SELECT id, CHARSET(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) +AS result_cs, +HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) AS bytes +FROM t3 ORDER BY id; +id result_cs bytes +1 latin1 7B2261223A20312C202262223A205B22E9225D7D +2 latin1 7B2261223A20312C202262223A205B22E6BCA2225D7D +SELECT id, HEX(JSON_ARRAY_APPEND(CONVERT(_utf8mb4'[1]' USING latin1), '$', j)) +AS bytes +FROM t3 ORDER BY id; +id bytes +1 5B312C205B22E9225D5D +2 5B312C205B22E6BCA2225D5D +# read the spliced value back out of the document it landed in +SELECT id, HEX(JSON_EXTRACT( +JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j), +'$.b')) AS read_back +FROM t3 ORDER BY id; +id read_back +1 5B22E9225D +2 5B22E6BCA2225D +# +# 6. Values that are what they say they are, which must not move. +# +# Note the scalars among them. A column holding a JSON string is +# spliced with one set of quotes, not two, which is the difference +# between being spliced as JSON and being quoted as a string. +# +CREATE TABLE t4 (id INT, j JSON); +INSERT INTO t4 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t4 VALUES (2, '[1, 2, 3]'); +INSERT INTO t4 VALUES (3, '"a string"'); +INSERT INTO t4 VALUES (4, '42'); +INSERT INTO t4 VALUES (5, 'null'); +INSERT INTO t4 VALUES (6, 'true'); +INSERT INTO t4 VALUES (7, '{"compact":[1,2],"no":"spaces"}'); +SELECT id, JSON_ARRAY(j) AS v FROM t4 ORDER BY id; +id v +1 [{"a": 1, "b": [1, 2]}] +2 [[1, 2, 3]] +3 ["a string"] +4 [42] +5 [null] +6 [true] +7 [{"compact":[1,2],"no":"spaces"}] +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 ORDER BY id; +id v +1 {"k": {"a": 1, "b": [1, 2]}} +2 {"k": [1, 2, 3]} +3 {"k": "a string"} +4 {"k": 42} +5 {"k": null} +6 {"k": true} +7 {"k": {"compact":[1,2],"no":"spaces"}} +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t4 ORDER BY id; +id v +1 [1, {"a": 1, "b": [1, 2]}, "z"] +2 [1, [1, 2, 3], "z"] +3 [1, "a string", "z"] +4 [1, 42, "z"] +5 [1, null, "z"] +6 [1, true, "z"] +7 [1, {"compact":[1,2],"no":"spaces"}, "z"] +SELECT JSON_ARRAYAGG(j) AS v FROM t4; +v +[{"a": 1, "b": [1, 2]},[1, 2, 3],"a string",42,null,true,{"compact":[1,2],"no":"spaces"}] +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4; +v +{"1":{"a": 1, "b": [1, 2]}, "2":[1, 2, 3], "3":"a string", "4":42, "5":null, "6":true, "7":{"compact":[1,2],"no":"spaces"}} +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v FROM t4 ORDER BY id; +id v +1 {"x": 0, "k": {"a": 1, "b": [1, 2]}} +2 {"x": 0, "k": [1, 2, 3]} +3 {"x": 0, "k": "a string"} +4 {"x": 0, "k": 42} +5 {"x": 0, "k": null} +6 {"x": 0, "k": true} +7 {"x": 0, "k": {"compact": [1, 2], "no": "spaces"}} +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document FROM t4 ORDER BY id; +id built_a_document +1 1 +2 1 +3 1 +4 1 +5 1 +6 1 +7 1 +# a NULL value is spliced as the JSON null, not as the document +INSERT INTO t4 VALUES (8, NULL); +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id = 8; +id v +8 [null] +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 WHERE id = 8; +id v +8 {"k": null} +# +# 7. Whether what the constructors build is already in the loose +# form, or would still be changed by reformatting it. +# +# A value that is spliced is copied exactly as it was written, so +# the spacing of the result is the spacing the value arrived with. +# A constructor is therefore only in the loose form when the +# documents given to it are. +# +CREATE TABLE t5 (id INT, j JSON); +INSERT INTO t5 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t5 VALUES (2, '{"a":1,"b":[1,2]}'); +INSERT INTO t5 VALUES (3, '[ 1 ,2, 3 ]'); +# with nothing spliced, and with a document spliced in +SELECT JSON_ARRAY(1, 'a', TRUE, NULL) AS built, +JSON_ARRAY(1, 'a', TRUE, NULL) = JSON_LOOSE(JSON_ARRAY(1, 'a', TRUE, NULL)) +AS already_loose; +built already_loose +[1, "a", true, null] 1 +SELECT JSON_OBJECT('a', 1, 'b', 'x') AS built, +JSON_OBJECT('a', 1, 'b', 'x') = JSON_LOOSE(JSON_OBJECT('a', 1, 'b', 'x')) +AS already_loose; +built already_loose +{"a": 1, "b": "x"} 1 +SELECT id, JSON_ARRAY(j) AS built, +JSON_ARRAY(j) = JSON_LOOSE(JSON_ARRAY(j)) AS already_loose +FROM t5 ORDER BY id; +id built already_loose +1 [{"a": 1, "b": [1, 2]}] 1 +2 [{"a":1,"b":[1,2]}] 0 +3 [[ 1 ,2, 3 ]] 0 +SELECT id, JSON_OBJECT('k', j) AS built, +JSON_OBJECT('k', j) = JSON_LOOSE(JSON_OBJECT('k', j)) AS already_loose +FROM t5 ORDER BY id; +id built already_loose +1 {"k": {"a": 1, "b": [1, 2]}} 1 +2 {"k": {"a":1,"b":[1,2]}} 0 +3 {"k": [ 1 ,2, 3 ]} 0 +# the aggregates write their own separators, which are not the +# loose ones. A group of one never writes a separator at all, so +# both a single row and several are recorded here. +INSERT INTO t5 VALUES (4, '{"c": 3}'); +SELECT JSON_ARRAYAGG(j) AS built, +JSON_ARRAYAGG(j) = JSON_LOOSE(JSON_ARRAYAGG(j)) AS already_loose +FROM t5 WHERE id = 1; +built already_loose +[{"a": 1, "b": [1, 2]}] 1 +SELECT JSON_ARRAYAGG(j ORDER BY id) AS built, +JSON_ARRAYAGG(j ORDER BY id) = JSON_LOOSE(JSON_ARRAYAGG(j ORDER BY id)) +AS already_loose +FROM t5 WHERE id IN (1, 4); +built already_loose +[{"a": 1, "b": [1, 2]},{"c": 3}] 0 +SELECT JSON_OBJECTAGG(id, j) AS built, +JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose +FROM t5 WHERE id = 1; +built already_loose +{"1":{"a": 1, "b": [1, 2]}} 0 +SELECT JSON_OBJECTAGG(id, j) AS built, +JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose +FROM t5 WHERE id IN (1, 4); +built already_loose +{"1":{"a": 1, "b": [1, 2]}, "4":{"c": 3}} 0 +# a mutator reads back everything it built, so its spacing is its +# own regardless of what was spliced +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS built, +JSON_SET('{"x": 0}', '$.k', j) = JSON_LOOSE(JSON_SET('{"x": 0}', '$.k', j)) +AS already_loose +FROM t5 ORDER BY id; +id built already_loose +1 {"x": 0, "k": {"a": 1, "b": [1, 2]}} 1 +2 {"x": 0, "k": {"a": 1, "b": [1, 2]}} 1 +3 {"x": 0, "k": [1, 2, 3]} 1 +4 {"x": 0, "k": {"c": 3}} 1 +# +# 8. What the aggregates do with a row they cannot write out. +# +# An aggregate writes the separator before it asks for the element, +# so a row that yields nothing leaves a separator with nothing on +# one side of it. There are two reasons a row can yield nothing +# and they are not reported alike. A character that cannot be +# written into a document at all is a property of the data, and +# gets a note naming the row; a buffer that would not grow is not, +# and has already stopped the statement by the time it is noticed. +# Either way what is left is a group with a gap in it, which is not +# a document and is not returned. +# +# A KEY that cannot be written is neither of those. It goes in as +# the empty key with its pair finished round it, so the object is a +# document whose keys are not the ones that were asked for, and the +# note is the whole of what is said about it. +# +# cp1250 is used here because it admits every byte, so a column can +# hold one that stands for no character at all. +# +CREATE TABLE t6 (id INT, c VARCHAR(16) CHARACTER SET cp1250); +INSERT INTO t6 VALUES (1, X'81'), (2, 'ok'), (3, X'8881'); +SELECT id, HEX(c) AS stored FROM t6 ORDER BY id; +id stored +1 81 +2 6F6B +3 8881 +SELECT JSON_ARRAYAGG(c) AS v FROM t6; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +SELECT JSON_OBJECTAGG(id, c) AS v FROM t6; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_objectagg' at position 0 +Note 4035 Broken JSON string in argument 2 to function 'json_objectagg' at position 0 +# and where it is the key that cannot be written +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +v +{"":1, "ok":2, "":3} +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_objectagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_objectagg' at position 0 +# a group with no such row is untouched +SELECT JSON_ARRAYAGG(c) AS v FROM t6 WHERE id = 2; +v +["ok"] +# the same rows through the tree path, which writes the elements out +# when the group is asked for rather than as they arrive +SELECT JSON_ARRAYAGG(c ORDER BY id) AS v FROM t6; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +SELECT JSON_ARRAYAGG(DISTINCT c) AS v FROM t6; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +Note 4035 Broken JSON string in argument 1 to function 'json_arrayagg' at position 0 +# a copy of the aggregate, which is made without fixing it again. +# The values here are valid, so nothing should be said about them. +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v +FROM t6 GROUP BY id WITH ROLLUP; +id v +1 005B005B002200780022005D005D +2 005B005B002200780022005D005D +3 005B005B002200780022005D005D +NULL 005B005B002200780022005D002C005B002200780022005D002C005B002200780022005D005D +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v +FROM t6 GROUP BY id; +id v +1 005B005B002200780022005D005D +2 005B005B002200780022005D005D +3 005B005B002200780022005D005D +# nesting composed by the aggregate itself, which puts each value +# inside one array or object of its own +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, +JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj +FROM t2 WHERE id = 31; +wrapped_once wrapped_once_obj +NULL NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_objectagg' at position 63 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_arrayagg' at position 63 +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, +JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj +FROM t2 WHERE id = 30; +wrapped_once wrapped_once_obj +1 1 +# +# 9. The same character in a key of a constructor. +# +# A key is written by the same routine that writes one for the +# aggregate, and a character no document can carry stops it the +# same way. The constructors read nothing back, so a key and a +# value alike throw the whole result away here; all that is left +# of either is what is said about it, and both sides say which +# argument it was. +# +# Said at note level, where the same constructor refusing a value +# that did not parse says it at warning level. Both answer NULL. +# The difference is that refusing a character is what a released +# server did and refusing an unparsed value is not, so only the +# second is raised to where strict mode can act on it. +# +# the value side, which has always been named +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 0 +SELECT JSON_OBJECT('k', c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_object' at position 0 +# the key side, in the first pair and in a later one +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_object' at position 0 +SELECT JSON_OBJECT('a', 1, c, 2) AS v FROM t6 WHERE id = 3; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 3 to function 'json_object' at position 0 +# a key and a value of the same row, which stops at the key +SELECT JSON_OBJECT(c, c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_object' at position 0 +# a row whose key can be written says nothing +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 2; +v +{"ok": 1} +# +# 10. The same character through JSON_QUOTE, which writes a value out +# with the routine the constructors use but is not building a +# document of its own. +# +# It has always answered NULL for a character it cannot write and +# still does. What it never did was say so, and it was the only +# caller of that routine which did not. +# +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_quote' at position 0 +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 3; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_quote' at position 0 +# a row it can write says nothing +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 2; +v +"ok" +# the same value reaching a constructor instead, which has always +# named the argument +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; +v +NULL +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_array' at position 0 +# +# 11. What a constructor or an aggregate returns when what it put +# together is not a document. +# +# These four have no closing pass over what they built, so whatever +# went in wrong is in the answer, and every way that can happen is +# above: a value typed as JSON that does not parse, one that parses +# as nothing at all, a nesting that only goes over the limit once +# the value is wrapped, and a character no document can carry, which +# leaves an aggregate's separator with nothing beside it. +# +# None of the four is a document and none of them is returned. The +# complaint saying what was wrong is raised where it always was; +# what follows it is NULL rather than the bytes. +# +# Two cases that look like these and are not stay as they were. A +# KEY that could not be written leaves the pair around it whole, so +# the object is a document whose keys are not the ones asked for. +# A group cut to fit its length limit is cut back to the last whole +# element, so the brackets go round a document as well. +# +# a value typed as JSON that does not parse, and one that parses as +# nothing at all +SELECT id, JSON_ARRAY(j) IS NULL AS refused, +JSON_OBJECT('k', j) IS NULL AS refused_obj +FROM t1 WHERE id >= 10 ORDER BY id; +id refused refused_obj +10 1 1 +11 1 1 +12 1 1 +20 1 1 +21 1 1 +22 1 1 +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, +JSON_OBJECTAGG(id, j) IS NULL AS refused_obj +FROM t1 WHERE id >= 10; +refused refused_obj +1 1 +# a nesting that only goes over the limit once the value is wrapped +SELECT id, JSON_ARRAY(j) IS NULL AS refused, +JSON_OBJECT('k', j) IS NULL AS refused_obj +FROM t2 ORDER BY id; +id refused refused_obj +30 0 0 +31 1 1 +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, +JSON_OBJECTAGG(id, j) IS NULL AS refused_obj +FROM t2 WHERE id = 31; +refused refused_obj +1 1 +# a character no document can carry, in a value +SELECT JSON_ARRAYAGG(c) IS NULL AS refused, +JSON_OBJECTAGG(id, c) IS NULL AS refused_obj FROM t6; +refused refused_obj +1 1 +# the same character in a KEY, which leaves a document behind +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +v +{"":1, "ok":2, "":3} +# a group cut to fit its length limit +SET SESSION group_concat_max_len= 28; +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +v +[{"a": 1, "b": [1, 2]}] +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; +v +{"1":{"a": 1, "b": [1, 2]}} +SET SESSION group_concat_max_len= DEFAULT; +# and what IS a document comes back as it always did +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id <= 2 ORDER BY id; +id v +1 [{"a": 1, "b": [1, 2]}] +2 [[1, 2, 3]] +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +v +[{"a": 1, "b": [1, 2]},[1, 2, 3]] +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; +v +{"1":{"a": 1, "b": [1, 2]}, "2":[1, 2, 3]} +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/main/func_json_embed.test b/mysql-test/main/func_json_embed.test new file mode 100644 index 0000000000000..aac6a13741a94 --- /dev/null +++ b/mysql-test/main/func_json_embed.test @@ -0,0 +1,419 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Splicing a value that is typed as JSON into a document. +--echo # +--echo # A value whose type is JSON is spliced into the result as it +--echo # stands rather than quoted as a string. This is not only about +--echo # objects and arrays: a JSON number, string or boolean is spliced +--echo # the same way, which is what keeps a stored "a string" from coming +--echo # back out with its quotes escaped. +--echo # +--echo # Being typed as JSON and being JSON are different things: the type +--echo # comes from a check constraint, and a constraint can be switched +--echo # off for a statement, leaving bytes behind that it would have +--echo # rejected. Nothing between the column and the result reads those +--echo # bytes. +--echo # +--echo # The constructors are the interesting half, because they have no +--echo # closing pass over what they built. Whatever they splice is what +--echo # the caller gets. +--echo # +--echo # A value that does not parse is spliced all the same, byte for +--echo # byte. What the constructor has then built is not a document, so +--echo # it returns NULL instead of it, and a warning names the argument +--echo # that was responsible. A warning and not a note, because it is +--echo # the only word the statement gets about why it was given NULL, +--echo # and because a statement running under strict mode then stops at +--echo # the value rather than finishing without one. A function that +--echo # reads its whole answer back has a second complaint to make about +--echo # the same value, and leaves this one a note. +--echo # +--echo # Non ASCII characters below are written as the bytes that encode +--echo # them, so this file needs no character set of its own: +--echo # C3 A9 is U+00E9, one byte in latin1 (E9) +--echo # E6 BC A2 is U+6F22, which latin1 has no room for +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Which values are spliced as documents and which are quoted. +--echo # + +CREATE TABLE t1 (id INT, j JSON, txt LONGTEXT); +INSERT INTO t1 VALUES (1, '{"a": 1}', '{"a": 1}'); +SELECT JSON_ARRAY(j) AS typed, JSON_ARRAY(txt) AS untyped FROM t1; +--echo # a function that returns JSON is typed the same way +SELECT JSON_ARRAY(JSON_COMPACT('[1, 2]')) AS from_function; +SELECT JSON_ARRAY(CONCAT(j, '')) AS through_concat FROM t1; + +--echo # +--echo # 2. A column holding bytes its constraint would have rejected. +--echo # + +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (10, '{"a":1,', NULL); +INSERT INTO t1 VALUES (11, 'not json at all', NULL); +INSERT INTO t1 VALUES (12, '{"a":1} and then some', NULL); +SET SESSION check_constraint_checks = ON; + +SELECT id, JSON_ARRAY(j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS built_a_document + FROM t1 WHERE id >= 10 ORDER BY id; + +--echo # more than one element, so the value is spliced in the middle of a +--echo # document rather than as the whole of it +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_OBJECT('before', 1, 'k', j, 'after', 2) AS v + FROM t1 WHERE id >= 10 ORDER BY id; + +--echo # the aggregates, which splice a value per row +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id >= 10; +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS built_a_document + FROM t1 WHERE id >= 10; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t1 WHERE id >= 10; +SELECT JSON_VALID(JSON_OBJECTAGG(id, j)) AS built_a_document + FROM t1 WHERE id >= 10; +--echo # a group mixing a good row with a bad one +SELECT JSON_ARRAYAGG(j) AS v FROM t1 WHERE id = 1 OR id = 10; + +--echo # the mutators splice the same way, but read back what they built +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT id, JSON_ARRAY_APPEND('[0]', '$', j) AS v + FROM t1 WHERE id >= 10 ORDER BY id; + +--echo # +--echo # 3. Bytes that are not a document and would not have looked wrong +--echo # afterwards. +--echo # +--echo # An empty value leaves nothing behind at all, so what is built +--echo # round it parses and reports one element fewer than it was given. +--echo # A document like that gives no sign of what happened to it, and +--echo # is exactly what is not returned: the row is refused, and the +--echo # warning says which argument it was. +--echo # + +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES (20, '', NULL); +INSERT INTO t1 VALUES (21, ' ', NULL); +INSERT INTO t1 VALUES (22, '// nothing here', NULL); +SET SESSION check_constraint_checks = ON; + +SELECT id, JSON_ARRAY(j) AS v, JSON_VALID(JSON_ARRAY(j)) AS looks_fine, + JSON_LENGTH(JSON_ARRAY(j)) AS elements + FROM t1 WHERE id >= 20 ORDER BY id; +SELECT id, JSON_ARRAY(1, j, 2) AS v, JSON_VALID(JSON_ARRAY(1, j, 2)) AS looks_fine, + JSON_LENGTH(JSON_ARRAY(1, j, 2)) AS elements + FROM t1 WHERE id >= 20 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v, JSON_VALID(JSON_OBJECT('k', j)) AS looks_fine + FROM t1 WHERE id >= 20 ORDER BY id; +SELECT JSON_ARRAYAGG(j) AS v, JSON_VALID(JSON_ARRAYAGG(j)) AS looks_fine + FROM t1 WHERE id >= 20; + +--echo # +--echo # 4. Nesting that only goes over the limit once the value is wrapped. +--echo # +--echo # The scanner takes 31 nested structures. A document of exactly 31 +--echo # is a document; put it inside one more and it is not, so the +--echo # constructor that wrapped it has nothing to return. A mutator +--echo # reads its whole answer back and finds out there, as it always +--echo # did. +--echo # + +CREATE TABLE t2 (id INT, j JSON); +INSERT INTO t2 VALUES (31, CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))); +INSERT INTO t2 VALUES (30, CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30))); +SELECT id, JSON_VALID(j) AS on_its_own, JSON_DEPTH(j) AS deep FROM t2 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS wrapped_once, + LENGTH(JSON_ARRAY(j)) AS bytes FROM t2 ORDER BY id; +SELECT id, JSON_VALID(JSON_OBJECT('k', j)) AS wrapped_once FROM t2 ORDER BY id; +--echo # without a table in the way +SELECT JSON_VALID(JSON_ARRAY(JSON_COMPACT( + CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31))))) AS wrapped_once; +--echo # a mutator splices at the depth the path reaches, and reads back +SELECT id, JSON_VALID(JSON_SET('{}', '$.k', j)) AS spliced FROM t2 ORDER BY id; + +--echo # +--echo # 5. A document written in a different character set from the one it +--echo # is being spliced into. +--echo # +--echo # The constructors agree a character set across their arguments +--echo # first, so the value arrives already converted. The mutators take +--echo # the character set of the document they are editing and convert +--echo # nothing, so the value arrives as it was written. +--echo # + +CREATE TABLE t3 (id INT, j JSON); +INSERT INTO t3 VALUES (1, _utf8mb4 0x5B22C3A9225D); +INSERT INTO t3 VALUES (2, _utf8mb4 0x5B22E6BCA2225D); +SELECT id, CHARSET(j) AS column_cs, HEX(j) AS bytes FROM t3 ORDER BY id; + +--echo # spliced by a constructor whose other argument is latin1 +SELECT id, CHARSET(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS result_cs, + HEX(JSON_ARRAY(CONVERT('x' USING latin1), j)) AS bytes + FROM t3 ORDER BY id; + +--echo # spliced by a mutator into a latin1 document +SELECT id, CHARSET(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) + AS result_cs, + HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j)) AS bytes + FROM t3 ORDER BY id; +SELECT id, HEX(JSON_ARRAY_APPEND(CONVERT(_utf8mb4'[1]' USING latin1), '$', j)) + AS bytes + FROM t3 ORDER BY id; +--echo # read the spliced value back out of the document it landed in +SELECT id, HEX(JSON_EXTRACT( + JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), '$.b', j), + '$.b')) AS read_back + FROM t3 ORDER BY id; + +--echo # +--echo # 6. Values that are what they say they are, which must not move. +--echo # +--echo # Note the scalars among them. A column holding a JSON string is +--echo # spliced with one set of quotes, not two, which is the difference +--echo # between being spliced as JSON and being quoted as a string. +--echo # + +CREATE TABLE t4 (id INT, j JSON); +INSERT INTO t4 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t4 VALUES (2, '[1, 2, 3]'); +INSERT INTO t4 VALUES (3, '"a string"'); +INSERT INTO t4 VALUES (4, '42'); +INSERT INTO t4 VALUES (5, 'null'); +INSERT INTO t4 VALUES (6, 'true'); +INSERT INTO t4 VALUES (7, '{"compact":[1,2],"no":"spaces"}'); +SELECT id, JSON_ARRAY(j) AS v FROM t4 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 ORDER BY id; +SELECT id, JSON_ARRAY(1, j, 'z') AS v FROM t4 ORDER BY id; +SELECT JSON_ARRAYAGG(j) AS v FROM t4; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4; +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS v FROM t4 ORDER BY id; +SELECT id, JSON_VALID(JSON_ARRAY(j)) AS built_a_document FROM t4 ORDER BY id; +--echo # a NULL value is spliced as the JSON null, not as the document +INSERT INTO t4 VALUES (8, NULL); +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id = 8; +SELECT id, JSON_OBJECT('k', j) AS v FROM t4 WHERE id = 8; + +--echo # +--echo # 7. Whether what the constructors build is already in the loose +--echo # form, or would still be changed by reformatting it. +--echo # +--echo # A value that is spliced is copied exactly as it was written, so +--echo # the spacing of the result is the spacing the value arrived with. +--echo # A constructor is therefore only in the loose form when the +--echo # documents given to it are. +--echo # + +CREATE TABLE t5 (id INT, j JSON); +INSERT INTO t5 VALUES (1, '{"a": 1, "b": [1, 2]}'); +INSERT INTO t5 VALUES (2, '{"a":1,"b":[1,2]}'); +INSERT INTO t5 VALUES (3, '[ 1 ,2, 3 ]'); + +--echo # with nothing spliced, and with a document spliced in +# A view keeps its body as printed text, and a boolean literal prints as the +# number it equals, so read through one this array would hold a 1 rather +# than a true. +--disable_view_protocol +SELECT JSON_ARRAY(1, 'a', TRUE, NULL) AS built, + JSON_ARRAY(1, 'a', TRUE, NULL) = JSON_LOOSE(JSON_ARRAY(1, 'a', TRUE, NULL)) + AS already_loose; +--enable_view_protocol +SELECT JSON_OBJECT('a', 1, 'b', 'x') AS built, + JSON_OBJECT('a', 1, 'b', 'x') = JSON_LOOSE(JSON_OBJECT('a', 1, 'b', 'x')) + AS already_loose; +SELECT id, JSON_ARRAY(j) AS built, + JSON_ARRAY(j) = JSON_LOOSE(JSON_ARRAY(j)) AS already_loose + FROM t5 ORDER BY id; +SELECT id, JSON_OBJECT('k', j) AS built, + JSON_OBJECT('k', j) = JSON_LOOSE(JSON_OBJECT('k', j)) AS already_loose + FROM t5 ORDER BY id; + +--echo # the aggregates write their own separators, which are not the +--echo # loose ones. A group of one never writes a separator at all, so +--echo # both a single row and several are recorded here. +INSERT INTO t5 VALUES (4, '{"c": 3}'); +SELECT JSON_ARRAYAGG(j) AS built, + JSON_ARRAYAGG(j) = JSON_LOOSE(JSON_ARRAYAGG(j)) AS already_loose + FROM t5 WHERE id = 1; +SELECT JSON_ARRAYAGG(j ORDER BY id) AS built, + JSON_ARRAYAGG(j ORDER BY id) = JSON_LOOSE(JSON_ARRAYAGG(j ORDER BY id)) + AS already_loose + FROM t5 WHERE id IN (1, 4); +SELECT JSON_OBJECTAGG(id, j) AS built, + JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose + FROM t5 WHERE id = 1; +SELECT JSON_OBJECTAGG(id, j) AS built, + JSON_OBJECTAGG(id, j) = JSON_LOOSE(JSON_OBJECTAGG(id, j)) AS already_loose + FROM t5 WHERE id IN (1, 4); + +--echo # a mutator reads back everything it built, so its spacing is its +--echo # own regardless of what was spliced +SELECT id, JSON_SET('{"x": 0}', '$.k', j) AS built, + JSON_SET('{"x": 0}', '$.k', j) = JSON_LOOSE(JSON_SET('{"x": 0}', '$.k', j)) + AS already_loose + FROM t5 ORDER BY id; + +--echo # +--echo # 8. What the aggregates do with a row they cannot write out. +--echo # +--echo # An aggregate writes the separator before it asks for the element, +--echo # so a row that yields nothing leaves a separator with nothing on +--echo # one side of it. There are two reasons a row can yield nothing +--echo # and they are not reported alike. A character that cannot be +--echo # written into a document at all is a property of the data, and +--echo # gets a note naming the row; a buffer that would not grow is not, +--echo # and has already stopped the statement by the time it is noticed. +--echo # Either way what is left is a group with a gap in it, which is not +--echo # a document and is not returned. +--echo # +--echo # A KEY that cannot be written is neither of those. It goes in as +--echo # the empty key with its pair finished round it, so the object is a +--echo # document whose keys are not the ones that were asked for, and the +--echo # note is the whole of what is said about it. +--echo # +--echo # cp1250 is used here because it admits every byte, so a column can +--echo # hold one that stands for no character at all. +--echo # + +CREATE TABLE t6 (id INT, c VARCHAR(16) CHARACTER SET cp1250); +INSERT INTO t6 VALUES (1, X'81'), (2, 'ok'), (3, X'8881'); +SELECT id, HEX(c) AS stored FROM t6 ORDER BY id; +SELECT JSON_ARRAYAGG(c) AS v FROM t6; +SELECT JSON_OBJECTAGG(id, c) AS v FROM t6; +--echo # and where it is the key that cannot be written +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +--echo # a group with no such row is untouched +SELECT JSON_ARRAYAGG(c) AS v FROM t6 WHERE id = 2; + +--echo # the same rows through the tree path, which writes the elements out +--echo # when the group is asked for rather than as they arrive +SELECT JSON_ARRAYAGG(c ORDER BY id) AS v FROM t6; +SELECT JSON_ARRAYAGG(DISTINCT c) AS v FROM t6; + +--echo # a copy of the aggregate, which is made without fixing it again. +--echo # The values here are valid, so nothing should be said about them. +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v + FROM t6 GROUP BY id WITH ROLLUP; +SELECT id, HEX(JSON_ARRAYAGG(JSON_ARRAY(CONVERT('x' USING ucs2)))) AS v + FROM t6 GROUP BY id; + +--echo # nesting composed by the aggregate itself, which puts each value +--echo # inside one array or object of its own +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, + JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj + FROM t2 WHERE id = 31; +SELECT JSON_VALID(JSON_ARRAYAGG(j)) AS wrapped_once, + JSON_VALID(JSON_OBJECTAGG(id, j)) AS wrapped_once_obj + FROM t2 WHERE id = 30; + +--echo # +--echo # 9. The same character in a key of a constructor. +--echo # +--echo # A key is written by the same routine that writes one for the +--echo # aggregate, and a character no document can carry stops it the +--echo # same way. The constructors read nothing back, so a key and a +--echo # value alike throw the whole result away here; all that is left +--echo # of either is what is said about it, and both sides say which +--echo # argument it was. +--echo # +--echo # Said at note level, where the same constructor refusing a value +--echo # that did not parse says it at warning level. Both answer NULL. +--echo # The difference is that refusing a character is what a released +--echo # server did and refusing an unparsed value is not, so only the +--echo # second is raised to where strict mode can act on it. +--echo # + +--echo # the value side, which has always been named +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; +SELECT JSON_OBJECT('k', c) AS v FROM t6 WHERE id = 1; +--echo # the key side, in the first pair and in a later one +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 1; +SELECT JSON_OBJECT('a', 1, c, 2) AS v FROM t6 WHERE id = 3; +--echo # a key and a value of the same row, which stops at the key +SELECT JSON_OBJECT(c, c) AS v FROM t6 WHERE id = 1; +--echo # a row whose key can be written says nothing +SELECT JSON_OBJECT(c, 1) AS v FROM t6 WHERE id = 2; + +--echo # +--echo # 10. The same character through JSON_QUOTE, which writes a value out +--echo # with the routine the constructors use but is not building a +--echo # document of its own. +--echo # +--echo # It has always answered NULL for a character it cannot write and +--echo # still does. What it never did was say so, and it was the only +--echo # caller of that routine which did not. +--echo # + +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 1; +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 3; +--echo # a row it can write says nothing +SELECT JSON_QUOTE(c) AS v FROM t6 WHERE id = 2; +--echo # the same value reaching a constructor instead, which has always +--echo # named the argument +SELECT JSON_ARRAY(c) AS v FROM t6 WHERE id = 1; + +--echo # +--echo # 11. What a constructor or an aggregate returns when what it put +--echo # together is not a document. +--echo # +--echo # These four have no closing pass over what they built, so whatever +--echo # went in wrong is in the answer, and every way that can happen is +--echo # above: a value typed as JSON that does not parse, one that parses +--echo # as nothing at all, a nesting that only goes over the limit once +--echo # the value is wrapped, and a character no document can carry, which +--echo # leaves an aggregate's separator with nothing beside it. +--echo # +--echo # None of the four is a document and none of them is returned. The +--echo # complaint saying what was wrong is raised where it always was; +--echo # what follows it is NULL rather than the bytes. +--echo # +--echo # Two cases that look like these and are not stay as they were. A +--echo # KEY that could not be written leaves the pair around it whole, so +--echo # the object is a document whose keys are not the ones asked for. +--echo # A group cut to fit its length limit is cut back to the last whole +--echo # element, so the brackets go round a document as well. +--echo # + +--disable_warnings +--echo # a value typed as JSON that does not parse, and one that parses as +--echo # nothing at all +SELECT id, JSON_ARRAY(j) IS NULL AS refused, + JSON_OBJECT('k', j) IS NULL AS refused_obj + FROM t1 WHERE id >= 10 ORDER BY id; +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, + JSON_OBJECTAGG(id, j) IS NULL AS refused_obj + FROM t1 WHERE id >= 10; +--echo # a nesting that only goes over the limit once the value is wrapped +SELECT id, JSON_ARRAY(j) IS NULL AS refused, + JSON_OBJECT('k', j) IS NULL AS refused_obj + FROM t2 ORDER BY id; +SELECT JSON_ARRAYAGG(j) IS NULL AS refused, + JSON_OBJECTAGG(id, j) IS NULL AS refused_obj + FROM t2 WHERE id = 31; +--echo # a character no document can carry, in a value +SELECT JSON_ARRAYAGG(c) IS NULL AS refused, + JSON_OBJECTAGG(id, c) IS NULL AS refused_obj FROM t6; +--echo # the same character in a KEY, which leaves a document behind +SELECT JSON_OBJECTAGG(c, id) AS v FROM t6; +--enable_warnings + +--echo # a group cut to fit its length limit +SET SESSION group_concat_max_len= 28; +--disable_warnings +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; +--enable_warnings +SET SESSION group_concat_max_len= DEFAULT; + +--echo # and what IS a document comes back as it always did +SELECT id, JSON_ARRAY(j) AS v FROM t4 WHERE id <= 2 ORDER BY id; +SELECT JSON_ARRAYAGG(j ORDER BY id) AS v FROM t4 WHERE id <= 2; +SELECT JSON_OBJECTAGG(id, j) AS v FROM t4 WHERE id <= 2; + +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/main/func_json_emit_killed.result b/mysql-test/main/func_json_emit_killed.result new file mode 100644 index 0000000000000..c46cef55f3795 --- /dev/null +++ b/mysql-test/main/func_json_emit_killed.result @@ -0,0 +1,160 @@ +SET @old_debug= @@debug_dbug; +SET debug_dbug='+d,json_kill_while_emitting'; +# +# 1. Values written out during the reading +# +# An object and an array, each with more than one member, so +# the walk takes several steps and the kill lands inside it. +# +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS obj_value; +ERROR 70100: Query execution was interrupted +SELECT JSON_EXTRACT('{"a": [1, 2, 3]}', '$.a') AS array_value; +ERROR 70100: Query execution was interrupted +SELECT JSON_EXTRACT('{"a": {"b": [1, 2], "c": {"d": 3}}}', '$.a') AS nested; +ERROR 70100: Query execution was interrupted +# +# Written the same way when more than one path is asked for, +# and when the value is reached through a wildcard. +# +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.a', '$.b') AS two_paths; +ERROR 70100: Query execution was interrupted +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.*') AS wildcard; +ERROR 70100: Query execution was interrupted +# +# 2. Values that are copied rather than written out +# +# A scalar is punctuated with nothing, so it is taken across as +# it stands and there is no walk to be stopped in. These are +# answered, and that is what says the arming above reaches the +# walk and only the walk. +# +SELECT JSON_EXTRACT('{"a": 1}', '$.a') AS scalar_value; +scalar_value +1 +SELECT JSON_EXTRACT('{"a": "text"}', '$.a') AS scalar_string; +scalar_string +"text" +SELECT JSON_EXTRACT('{"a": null, "b": true}', '$.a', '$.b') AS two_scalars; +two_scalars +[null, true] +# Nothing matched, so nothing was written at all. +SELECT JSON_EXTRACT('{"a": 1}', '$.zz') AS no_match; +no_match +NULL +# +# 3. Answers that are read back after being composed +# +# These build an answer by copying the parts of the document +# they are not changing and putting the new part between them, +# which is no walk of their own. Then they read the whole of +# it back to settle its formatting, and that reading writes as it +# goes: it is the same walk section 1 is stopped in, reached +# from the other end. So the kill lands in it. +# +# The reading that finds the place to edit lets go of a killed +# query as well, a token at a time, and is covered by +# func_json_notembedded. +# +SELECT JSON_INSERT('{"a": {"b": 1, "c": 2}}', '$.d', 4) AS insert_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_REMOVE('{"a": {"b": 1, "c": 2}}', '$.a.b') AS remove_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_SET('{"a": [1, 2, 3]}', '$.a[0]', 9) AS set_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_REPLACE('{"a": [1, 2, 3]}', '$.a[0]', 9) AS replace_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_ARRAY_APPEND('{"a": [1, 2]}', '$.a', 3) AS append_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_ARRAY_INSERT('{"a": [1, 2]}', '$.a[0]', 9) AS array_insert_done; +ERROR 70100: Query execution was interrupted +SELECT JSON_MERGE_PATCH('{"a": {"b": 1}}', '{"a": {"c": 2}}') AS merge_patch_done; +ERROR 70100: Query execution was interrupted +# +# The same with a document answering is_valid, so that the +# answer is passed as it is composed and there is no +# reading back at all. Answered - which is what says the +# ones above they match are stopped in the reading back and +# nowhere else, the composing being the same either way. +# +SELECT JSON_INSERT(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.d', 4) +AS insert_is_valid; +insert_is_valid +{"a": {"b": 1, "c": 2}, "d": 4} +SELECT JSON_REMOVE(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.a.b') +AS remove_is_valid; +remove_is_valid +{"a": {"c": 2}} +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', JSON_OBJECT('b', 1)), +JSON_OBJECT('a', JSON_OBJECT('c', 2))) +AS merge_patch_is_valid; +merge_patch_is_valid +{"a": {"b": 1, "c": 2}} +# +# 4. A value written out during the reading +# +# A value going into a document is written out again where the +# document it joins is is_valid and is_nice and the value is +# not, that being the one way the value's own formatting can +# still be put right. That writing IS a walk, so the kill +# below is raised inside it. +# +# What these show is that a kill arriving in the middle of that +# walk is carried out - the walk gives up, the bytes fall back +# to what they were, and the statement ends the way a killed +# statement ends. They do NOT show which reading noticed it: +# the flag stays raised once it is raised, and a statement that +# ignored it here would still end this way at the next step. +# How soon it is noticed is the part that cannot be written +# down as an expected result. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, '{"b":1,"c":2}'), (2, '[1,2,3]'); +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS value_walked FROM t1; +ERROR 70100: Query execution was interrupted +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', j) AS value_walked_set FROM t1; +ERROR 70100: Query execution was interrupted +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', j) AS value_walked_append +FROM t1; +ERROR 70100: Query execution was interrupted +# +# Three more, sorted by whether anything walks in them. +# +# A value already formatted the way the answer needs is taken +# across without being read at all, so there is no walk. It +# has to come from something that does not walk either - a +# document BUILT out of values, not one cut out of another, +# since cutting one out is itself a walk and would be what the +# kill landed in. +# +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_OBJECT('x', 1, 'y', 2)) +AS value_taken; +value_taken +{"a": 1, "d": {"x": 1, "y": 2}} +# +# A document nobody attested sends the whole answer through +# the reading back, and a value going into that is left as it +# arrived - so there is no walk of the value's own here. The +# reading back is one, though, so this is stopped where the +# statements in section 3 are. +# +SELECT JSON_INSERT('{"a": 1}', '$.d', j) AS value_left FROM t1; +ERROR 70100: Query execution was interrupted +# +# A scalar carries no punctuation, so it is copied like the +# scalars in section 2 and attests to the same reason. +# +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '11'), (2, '"text"'); +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS scalar_value FROM t2; +scalar_value +{"a": 1, "d": 11} +{"a": 1, "d": "text"} +DROP TABLE t1; +DROP TABLE t2; +# +# 5. A kill is let go of once the statement it was aimed at is over +# +SET debug_dbug= @old_debug; +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS after_disarming; +after_disarming +{"b": 1, "c": 2} diff --git a/mysql-test/main/func_json_emit_killed.test b/mysql-test/main/func_json_emit_killed.test new file mode 100644 index 0000000000000..e1c602348b425 --- /dev/null +++ b/mysql-test/main/func_json_emit_killed.test @@ -0,0 +1,182 @@ +# +# A killed query has to be let go of while a result is being written, +# not only before the writing starts. +# +# The existing coverage arms the kill at the top of each function, so +# what it reaches is the first reading step of all - a check that was +# already there and would still be there whatever the writing did. The +# writing itself has never been reached under a kill. These arm one +# partway through the walk that does the writing, so that the walk is +# where the killed statement is standing when it has to let go. +# +# What that shows is that a kill arriving in the middle of the writing +# is carried out at all: the walk gives up, nothing is left half +# written, and the statement ends the way a killed statement ends. It +# does NOT show which reading noticed it. Once the flag is raised it +# stays raised, so a walk that ignored it would still end this way at +# the next step, and no expected result can tell the two apart. How +# soon a kill is noticed is a matter of how long the statement runs, +# which is not something a result file can hold. +# +# What is sorted below is not which function was asked but whether a +# walk happens at all. A statement that writes a document out, that +# writes a value into one, or that reads its own answer back to settle +# its formatting, walks - and lets go. A statement that only copies +# bytes from one place to another has nothing to be stopped in, and +# answers. Which side a function falls on is decided by what it was +# handed, so the same function stands on both. +# + +--source include/have_debug.inc + +SET @old_debug= @@debug_dbug; +SET debug_dbug='+d,json_kill_while_emitting'; + +--echo # +--echo # 1. Values written out during the reading +--echo # +--echo # An object and an array, each with more than one member, so +--echo # the walk takes several steps and the kill lands inside it. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS obj_value; +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": [1, 2, 3]}', '$.a') AS array_value; +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": {"b": [1, 2], "c": {"d": 3}}}', '$.a') AS nested; + +--echo # +--echo # Written the same way when more than one path is asked for, +--echo # and when the value is reached through a wildcard. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.a', '$.b') AS two_paths; +--error ER_QUERY_INTERRUPTED +SELECT JSON_EXTRACT('{"a": [1, 2], "b": [3, 4]}', '$.*') AS wildcard; + +--echo # +--echo # 2. Values that are copied rather than written out +--echo # +--echo # A scalar is punctuated with nothing, so it is taken across as +--echo # it stands and there is no walk to be stopped in. These are +--echo # answered, and that is what says the arming above reaches the +--echo # walk and only the walk. +--echo # +SELECT JSON_EXTRACT('{"a": 1}', '$.a') AS scalar_value; +SELECT JSON_EXTRACT('{"a": "text"}', '$.a') AS scalar_string; +SELECT JSON_EXTRACT('{"a": null, "b": true}', '$.a', '$.b') AS two_scalars; +--echo # Nothing matched, so nothing was written at all. +SELECT JSON_EXTRACT('{"a": 1}', '$.zz') AS no_match; + +--echo # +--echo # 3. Answers that are read back after being composed +--echo # +--echo # These build an answer by copying the parts of the document +--echo # they are not changing and putting the new part between them, +--echo # which is no walk of their own. Then they read the whole of +--echo # it back to settle its formatting, and that reading writes as it +--echo # goes: it is the same walk section 1 is stopped in, reached +--echo # from the other end. So the kill lands in it. +--echo # +--echo # The reading that finds the place to edit lets go of a killed +--echo # query as well, a token at a time, and is covered by +--echo # func_json_notembedded. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_INSERT('{"a": {"b": 1, "c": 2}}', '$.d', 4) AS insert_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_REMOVE('{"a": {"b": 1, "c": 2}}', '$.a.b') AS remove_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_SET('{"a": [1, 2, 3]}', '$.a[0]', 9) AS set_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_REPLACE('{"a": [1, 2, 3]}', '$.a[0]', 9) AS replace_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_ARRAY_APPEND('{"a": [1, 2]}', '$.a', 3) AS append_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_ARRAY_INSERT('{"a": [1, 2]}', '$.a[0]', 9) AS array_insert_done; +--error ER_QUERY_INTERRUPTED +SELECT JSON_MERGE_PATCH('{"a": {"b": 1}}', '{"a": {"c": 2}}') AS merge_patch_done; + +--echo # +--echo # The same with a document answering is_valid, so that the +--echo # answer is passed as it is composed and there is no +--echo # reading back at all. Answered - which is what says the +--echo # ones above they match are stopped in the reading back and +--echo # nowhere else, the composing being the same either way. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.d', 4) + AS insert_is_valid; +SELECT JSON_REMOVE(JSON_OBJECT('a', JSON_OBJECT('b', 1, 'c', 2)), '$.a.b') + AS remove_is_valid; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', JSON_OBJECT('b', 1)), + JSON_OBJECT('a', JSON_OBJECT('c', 2))) + AS merge_patch_is_valid; + +--echo # +--echo # 4. A value written out during the reading +--echo # +--echo # A value going into a document is written out again where the +--echo # document it joins is is_valid and is_nice and the value is +--echo # not, that being the one way the value's own formatting can +--echo # still be put right. That writing IS a walk, so the kill +--echo # below is raised inside it. +--echo # +--echo # What these show is that a kill arriving in the middle of that +--echo # walk is carried out - the walk gives up, the bytes fall back +--echo # to what they were, and the statement ends the way a killed +--echo # statement ends. They do NOT show which reading noticed it: +--echo # the flag stays raised once it is raised, and a statement that +--echo # ignored it here would still end this way at the next step. +--echo # How soon it is noticed is the part that cannot be written +--echo # down as an expected result. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t1 VALUES (1, '{"b":1,"c":2}'), (2, '[1,2,3]'); + +--error ER_QUERY_INTERRUPTED +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS value_walked FROM t1; +--error ER_QUERY_INTERRUPTED +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', j) AS value_walked_set FROM t1; +--error ER_QUERY_INTERRUPTED +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', j) AS value_walked_append + FROM t1; + +--echo # +--echo # Three more, sorted by whether anything walks in them. +--echo # +--echo # A value already formatted the way the answer needs is taken +--echo # across without being read at all, so there is no walk. It +--echo # has to come from something that does not walk either - a +--echo # document BUILT out of values, not one cut out of another, +--echo # since cutting one out is itself a walk and would be what the +--echo # kill landed in. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_OBJECT('x', 1, 'y', 2)) + AS value_taken; + +--echo # +--echo # A document nobody attested sends the whole answer through +--echo # the reading back, and a value going into that is left as it +--echo # arrived - so there is no walk of the value's own here. The +--echo # reading back is one, though, so this is stopped where the +--echo # statements in section 3 are. +--echo # +--error ER_QUERY_INTERRUPTED +SELECT JSON_INSERT('{"a": 1}', '$.d', j) AS value_left FROM t1; + +--echo # +--echo # A scalar carries no punctuation, so it is copied like the +--echo # scalars in section 2 and attests to the same reason. +--echo # +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '11'), (2, '"text"'); +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', j) AS scalar_value FROM t2; + +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # 5. A kill is let go of once the statement it was aimed at is over +--echo # +SET debug_dbug= @old_debug; +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS after_disarming; diff --git a/mysql-test/main/func_json_escape_pair.result b/mysql-test/main/func_json_escape_pair.result new file mode 100644 index 0000000000000..1da874618d306 --- /dev/null +++ b/mysql-test/main/func_json_escape_pair.result @@ -0,0 +1,62 @@ +CREATE TABLE t1 (n VARCHAR(20) CHARACTER SET latin1, +t VARCHAR(20) CHARACTER SET utf8mb3, +w VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES ('{}', '{}', _utf8mb4 0xEFBFBF), +('{}', '{}', _utf8mb4 0xF0908080), +('{}', '{}', _utf8mb4 0xF09F9880), +('{}', '{}', _utf8mb4 0xF48FBFBF); +# +# The last character of the first plane takes one escape, and the +# three past it take two. +# +SELECT JSON_SET(n, '$.b', w) AS written, +JSON_VALID(JSON_SET(n, '$.b', w)) AS valid FROM t1; +written valid +{"b": "\uFFFF"} 1 +{"b": "\uD800\uDC00"} 1 +{"b": "\uD83D\uDE00"} 1 +{"b": "\uDBFF\uDFFF"} 1 +# +# The same into a document written in the other set that stops at +# the first plane. That set carries the first of the four as it +# stands, so that one is not escaped at all - read here as bytes, +# to keep the connection's own set out of the answer. +# +SELECT HEX(JSON_SET(t, '$.b', w)) AS bytes, +JSON_VALID(JSON_SET(t, '$.b', w)) AS valid FROM t1; +bytes valid +7B2262223A2022EFBFBF227D 1 +7B2262223A20225C75443830305C7544433030227D 1 +7B2262223A20225C75443833445C7544453030227D 1 +7B2262223A20225C75444246465C7544464646227D 1 +# +# The four other functions that write a value into a document they +# were handed answer the same way. +# +SELECT JSON_INSERT(n, '$.b', w) AS inserted, +JSON_REPLACE(JSON_SET(n, '$.b', 1), '$.b', w) AS replaced FROM t1; +inserted replaced +{"b": "\uFFFF"} {"b": "\uFFFF"} +{"b": "\uD800\uDC00"} {"b": "\uD800\uDC00"} +{"b": "\uD83D\uDE00"} {"b": "\uD83D\uDE00"} +{"b": "\uDBFF\uDFFF"} {"b": "\uDBFF\uDFFF"} +SELECT JSON_ARRAY_APPEND(CONCAT('[', 1, ']'), '$', w) AS appended, +JSON_ARRAY_INSERT(CONCAT('[', 1, ']'), '$[0]', w) AS array_inserted +FROM t1; +appended array_inserted +[1, "\uFFFF"] ["\uFFFF", 1] +[1, "\uD800\uDC00"] ["\uD800\uDC00", 1] +[1, "\uD83D\uDE00"] ["\uD83D\uDE00", 1] +[1, "\uDBFF\uDFFF"] ["\uDBFF\uDFFF", 1] +# +# What the writing put there, read as bytes. The escape is ASCII +# whatever the document is written in, so the two units stand out +# as two of them. +# +SELECT HEX(JSON_SET(n, '$.b', w)) AS bytes FROM t1; +bytes +7B2262223A20225C7546464646227D +7B2262223A20225C75443830305C7544433030227D +7B2262223A20225C75443833445C7544453030227D +7B2262223A20225C75444246465C7544464646227D +DROP TABLE t1; diff --git a/mysql-test/main/func_json_escape_pair.test b/mysql-test/main/func_json_escape_pair.test new file mode 100644 index 0000000000000..29d5f01d0caf1 --- /dev/null +++ b/mysql-test/main/func_json_escape_pair.test @@ -0,0 +1,56 @@ +# +# Writing out a character the document cannot carry. +# +# A character outside the first plane is two UTF-16 units, and JSON +# spells it as two escapes, one unit apiece. Anything shorter is not a +# document: a reader takes the first escape, looks for the second, and +# stops where it is not. +# +# Only a document whose character set cannot carry the character is +# written this way, so the functions that meet it are the ones that take +# their character set from the document they were handed and aggregate +# nothing with it. +# + +CREATE TABLE t1 (n VARCHAR(20) CHARACTER SET latin1, + t VARCHAR(20) CHARACTER SET utf8mb3, + w VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES ('{}', '{}', _utf8mb4 0xEFBFBF), + ('{}', '{}', _utf8mb4 0xF0908080), + ('{}', '{}', _utf8mb4 0xF09F9880), + ('{}', '{}', _utf8mb4 0xF48FBFBF); + +--echo # +--echo # The last character of the first plane takes one escape, and the +--echo # three past it take two. +--echo # +SELECT JSON_SET(n, '$.b', w) AS written, + JSON_VALID(JSON_SET(n, '$.b', w)) AS valid FROM t1; + +--echo # +--echo # The same into a document written in the other set that stops at +--echo # the first plane. That set carries the first of the four as it +--echo # stands, so that one is not escaped at all - read here as bytes, +--echo # to keep the connection's own set out of the answer. +--echo # +SELECT HEX(JSON_SET(t, '$.b', w)) AS bytes, + JSON_VALID(JSON_SET(t, '$.b', w)) AS valid FROM t1; + +--echo # +--echo # The four other functions that write a value into a document they +--echo # were handed answer the same way. +--echo # +SELECT JSON_INSERT(n, '$.b', w) AS inserted, + JSON_REPLACE(JSON_SET(n, '$.b', 1), '$.b', w) AS replaced FROM t1; +SELECT JSON_ARRAY_APPEND(CONCAT('[', 1, ']'), '$', w) AS appended, + JSON_ARRAY_INSERT(CONCAT('[', 1, ']'), '$[0]', w) AS array_inserted + FROM t1; + +--echo # +--echo # What the writing put there, read as bytes. The escape is ASCII +--echo # whatever the document is written in, so the two units stand out +--echo # as two of them. +--echo # +SELECT HEX(JSON_SET(n, '$.b', w)) AS bytes FROM t1; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_format.result b/mysql-test/main/func_json_format.result new file mode 100644 index 0000000000000..ed0d9da85f3d1 --- /dev/null +++ b/mysql-test/main/func_json_format.result @@ -0,0 +1,508 @@ +# +# Behavioral baseline: exact output formatting of the JSON functions. +# +# Every JSON producer is exercised with input in several formattings +# (compact, already-spaced, whitespace-heavy) so that the exact bytes +# each function emits are recorded. HEX() is used next to the plain +# output wherever whitespace decides the answer, because leading and +# trailing space is easy to lose in a result file. +# +# +# 1. Mutating functions. These reformat their document argument. +# +# --- JSON_SET --- +SELECT JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2], "c": {"d": "x"}} +SELECT HEX(JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D2C202263223A207B2264223A202278227D7D +SELECT JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2], "c": {"d": "x"}} +SELECT HEX(JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D2C202263223A207B2264223A202278227D7D +SELECT JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +SELECT HEX(JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D7D +# a path that matches nothing still reformats the document +SELECT JSON_SET('{"a":1,"b":[1,2]}', '$.zzz.deeper', 2) AS v; +v +{"a": 1, "b": [1, 2]} +# scalar document +SELECT JSON_SET('1', '$', 2) AS v; +v +2 +SELECT JSON_SET('"str"', '$.a', 2) AS v; +v +"str" +# --- JSON_INSERT --- +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2], "new": 2} +SELECT HEX(JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2)) AS h; +h +7B2261223A20312C202262223A205B312C20325D2C20226E6577223A20327D +SELECT JSON_INSERT('{"a": 1, "b": [1, 2]}', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2], "new": 2} +SELECT JSON_INSERT('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2], "new": 2} +# existing key is not replaced, document is still reformatted +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.a', 9) AS v; +v +{"a": 1, "b": [1, 2]} +# --- JSON_REPLACE --- +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +SELECT HEX(JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2)) AS h; +h +7B2261223A20322C202262223A205B312C20325D7D +SELECT JSON_REPLACE('{"a": 1, "b": [1, 2]}', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +SELECT JSON_REPLACE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +v +{"a": 2, "b": [1, 2]} +# missing key is not inserted, document is still reformatted +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.new', 2) AS v; +v +{"a": 1, "b": [1, 2]} +# --- JSON_REMOVE --- +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a') AS v; +v +{"b": [1, 2], "c": {"d": "x"}} +SELECT HEX(JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a')) AS h; +h +7B2262223A205B312C20325D2C202263223A207B2264223A202278227D7D +SELECT JSON_REMOVE('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a') AS v; +v +{"b": [1, 2], "c": {"d": "x"}} +SELECT JSON_REMOVE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a') AS v; +v +{"b": [1, 2]} +# removal at the cut boundaries: first, middle and last member +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.b') AS v; +v +{"a": 1, "c": {"d": "x"}} +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.c') AS v; +v +{"a": 1, "b": [1, 2]} +SELECT JSON_REMOVE('[1,2,3]', '$[0]') AS v; +v +[2, 3] +SELECT JSON_REMOVE('[1,2,3]', '$[1]') AS v; +v +[1, 3] +SELECT JSON_REMOVE('[1,2,3]', '$[2]') AS v; +v +[1, 2] +# removing the only member +SELECT JSON_REMOVE('{"a":1}', '$.a') AS v, HEX(JSON_REMOVE('{"a":1}', '$.a')) AS h; +v h +{} 7B7D +SELECT JSON_REMOVE('[1]', '$[0]') AS v, HEX(JSON_REMOVE('[1]', '$[0]')) AS h; +v h +[] 5B5D +# --- JSON_ARRAY_APPEND --- +SELECT JSON_ARRAY_APPEND('[1,2]', '$', 3) AS v; +v +[1, 2, 3] +SELECT HEX(JSON_ARRAY_APPEND('[1,2]', '$', 3)) AS h; +h +5B312C20322C20335D +SELECT JSON_ARRAY_APPEND('[1, 2]', '$', 3) AS v; +v +[1, 2, 3] +SELECT JSON_ARRAY_APPEND('{"a":1,"b":[1,2]}', '$.b', 3) AS v; +v +{"a": 1, "b": [1, 2, 3]} +# appending to a scalar wraps it into an array +SELECT JSON_ARRAY_APPEND('{"a":1}', '$.a', 2) AS v; +v +{"a": [1, 2]} +# --- JSON_ARRAY_INSERT --- +SELECT JSON_ARRAY_INSERT('[1,2]', '$[0]', 0) AS v; +v +[0, 1, 2] +SELECT HEX(JSON_ARRAY_INSERT('[1,2]', '$[0]', 0)) AS h; +h +5B302C20312C20325D +SELECT JSON_ARRAY_INSERT('[1, 2]', '$[1]', 0) AS v; +v +[1, 0, 2] +SELECT JSON_ARRAY_INSERT('[1,2]', '$[9]', 0) AS v; +v +[1, 2, 0] +SELECT JSON_ARRAY_INSERT('{"a":1,"b":[1,2]}', '$.b[0]', 0) AS v; +v +{"a": 1, "b": [0, 1, 2]} +# --- JSON_MERGE / JSON_MERGE_PRESERVE --- +SELECT JSON_MERGE('[1,2]', '[3,4]') AS v, HEX(JSON_MERGE('[1,2]', '[3,4]')) AS h; +v h +[1, 2, 3, 4] 5B312C20322C20332C20345D +SELECT JSON_MERGE('{"a":1}', '{"b":2}') AS v; +v +{"a": 1, "b": 2} +SELECT JSON_MERGE('{"a":1}', '{"a":2}') AS v; +v +{"a": [1, 2]} +SELECT JSON_MERGE_PRESERVE('{"a":1}', '{"a":2}') AS v; +v +{"a": [1, 2]} +SELECT JSON_MERGE('1', '2') AS v; +v +[1, 2] +# three-way merge +SELECT JSON_MERGE('[1]', '[2]', '[3]') AS v; +v +[1, 2, 3] +# --- JSON_MERGE_PATCH --- +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS v; +v +{"a": 1, "b": 2} +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '{"b":2}')) AS h; +h +7B2261223A20312C202262223A20327D +SELECT JSON_MERGE_PATCH('{"a":1}', '{"a":null}') AS v; +v +{} +SELECT JSON_MERGE_PATCH('{"a":{"b":1,"c":2}}', '{"a":{"b":9}}') AS v; +v +{"a": {"b": 9, "c": 2}} +# a non-object patch is adopted wholesale +SELECT JSON_MERGE_PATCH('{"a":1}', '[1, 2]') AS v; +v +[1, 2] +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '[1, 2]')) AS h; +h +5B312C20325D +SELECT JSON_MERGE_PATCH('{"a":1}', '"str"') AS v; +v +"str" +# the last argument decides when it is not an object +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}', '[9]') AS v; +v +[9] +# +# 2. Constructors. These copy their arguments and do NOT reformat. +# +# plain strings are quoted, JSON-typed arguments are embedded raw +SELECT JSON_ARRAY(1, 'str', NULL, TRUE) AS v; +v +[1, "str", null, true] +SELECT JSON_ARRAY() AS v, HEX(JSON_ARRAY()) AS h; +v h +[] 5B5D +SELECT JSON_OBJECT() AS v, HEX(JSON_OBJECT()) AS h; +v h +{} 7B7D +SELECT JSON_OBJECT('a', 1, 'b', 'str') AS v; +v +{"a": 1, "b": "str"} +SELECT HEX(JSON_OBJECT('a', 1, 'b', 'str')) AS h; +h +7B2261223A20312C202262223A2022737472227D +# embedded document arguments keep their own formatting +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1,"b": 2}')) AS v; +v +[{"a": 1,"b": 2}] +SELECT JSON_ARRAY(JSON_DETAILED('{"a":1}')) AS v; +v +[{"a":1}] +SELECT JSON_ARRAY(JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +v +[{"a":1,"b": 2}] +SELECT JSON_OBJECT('k', JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +v +{"k": {"a":1,"b": 2}} +# a nested constructor call +SELECT JSON_ARRAY(JSON_ARRAY(1,2), JSON_OBJECT('a',1)) AS v; +v +[[1, 2], {"a": 1}] +SELECT JSON_OBJECT('a', JSON_ARRAY(1,2)) AS v; +v +{"a": [1, 2]} +# a mutator result embedded in a constructor +SELECT JSON_ARRAY(JSON_SET('{"a":1,"b":2}', '$.a', 9)) AS v; +v +[{"a": 9, "b": 2}] +# +# 3. JSON_EXTRACT: formatting and validation depend on the evaluation +# context. In string context the document is scanned to the end and +# the result is reformatted; in numeric context the scan stops at the +# first match and nothing is reformatted. +# +SELECT JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a') AS v; +v +{"b": 1, "c": 2} +SELECT HEX(JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a')) AS h; +h +7B2262223A20312C202263223A20327D +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS v; +v +{"b": 1, "c": 2} +# several paths produce an array wrapper +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS v; +v +[1, 2] +SELECT HEX(JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b')) AS h; +h +5B312C20325D +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.*') AS v; +v +[1, 2] +# string context rejects trailing garbage after the document +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +# numeric context does not reach the trailing garbage +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') + 0 AS v; +v +1 +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') * 1.0 AS v; +v +1 +SELECT CAST(JSON_EXTRACT('{"a":1} trailing', '$.a') AS SIGNED) AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 9 +# the same asymmetry with a structurally broken tail +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') + 0 AS v; +v +1 +# numeric context on a non-scalar match +SELECT JSON_EXTRACT('{"a":[1,2]}', '$.a') + 0 AS v; +v +0 +# +# 4. JSON_QUERY and JSON_VALUE return a slice of the input and keep the +# input formatting. +# +SELECT JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a') AS v; +v +{"b":1, "c":2} +SELECT HEX(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a')) AS h; +h +7B2262223A312C2020202263223A327D +SELECT JSON_QUERY('{"a":[1, 2]}', '$.a') AS v; +v +[1, 2] +# JSON_QUERY only returns objects and arrays +SELECT JSON_QUERY('{"a":1}', '$.a') AS v; +v +NULL +SELECT JSON_VALUE('{"a":1}', '$.a') AS v; +v +1 +SELECT JSON_VALUE('{"a":"s t"}', '$.a') AS v, HEX(JSON_VALUE('{"a":"s t"}', '$.a')) AS h; +v h +s t 73202074 +# JSON_VALUE unquotes, so escapes are resolved +SELECT JSON_VALUE('{"a":"q\\"q"}', '$.a') AS v; +v +q"q +SELECT JSON_VALUE('{"a":[1,2]}', '$.a') AS v; +v +NULL +# a slice taken past trailing garbage +SELECT JSON_QUERY('{"a":{"b":1}} trailing', '$.a') AS v; +v +{"b":1} +SELECT JSON_VALUE('{"a":1} trailing', '$.a') AS v; +v +1 +# +# 5. The explicit formatters. These stay after the mutator epilogues +# are gone, so their exact bytes matter. +# +SELECT JSON_COMPACT('{"a": 1, "b": [1, 2]}') AS v; +v +{"a":1,"b":[1,2]} +SELECT HEX(JSON_COMPACT('{"a": 1, "b": [1, 2]}')) AS h; +h +7B2261223A312C2262223A5B312C325D7D +SELECT JSON_LOOSE('{"a":1,"b":[1,2]}') AS v; +v +{"a": 1, "b": [1, 2]} +SELECT HEX(JSON_LOOSE('{"a":1,"b":[1,2]}')) AS h; +h +7B2261223A20312C202262223A205B312C20325D7D +SELECT JSON_DETAILED('{"a":1,"b":[1,2]}') AS v; +v +{ + "a": 1, + "b": + [ + 1, + 2 + ] +} +SELECT HEX(JSON_DETAILED('{"a":1,"b":[1,2]}')) AS h; +h +7B0A202020202261223A20312C0A202020202262223A200A202020205B0A2020202020202020312C0A2020202020202020320A202020205D0A7D +SELECT JSON_PRETTY('{"a":1,"b":[1,2]}') AS v; +v +{ + "a": 1, + "b": + [ + 1, + 2 + ] +} +SELECT HEX(JSON_PRETTY('{"a":1,"b":[1,2]}')) AS h; +h +7B0A202020202261223A20312C0A202020202262223A200A202020205B0A2020202020202020312C0A2020202020202020320A202020205D0A7D +# JSON_DETAILED with an explicit tab size +SELECT HEX(JSON_DETAILED('{"a":1}', 2)) AS h; +h +7B0A20202261223A20310A7D +SELECT HEX(JSON_DETAILED('{"a":1}', 0)) AS h; +h +7B0A2261223A20310A7D +# empty containers +SELECT HEX(JSON_LOOSE('{}')) AS h, HEX(JSON_LOOSE('[]')) AS h2; +h h2 +7B7D 5B5D +SELECT HEX(JSON_DETAILED('{}')) AS h, HEX(JSON_DETAILED('[]')) AS h2; +h h2 +7B0A7D 5B5D +# the formatters are a pass-through in JSON context +SELECT JSON_ARRAY(JSON_LOOSE('{"a":1}')) AS v; +v +[{"a":1}] +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1}')) AS v; +v +[{"a": 1}] +# +# 6. The remaining functions, for completeness of the surface. +# +SELECT JSON_KEYS('{"a":1,"b":2}') AS v, HEX(JSON_KEYS('{"a":1,"b":2}')) AS h; +v h +["a", "b"] 5B2261222C202262225D +SELECT JSON_KEYS('{"a":{"b":1}}', '$.a') AS v; +v +["b"] +SELECT JSON_KEYS('{}') AS v; +v +[] +SELECT JSON_QUOTE('a"b') AS v, HEX(JSON_QUOTE('a"b')) AS h; +v h +"a\"b" 22615C226222 +SELECT JSON_UNQUOTE('"a\\"b"') AS v; +v +a"b +SELECT JSON_UNQUOTE('not quoted') AS v; +v +not quoted +SELECT JSON_NORMALIZE('{"b":1,"a":2}') AS v; +v +{"a":2.0E0,"b":1.0E0} +SELECT HEX(JSON_NORMALIZE('{"b":1,"a":2}')) AS h; +h +7B2261223A322E3045302C2262223A312E3045307D +SELECT JSON_TYPE('{}') AS o, JSON_TYPE('[]') AS a, JSON_TYPE('1') AS i, +JSON_TYPE('1.5') AS d, JSON_TYPE('"s"') AS s, JSON_TYPE('true') AS b, +JSON_TYPE('null') AS n; +o a i d s b n +OBJECT ARRAY INTEGER DOUBLE STRING BOOLEAN NULL +SELECT JSON_LENGTH('{"a":1,"b":2}') AS v, JSON_LENGTH('[1,2,3]') AS a; +v a +2 3 +SELECT JSON_DEPTH('1') AS s, JSON_DEPTH('[]') AS e, JSON_DEPTH('[[1]]') AS n; +s e n +1 1 3 +SELECT JSON_VALID('{"a":1}') AS ok, JSON_VALID('{') AS bad, JSON_VALID(NULL) AS n; +ok bad n +1 0 NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_EXISTS('{"a":1}', '$.a') AS y, JSON_EXISTS('{"a":1}', '$.b') AS n; +y n +1 0 +SELECT JSON_CONTAINS('{"a":1,"b":2}', '1', '$.a') AS y; +y +1 +SELECT JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a', '$.z') AS y; +y +1 +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS y; +y +1 +SELECT JSON_EQUALS('{"a":1,"b":2}', '{"b":2,"a":1}') AS y; +y +1 +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS v; +v +"$.a" +SELECT JSON_SEARCH('["x","x"]', 'all', 'x') AS v; +v +["$[0]", "$[1]"] +SELECT HEX(JSON_SEARCH('["x","x"]', 'all', 'x')) AS h; +h +5B22245B305D222C2022245B315D225D +# +# 7. Chained calls. A JSON function feeding another JSON function is +# the shape the reparse work targets, so the current bytes of every +# combination are recorded here. +# +SELECT JSON_SET(JSON_SET('{"a":1,"b":2}', '$.a', 9), '$.b', 8) AS v; +v +{"a": 9, "b": 8} +SELECT JSON_INSERT(JSON_REMOVE('{"a":1,"b":2}', '$.a'), '$.c', 3) AS v; +v +{"b": 2, "c": 3} +SELECT JSON_MERGE(JSON_ARRAY(1,2), JSON_ARRAY(3)) AS v; +v +[1, 2, 3] +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a',1), JSON_OBJECT('b',2)) AS v; +v +{"a": 1, "b": 2} +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', 2) AS v; +v +[1, 2] +SELECT JSON_EXTRACT(JSON_SET('{"a":1}', '$.b', 2), '$') AS v; +v +{"a": 1, "b": 2} +SELECT JSON_QUERY(JSON_SET('{"a":{"b":1}}', '$.a.c', 2), '$.a') AS v; +v +{"b": 1, "c": 2} +SELECT JSON_COMPACT(JSON_SET('{"a":1}', '$.b', 2)) AS v; +v +{"a":1,"b":2} +SELECT JSON_LOOSE(JSON_COMPACT('{"a": 1}')) AS v; +v +{"a": 1} +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', 2)) AS v; +v +1 +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_ARRAY(1))) AS v; +v +3 +# a mutator over an extracted fragment +SELECT JSON_SET(JSON_EXTRACT('{"a":{"b":1}}', '$.a'), '$.c', 2) AS v; +v +{"b": 1, "c": 2} +# a mutator over a query slice that kept odd input spacing +SELECT JSON_SET(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a'), '$.d', 3) AS v; +v +{"b": 1, "c": 2, "d": 3} +# a constructor fed by a mutator fed by a constructor +SELECT JSON_OBJECT('k', JSON_SET(JSON_OBJECT('a', 1), '$.b', 2)) AS v; +v +{"k": {"a": 1, "b": 2}} diff --git a/mysql-test/main/func_json_format.test b/mysql-test/main/func_json_format.test new file mode 100644 index 0000000000000..ea21750b0959a --- /dev/null +++ b/mysql-test/main/func_json_format.test @@ -0,0 +1,240 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: exact output formatting of the JSON functions. +--echo # +--echo # Every JSON producer is exercised with input in several formattings +--echo # (compact, already-spaced, whitespace-heavy) so that the exact bytes +--echo # each function emits are recorded. HEX() is used next to the plain +--echo # output wherever whitespace decides the answer, because leading and +--echo # trailing space is easy to lose in a result file. +--echo # + +--echo # +--echo # 1. Mutating functions. These reformat their document argument. +--echo # + +--echo # --- JSON_SET --- +SELECT JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2) AS v; +SELECT HEX(JSON_SET('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a', 2)) AS h; +SELECT JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2) AS v; +SELECT HEX(JSON_SET('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a', 2)) AS h; +SELECT JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +SELECT HEX(JSON_SET('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2)) AS h; +--echo # a path that matches nothing still reformats the document +SELECT JSON_SET('{"a":1,"b":[1,2]}', '$.zzz.deeper', 2) AS v; +--echo # scalar document +SELECT JSON_SET('1', '$', 2) AS v; +SELECT JSON_SET('"str"', '$.a', 2) AS v; + +--echo # --- JSON_INSERT --- +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2) AS v; +SELECT HEX(JSON_INSERT('{"a":1,"b":[1,2]}', '$.new', 2)) AS h; +SELECT JSON_INSERT('{"a": 1, "b": [1, 2]}', '$.new', 2) AS v; +SELECT JSON_INSERT('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.new', 2) AS v; +--echo # existing key is not replaced, document is still reformatted +SELECT JSON_INSERT('{"a":1,"b":[1,2]}', '$.a', 9) AS v; + +--echo # --- JSON_REPLACE --- +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2) AS v; +SELECT HEX(JSON_REPLACE('{"a":1,"b":[1,2]}', '$.a', 2)) AS h; +SELECT JSON_REPLACE('{"a": 1, "b": [1, 2]}', '$.a', 2) AS v; +SELECT JSON_REPLACE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a', 2) AS v; +--echo # missing key is not inserted, document is still reformatted +SELECT JSON_REPLACE('{"a":1,"b":[1,2]}', '$.new', 2) AS v; + +--echo # --- JSON_REMOVE --- +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a') AS v; +SELECT HEX(JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.a')) AS h; +SELECT JSON_REMOVE('{"a": 1, "b": [1, 2], "c": {"d": "x"}}', '$.a') AS v; +SELECT JSON_REMOVE('{"a" : 1 ,\n\t"b" : [ 1 , 2 ] }', '$.a') AS v; +--echo # removal at the cut boundaries: first, middle and last member +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.b') AS v; +SELECT JSON_REMOVE('{"a":1,"b":[1,2],"c":{"d":"x"}}', '$.c') AS v; +SELECT JSON_REMOVE('[1,2,3]', '$[0]') AS v; +SELECT JSON_REMOVE('[1,2,3]', '$[1]') AS v; +SELECT JSON_REMOVE('[1,2,3]', '$[2]') AS v; +--echo # removing the only member +SELECT JSON_REMOVE('{"a":1}', '$.a') AS v, HEX(JSON_REMOVE('{"a":1}', '$.a')) AS h; +SELECT JSON_REMOVE('[1]', '$[0]') AS v, HEX(JSON_REMOVE('[1]', '$[0]')) AS h; + +--echo # --- JSON_ARRAY_APPEND --- +SELECT JSON_ARRAY_APPEND('[1,2]', '$', 3) AS v; +SELECT HEX(JSON_ARRAY_APPEND('[1,2]', '$', 3)) AS h; +SELECT JSON_ARRAY_APPEND('[1, 2]', '$', 3) AS v; +SELECT JSON_ARRAY_APPEND('{"a":1,"b":[1,2]}', '$.b', 3) AS v; +--echo # appending to a scalar wraps it into an array +SELECT JSON_ARRAY_APPEND('{"a":1}', '$.a', 2) AS v; + +--echo # --- JSON_ARRAY_INSERT --- +SELECT JSON_ARRAY_INSERT('[1,2]', '$[0]', 0) AS v; +SELECT HEX(JSON_ARRAY_INSERT('[1,2]', '$[0]', 0)) AS h; +SELECT JSON_ARRAY_INSERT('[1, 2]', '$[1]', 0) AS v; +SELECT JSON_ARRAY_INSERT('[1,2]', '$[9]', 0) AS v; +SELECT JSON_ARRAY_INSERT('{"a":1,"b":[1,2]}', '$.b[0]', 0) AS v; + +--echo # --- JSON_MERGE / JSON_MERGE_PRESERVE --- +SELECT JSON_MERGE('[1,2]', '[3,4]') AS v, HEX(JSON_MERGE('[1,2]', '[3,4]')) AS h; +SELECT JSON_MERGE('{"a":1}', '{"b":2}') AS v; +SELECT JSON_MERGE('{"a":1}', '{"a":2}') AS v; +SELECT JSON_MERGE_PRESERVE('{"a":1}', '{"a":2}') AS v; +SELECT JSON_MERGE('1', '2') AS v; +--echo # three-way merge +SELECT JSON_MERGE('[1]', '[2]', '[3]') AS v; + +--echo # --- JSON_MERGE_PATCH --- +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS v; +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '{"b":2}')) AS h; +SELECT JSON_MERGE_PATCH('{"a":1}', '{"a":null}') AS v; +SELECT JSON_MERGE_PATCH('{"a":{"b":1,"c":2}}', '{"a":{"b":9}}') AS v; +--echo # a non-object patch is adopted wholesale +SELECT JSON_MERGE_PATCH('{"a":1}', '[1, 2]') AS v; +SELECT HEX(JSON_MERGE_PATCH('{"a":1}', '[1, 2]')) AS h; +SELECT JSON_MERGE_PATCH('{"a":1}', '"str"') AS v; +--echo # the last argument decides when it is not an object +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}', '[9]') AS v; + +--echo # +--echo # 2. Constructors. These copy their arguments and do NOT reformat. +--echo # + +--echo # plain strings are quoted, JSON-typed arguments are embedded raw +# A view keeps its body as printed text, and a boolean literal prints as the +# number it equals, so read through one this array would hold a 1 rather +# than a true. +--disable_view_protocol +SELECT JSON_ARRAY(1, 'str', NULL, TRUE) AS v; +--enable_view_protocol +SELECT JSON_ARRAY() AS v, HEX(JSON_ARRAY()) AS h; +SELECT JSON_OBJECT() AS v, HEX(JSON_OBJECT()) AS h; +SELECT JSON_OBJECT('a', 1, 'b', 'str') AS v; +SELECT HEX(JSON_OBJECT('a', 1, 'b', 'str')) AS h; +--echo # embedded document arguments keep their own formatting +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1,"b": 2}')) AS v; +SELECT JSON_ARRAY(JSON_DETAILED('{"a":1}')) AS v; +SELECT JSON_ARRAY(JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +SELECT JSON_OBJECT('k', JSON_QUERY('{"x":{"a":1,"b": 2}}', '$.x')) AS v; +--echo # a nested constructor call +SELECT JSON_ARRAY(JSON_ARRAY(1,2), JSON_OBJECT('a',1)) AS v; +SELECT JSON_OBJECT('a', JSON_ARRAY(1,2)) AS v; +--echo # a mutator result embedded in a constructor +SELECT JSON_ARRAY(JSON_SET('{"a":1,"b":2}', '$.a', 9)) AS v; + +--echo # +--echo # 3. JSON_EXTRACT: formatting and validation depend on the evaluation +--echo # context. In string context the document is scanned to the end and +--echo # the result is reformatted; in numeric context the scan stops at the +--echo # first match and nothing is reformatted. +--echo # + +SELECT JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a') AS v; +SELECT HEX(JSON_EXTRACT('{"a":{"b":1,"c":2}}', '$.a')) AS h; +SELECT JSON_EXTRACT('{"a": {"b": 1, "c": 2}}', '$.a') AS v; +--echo # several paths produce an array wrapper +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS v; +SELECT HEX(JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b')) AS h; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.*') AS v; +--echo # string context rejects trailing garbage after the document +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') AS v; +--echo # numeric context does not reach the trailing garbage +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') + 0 AS v; +SELECT JSON_EXTRACT('{"a":1} trailing', '$.a') * 1.0 AS v; +SELECT CAST(JSON_EXTRACT('{"a":1} trailing', '$.a') AS SIGNED) AS v; +--echo # the same asymmetry with a structurally broken tail +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') AS v; +SELECT JSON_EXTRACT('{"a":1,"b":[}', '$.a') + 0 AS v; +--echo # numeric context on a non-scalar match +SELECT JSON_EXTRACT('{"a":[1,2]}', '$.a') + 0 AS v; + +--echo # +--echo # 4. JSON_QUERY and JSON_VALUE return a slice of the input and keep the +--echo # input formatting. +--echo # + +SELECT JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a') AS v; +SELECT HEX(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a')) AS h; +SELECT JSON_QUERY('{"a":[1, 2]}', '$.a') AS v; +--echo # JSON_QUERY only returns objects and arrays +SELECT JSON_QUERY('{"a":1}', '$.a') AS v; +SELECT JSON_VALUE('{"a":1}', '$.a') AS v; +SELECT JSON_VALUE('{"a":"s t"}', '$.a') AS v, HEX(JSON_VALUE('{"a":"s t"}', '$.a')) AS h; +--echo # JSON_VALUE unquotes, so escapes are resolved +SELECT JSON_VALUE('{"a":"q\\"q"}', '$.a') AS v; +SELECT JSON_VALUE('{"a":[1,2]}', '$.a') AS v; +--echo # a slice taken past trailing garbage +SELECT JSON_QUERY('{"a":{"b":1}} trailing', '$.a') AS v; +SELECT JSON_VALUE('{"a":1} trailing', '$.a') AS v; + +--echo # +--echo # 5. The explicit formatters. These stay after the mutator epilogues +--echo # are gone, so their exact bytes matter. +--echo # + +SELECT JSON_COMPACT('{"a": 1, "b": [1, 2]}') AS v; +SELECT HEX(JSON_COMPACT('{"a": 1, "b": [1, 2]}')) AS h; +SELECT JSON_LOOSE('{"a":1,"b":[1,2]}') AS v; +SELECT HEX(JSON_LOOSE('{"a":1,"b":[1,2]}')) AS h; +SELECT JSON_DETAILED('{"a":1,"b":[1,2]}') AS v; +SELECT HEX(JSON_DETAILED('{"a":1,"b":[1,2]}')) AS h; +SELECT JSON_PRETTY('{"a":1,"b":[1,2]}') AS v; +SELECT HEX(JSON_PRETTY('{"a":1,"b":[1,2]}')) AS h; +--echo # JSON_DETAILED with an explicit tab size +SELECT HEX(JSON_DETAILED('{"a":1}', 2)) AS h; +SELECT HEX(JSON_DETAILED('{"a":1}', 0)) AS h; +--echo # empty containers +SELECT HEX(JSON_LOOSE('{}')) AS h, HEX(JSON_LOOSE('[]')) AS h2; +SELECT HEX(JSON_DETAILED('{}')) AS h, HEX(JSON_DETAILED('[]')) AS h2; +--echo # the formatters are a pass-through in JSON context +SELECT JSON_ARRAY(JSON_LOOSE('{"a":1}')) AS v; +SELECT JSON_ARRAY(JSON_COMPACT('{"a": 1}')) AS v; + +--echo # +--echo # 6. The remaining functions, for completeness of the surface. +--echo # + +SELECT JSON_KEYS('{"a":1,"b":2}') AS v, HEX(JSON_KEYS('{"a":1,"b":2}')) AS h; +SELECT JSON_KEYS('{"a":{"b":1}}', '$.a') AS v; +SELECT JSON_KEYS('{}') AS v; +SELECT JSON_QUOTE('a"b') AS v, HEX(JSON_QUOTE('a"b')) AS h; +SELECT JSON_UNQUOTE('"a\\"b"') AS v; +SELECT JSON_UNQUOTE('not quoted') AS v; +SELECT JSON_NORMALIZE('{"b":1,"a":2}') AS v; +SELECT HEX(JSON_NORMALIZE('{"b":1,"a":2}')) AS h; +SELECT JSON_TYPE('{}') AS o, JSON_TYPE('[]') AS a, JSON_TYPE('1') AS i, + JSON_TYPE('1.5') AS d, JSON_TYPE('"s"') AS s, JSON_TYPE('true') AS b, + JSON_TYPE('null') AS n; +SELECT JSON_LENGTH('{"a":1,"b":2}') AS v, JSON_LENGTH('[1,2,3]') AS a; +SELECT JSON_DEPTH('1') AS s, JSON_DEPTH('[]') AS e, JSON_DEPTH('[[1]]') AS n; +SELECT JSON_VALID('{"a":1}') AS ok, JSON_VALID('{') AS bad, JSON_VALID(NULL) AS n; +SELECT JSON_EXISTS('{"a":1}', '$.a') AS y, JSON_EXISTS('{"a":1}', '$.b') AS n; +SELECT JSON_CONTAINS('{"a":1,"b":2}', '1', '$.a') AS y; +SELECT JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a', '$.z') AS y; +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS y; +SELECT JSON_EQUALS('{"a":1,"b":2}', '{"b":2,"a":1}') AS y; +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS v; +SELECT JSON_SEARCH('["x","x"]', 'all', 'x') AS v; +SELECT HEX(JSON_SEARCH('["x","x"]', 'all', 'x')) AS h; + +--echo # +--echo # 7. Chained calls. A JSON function feeding another JSON function is +--echo # the shape the reparse work targets, so the current bytes of every +--echo # combination are recorded here. +--echo # + +SELECT JSON_SET(JSON_SET('{"a":1,"b":2}', '$.a', 9), '$.b', 8) AS v; +SELECT JSON_INSERT(JSON_REMOVE('{"a":1,"b":2}', '$.a'), '$.c', 3) AS v; +SELECT JSON_MERGE(JSON_ARRAY(1,2), JSON_ARRAY(3)) AS v; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a',1), JSON_OBJECT('b',2)) AS v; +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', 2) AS v; +SELECT JSON_EXTRACT(JSON_SET('{"a":1}', '$.b', 2), '$') AS v; +SELECT JSON_QUERY(JSON_SET('{"a":{"b":1}}', '$.a.c', 2), '$.a') AS v; +SELECT JSON_COMPACT(JSON_SET('{"a":1}', '$.b', 2)) AS v; +SELECT JSON_LOOSE(JSON_COMPACT('{"a": 1}')) AS v; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', 2)) AS v; +SELECT JSON_DEPTH(JSON_SET('{"a":1}', '$.b', JSON_ARRAY(1))) AS v; +--echo # a mutator over an extracted fragment +SELECT JSON_SET(JSON_EXTRACT('{"a":{"b":1}}', '$.a'), '$.c', 2) AS v; +--echo # a mutator over a query slice that kept odd input spacing +SELECT JSON_SET(JSON_QUERY('{"a":{"b":1, "c":2}}', '$.a'), '$.d', 3) AS v; +--echo # a constructor fed by a mutator fed by a constructor +SELECT JSON_OBJECT('k', JSON_SET(JSON_OBJECT('a', 1), '$.b', 2)) AS v; diff --git a/mysql-test/main/func_json_invalid.result b/mysql-test/main/func_json_invalid.result new file mode 100644 index 0000000000000..82c1fdffe6869 --- /dev/null +++ b/mysql-test/main/func_json_invalid.result @@ -0,0 +1,692 @@ +# +# Behavioral baseline: how the JSON functions attest to input that is +# not valid JSON, is NULL, is not a string at all, or contains bytes +# that JSON cannot carry unescaped. +# +# For every case the recorded observable is the whole trichotomy: +# the value (or SQL NULL), and the warning or error with its code. +# Results are shown through HEX() whenever the answer contains bytes +# that are not printable ASCII. +# +# +# 1. Documents that are not valid JSON. +# +# --- truncated at every structural position --- +SELECT JSON_VALID('') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a"') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a":') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a":1') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('{"a":1,') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('[') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('[1') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('[1,') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('"unterminated') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# --- structurally wrong --- +SELECT JSON_VALID('}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID(']') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID('{]') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('[}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('{a:1}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('{"a" 1}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 +SELECT JSON_VALID('{"a":1,,"b":2}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_VALID('{"a":1}{"b":2}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_VALID('{"a":1} trailing') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 9 +SELECT JSON_VALID('nul') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_VALID('TRUE') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID('01') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('1.') AS v; +v +1 +SELECT JSON_VALID('.1') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +SELECT JSON_VALID('1e') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +SELECT JSON_VALID('+1') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# --- bad escapes inside strings --- +SELECT JSON_VALID('{"a":"\\q"}') AS v; +v +1 +SELECT JSON_VALID('{"a":"\\u12"}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 10 +SELECT JSON_VALID('{"a":"\\uZZZZ"}') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_VALID('{"a":"trailing backslash\\"}') AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# 2. The same invalid documents fed to each function. This records +# which functions answer NULL with a warning and which raise an +# error, and with which code. +# +SELECT JSON_SET('{"a":1,', '$.a', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT JSON_INSERT('{"a":1,', '$.b', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_insert' +SELECT JSON_REPLACE('{"a":1,', '$.a', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_replace' +SELECT JSON_REMOVE('{"a":1,', '$.a') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_remove' +SELECT JSON_ARRAY_APPEND('[1,', '$', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array_append' +SELECT JSON_ARRAY_INSERT('[1,', '$[0]', 2) AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array_insert' +SELECT JSON_MERGE('[1,', '[2]') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +SELECT JSON_MERGE('[1]', '[2,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE_PATCH('{"a":1,', '{"b":2}') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_patch' +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +SELECT JSON_EXTRACT('{"a":1,', '$.a') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_QUERY('{"a":1,', '$.a') AS v; +v +NULL +SELECT JSON_VALUE('{"a":1,', '$.a') AS v; +v +1 +SELECT JSON_KEYS('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +SELECT JSON_LENGTH('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_length' +SELECT JSON_DEPTH('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_depth' +SELECT JSON_TYPE('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_type' +SELECT JSON_EXISTS('{"a":1,', '$.a') AS v; +v +1 +SELECT JSON_CONTAINS('{"a":1,', '1', '$.a') AS v; +v +1 +SELECT JSON_CONTAINS_PATH('{"a":1,', 'one', '$.a') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_OVERLAPS('[1,', '[1]') AS v; +v +1 +SELECT JSON_EQUALS('{"a":1,', '{"a":1}') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_equals' +SELECT JSON_SEARCH('{"a":"x",', 'one', 'x') AS v; +v +"$.a" +SELECT JSON_NORMALIZE('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize' +SELECT JSON_COMPACT('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_compact' +SELECT JSON_LOOSE('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_loose' +SELECT JSON_DETAILED('{"a":1,') AS v; +v +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_detailed' +SELECT JSON_UNQUOTE('"unterminated') AS v; +v +"unterminated +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_unquote' +SELECT JSON_QUOTE('{"a":1,') AS v; +v +"{\"a\":1," +# +# 3. Invalid path arguments. +# +SELECT JSON_EXTRACT('{"a":1}', '') AS v; +v +NULL +Warnings: +Warning 4041 Unexpected end of JSON path in argument 2 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":1}', 'a') AS v; +v +NULL +Warnings: +Warning 4042 Syntax error in JSON path in argument 2 to function 'json_extract' at position 1 +SELECT JSON_EXTRACT('{"a":1}', '$..a') AS v; +v +NULL +Warnings: +Warning 4042 Syntax error in JSON path in argument 2 to function 'json_extract' at position 3 +SELECT JSON_EXTRACT('{"a":1}', '$.') AS v; +v +NULL +Warnings: +Warning 4041 Unexpected end of JSON path in argument 2 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":1}', '$[') AS v; +v +NULL +Warnings: +Warning 4041 Unexpected end of JSON path in argument 2 to function 'json_extract' +SELECT JSON_SET('{"a":1}', '$.*', 2) AS v; +v +NULL +Warnings: +Warning 4044 Wildcards or range in JSON path not allowed in argument 2 to function 'json_set' +SELECT JSON_SET('{"a":1}', '$**.a', 2) AS v; +v +NULL +Warnings: +Warning 4044 Wildcards or range in JSON path not allowed in argument 2 to function 'json_set' +# a path containing a quote, which the key comparison must not accept +SELECT JSON_EXTRACT('{"a\\"b":1}', '$.a"b') AS v; +v +NULL +SELECT JSON_SET('{"a":1}', '$.a"b', 2) AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 11 +SELECT JSON_INSERT('{"a":1}', '$.a"b', 2) AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 11 +# +# 4. NULL arguments in every position. +# +SELECT JSON_SET(NULL, '$.a', 1) AS v; +v +NULL +SELECT JSON_SET('{"a":1}', NULL, 1) AS v; +v +NULL +SELECT JSON_SET('{"a":1}', '$.a', NULL) AS v; +v +{"a": null} +SELECT JSON_INSERT(NULL, '$.a', 1) AS v; +v +NULL +SELECT JSON_REMOVE(NULL, '$.a') AS v; +v +NULL +SELECT JSON_REMOVE('{"a":1}', NULL) AS v; +v +NULL +SELECT JSON_MERGE(NULL, '[1]') AS v; +v +NULL +SELECT JSON_MERGE('[1]', NULL) AS v; +v +NULL +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS v; +v +NULL +SELECT JSON_MERGE_PATCH('{"a":1}', NULL) AS v; +v +NULL +SELECT JSON_EXTRACT(NULL, '$.a') AS v; +v +NULL +SELECT JSON_EXTRACT('{"a":1}', NULL) AS v; +v +NULL +SELECT JSON_ARRAY(NULL) AS v; +v +[null] +SELECT JSON_OBJECT('a', NULL) AS v; +v +{"a": null} +SELECT JSON_OBJECT(NULL, 1) AS v; +v +{"": 1} +SELECT JSON_KEYS(NULL) AS v; +v +NULL +SELECT JSON_LENGTH(NULL) AS v; +v +NULL +SELECT JSON_DEPTH(NULL) AS v; +v +NULL +SELECT JSON_TYPE(NULL) AS v; +v +NULL +SELECT JSON_QUOTE(NULL) AS v; +v +NULL +SELECT JSON_UNQUOTE(NULL) AS v; +v +NULL +SELECT JSON_VALID(NULL) AS v; +v +NULL +SELECT JSON_COMPACT(NULL) AS v; +v +NULL +# a JSON null inside a document is not an SQL NULL +SELECT JSON_EXTRACT('{"a":null}', '$.a') AS v, JSON_TYPE(JSON_EXTRACT('{"a":null}', '$.a')) AS t; +v t +null NULL +SELECT JSON_VALUE('{"a":null}', '$.a') AS v; +v +NULL +SELECT JSON_VALUE('{"a":null}', '$.a') IS NULL AS is_null; +is_null +1 +# +# 5. Arguments that are not strings. The document argument is taken +# through its string value, so the conversion is part of the result. +# +SELECT JSON_VALID(1) AS v, JSON_VALID(1.5) AS d, JSON_VALID(TRUE) AS b; +v d b +1 1 1 +SELECT JSON_TYPE(1) AS v, JSON_TYPE(1.5) AS d; +v d +INTEGER DOUBLE +SELECT JSON_EXTRACT(1, '$') AS v; +v +1 +SELECT JSON_SET(1, '$', 2) AS v; +v +2 +SELECT JSON_DEPTH(1) AS v; +v +1 +SELECT JSON_VALID(DATE'2020-01-01') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 5 +SELECT JSON_EXTRACT(DATE'2020-01-01', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 5 +# numeric and temporal values as inserted values, not documents +SELECT JSON_ARRAY(1, 1.5, -0.0, 1e300, DATE'2020-01-01', TIME'10:20:30') AS v; +v +[1, 1.5, 0.0, 1e300, "2020-01-01", "10:20:30"] +SELECT JSON_OBJECT('i', 1, 'd', 1.5, 'dt', DATE'2020-01-01') AS v; +v +{"i": 1, "d": 1.5, "dt": "2020-01-01"} +SELECT JSON_SET('{}', '$.i', 1, '$.d', 1.5, '$.dt', DATE'2020-01-01') AS v; +v +{"i": 1, "d": 1.5, "dt": "2020-01-01"} +# binary values +SELECT JSON_ARRAY(X'41') AS v, HEX(JSON_ARRAY(X'41')) AS h; +v h +["A"] 5B2241225D +SELECT JSON_ARRAY(X'00') AS v, HEX(JSON_ARRAY(X'00')) AS h; +v h +["\u0000"] 5B225C7530303030225D +SELECT JSON_VALID(X'7B7D') AS v; +v +1 +# +# 6. Hostile strings: control characters and bytes that JSON requires +# to be escaped. These pin what the constructors emit today for +# values that cannot be carried literally. +# +# --- control characters through the constructors --- +SELECT HEX(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS h; +h +5B225C7530303031225D +SELECT HEX(JSON_ARRAY(CHAR(9 USING utf8mb4))) AS h; +h +5B225C74225D +SELECT HEX(JSON_ARRAY(CHAR(10 USING utf8mb4))) AS h; +h +5B225C6E225D +SELECT HEX(JSON_ARRAY(CHAR(13 USING utf8mb4))) AS h; +h +5B225C72225D +SELECT HEX(JSON_ARRAY(CHAR(27 USING utf8mb4))) AS h; +h +5B225C7530303142225D +SELECT HEX(JSON_ARRAY(CHAR(31 USING utf8mb4))) AS h; +h +5B225C7530303146225D +SELECT HEX(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS h; +h +7B226B223A20225C7530303031227D +# a control character in the KEY position +SELECT HEX(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS h; +h +7B226B5C7530303031223A20317D +# is the constructor output accepted back as a document? +SELECT JSON_VALID(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS v; +v +1 +SELECT JSON_VALID(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS v; +v +1 +SELECT JSON_VALID(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS v; +v +1 +# the same values through the mutators +SELECT HEX(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS h; +h +7B2261223A20225C7530303031227D +SELECT JSON_VALID(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS v; +v +1 +SELECT HEX(JSON_ARRAY_APPEND('[]', '$', CHAR(1 USING utf8mb4))) AS h; +h +5B225C7530303031225D +# and through JSON_QUOTE, which escapes for exactly this purpose +SELECT HEX(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS h; +h +225C753030303122 +SELECT HEX(JSON_QUOTE(CHAR(9 USING utf8mb4))) AS h; +h +225C7422 +SELECT JSON_VALID(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS v; +v +1 +# --- quote and backslash edges --- +SELECT JSON_ARRAY('a"b') AS v, HEX(JSON_ARRAY('a"b')) AS h; +v h +["a\"b"] 5B22615C2262225D +SELECT JSON_ARRAY('a\\b') AS v, HEX(JSON_ARRAY('a\\b')) AS h; +v h +["a\\b"] 5B22615C5C62225D +SELECT JSON_ARRAY('a\\"b') AS v; +v +["a\\\"b"] +SELECT JSON_ARRAY('"') AS v; +v +["\""] +SELECT JSON_ARRAY('\\') AS v; +v +["\\"] +SELECT JSON_OBJECT('a"b', 1) AS v; +v +{"a\"b": 1} +SELECT JSON_OBJECT('a\\b', 1) AS v; +v +{"a\\b": 1} +SELECT JSON_VALID(JSON_ARRAY('a"b')) AS q, JSON_VALID(JSON_ARRAY('a\\b')) AS bs; +q bs +1 1 +SELECT JSON_VALID(JSON_OBJECT('a"b', 1)) AS q, JSON_VALID(JSON_OBJECT('a\\b', 1)) AS bs; +q bs +1 1 +SELECT JSON_QUOTE('a"b') AS v, JSON_QUOTE('a\\b') AS v2; +v v2 +"a\"b" "a\\b" +SELECT JSON_UNQUOTE('"a\\\\b"') AS v; +v +a\b +SELECT JSON_UNQUOTE(JSON_QUOTE('a"b')) AS v; +v +a"b +# a value that already looks like a quoted JSON string +SELECT JSON_ARRAY('"quoted"') AS v; +v +["\"quoted\""] +SELECT JSON_ARRAY('{"a":1}') AS v; +v +["{\"a\":1}"] +SELECT JSON_VALID(JSON_ARRAY('{"a":1}')) AS v; +v +1 +# --- byte sequences that are not valid in the connection charset --- +SET NAMES utf8mb4; +SELECT HEX(JSON_ARRAY(_utf8mb4 X'41')) AS h; +h +5B2241225D +SELECT HEX(JSON_ARRAY(_utf8mb4 X'FF')) AS h; +ERROR HY000: Invalid utf8mb4 character string: 'FF' +SELECT HEX(JSON_ARRAY(_utf8mb4 X'C3')) AS h; +ERROR HY000: Invalid utf8mb4 character string: 'C3' +# the same bytes carried in as binary and converted +SELECT HEX(JSON_ARRAY(CONVERT(X'FF' USING utf8mb4))) AS h; +h +5B223F225D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF' +SELECT HEX(JSON_ARRAY(CONVERT(X'C3' USING utf8mb4))) AS h; +h +5B223F225D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xC3' +SELECT HEX(JSON_ARRAY(CONVERT(X'C328' USING utf8mb4))) AS h; +h +5B223F28225D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xC3(' +SELECT HEX(JSON_ARRAY(CONVERT(X'EDA080' USING utf8mb4))) AS h; +h +5B22EDA080225D +SELECT HEX(JSON_QUOTE(CONVERT(X'FF' USING utf8mb4))) AS h; +h +223F22 +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF' +# invalid bytes inside a document argument +SELECT JSON_VALID(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4)) AS v; +v +1 +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF"}' +SELECT HEX(JSON_SET(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.b', 1)) AS h; +h +7B2261223A20223F222C202262223A20317D +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF"}' +SELECT HEX(JSON_EXTRACT(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.a')) AS h; +h +223F22 +Warnings: +Warning 1300 Invalid utf8mb4 character string: '\xFF"}' +# an escape sequence naming a code point that utf8mb4 cannot encode +SELECT HEX(JSON_UNQUOTE('"\\ud800"')) AS h; +h +225C756438303022 +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_unquote' at position 7 +SELECT HEX(JSON_UNQUOTE('"\\ud800\\udc00"')) AS h; +h +F0908080 +SELECT JSON_VALID('"\\ud800"') AS v; +v +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 7 +# +# 7. Strict and non-strict sql_mode. The document argument never +# depends on sql_mode, but stores of the result do; both are +# recorded so that a later change of the store rule is visible. +# +CREATE TABLE t1 (a VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +Warnings: +Warning 1265 Data truncated for column 'a' at row 1 +SELECT a, JSON_VALID(a) AS still_valid FROM t1; +a still_valid +{"a": 9, " 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +DELETE FROM t1; +SET @@sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +ERROR 22001: Data too long for column 'a' at row 1 +SELECT COUNT(*) AS rows_stored FROM t1; +rows_stored +0 +SET @@sql_mode=DEFAULT; +DROP TABLE t1; +# +# 8. Errors raised before a document can be built at all, so that +# no partially built result is ever returned as a value. +# +SELECT JSON_OBJECT('a') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_OBJECT' +SELECT JSON_SET('{}') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_SET' +SELECT JSON_SET('{}', '$.a') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_SET' +SELECT JSON_REMOVE('{}') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_REMOVE' +SELECT JSON_KEYS() AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_KEYS' +SELECT JSON_MERGE('[1]') AS v; +ERROR 42000: Incorrect parameter count in the call to native function 'JSON_MERGE' diff --git a/mysql-test/main/func_json_invalid.test b/mysql-test/main/func_json_invalid.test new file mode 100644 index 0000000000000..b665870333bf7 --- /dev/null +++ b/mysql-test/main/func_json_invalid.test @@ -0,0 +1,269 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: how the JSON functions attest to input that is +--echo # not valid JSON, is NULL, is not a string at all, or contains bytes +--echo # that JSON cannot carry unescaped. +--echo # +--echo # For every case the recorded observable is the whole trichotomy: +--echo # the value (or SQL NULL), and the warning or error with its code. +--echo # Results are shown through HEX() whenever the answer contains bytes +--echo # that are not printable ASCII. +--echo # + +--echo # +--echo # 1. Documents that are not valid JSON. +--echo # + +--echo # --- truncated at every structural position --- +SELECT JSON_VALID('') AS v; +SELECT JSON_VALID('{') AS v; +SELECT JSON_VALID('{"a"') AS v; +SELECT JSON_VALID('{"a":') AS v; +SELECT JSON_VALID('{"a":1') AS v; +SELECT JSON_VALID('{"a":1,') AS v; +SELECT JSON_VALID('[') AS v; +SELECT JSON_VALID('[1') AS v; +SELECT JSON_VALID('[1,') AS v; +SELECT JSON_VALID('"unterminated') AS v; + +--echo # --- structurally wrong --- +SELECT JSON_VALID('}') AS v; +SELECT JSON_VALID(']') AS v; +SELECT JSON_VALID('{]') AS v; +SELECT JSON_VALID('[}') AS v; +SELECT JSON_VALID('{a:1}') AS v; +SELECT JSON_VALID('{"a" 1}') AS v; +SELECT JSON_VALID('{"a":1,,"b":2}') AS v; +SELECT JSON_VALID('{"a":1}{"b":2}') AS v; +SELECT JSON_VALID('{"a":1} trailing') AS v; +SELECT JSON_VALID('nul') AS v; +SELECT JSON_VALID('TRUE') AS v; +SELECT JSON_VALID('01') AS v; +SELECT JSON_VALID('1.') AS v; +SELECT JSON_VALID('.1') AS v; +SELECT JSON_VALID('1e') AS v; +SELECT JSON_VALID('+1') AS v; + +--echo # --- bad escapes inside strings --- +SELECT JSON_VALID('{"a":"\\q"}') AS v; +SELECT JSON_VALID('{"a":"\\u12"}') AS v; +SELECT JSON_VALID('{"a":"\\uZZZZ"}') AS v; +SELECT JSON_VALID('{"a":"trailing backslash\\"}') AS v; + +--echo # +--echo # 2. The same invalid documents fed to each function. This records +--echo # which functions answer NULL with a warning and which raise an +--echo # error, and with which code. +--echo # + +SELECT JSON_SET('{"a":1,', '$.a', 2) AS v; +SELECT JSON_INSERT('{"a":1,', '$.b', 2) AS v; +SELECT JSON_REPLACE('{"a":1,', '$.a', 2) AS v; +SELECT JSON_REMOVE('{"a":1,', '$.a') AS v; +SELECT JSON_ARRAY_APPEND('[1,', '$', 2) AS v; +SELECT JSON_ARRAY_INSERT('[1,', '$[0]', 2) AS v; +SELECT JSON_MERGE('[1,', '[2]') AS v; +SELECT JSON_MERGE('[1]', '[2,') AS v; +SELECT JSON_MERGE_PATCH('{"a":1,', '{"b":2}') AS v; +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2,') AS v; +SELECT JSON_EXTRACT('{"a":1,', '$.a') AS v; +SELECT JSON_QUERY('{"a":1,', '$.a') AS v; +SELECT JSON_VALUE('{"a":1,', '$.a') AS v; +SELECT JSON_KEYS('{"a":1,') AS v; +SELECT JSON_LENGTH('{"a":1,') AS v; +# JSON_DEPTH says it can never answer NULL, so a view column or a cursor's +# temporary table built from it cannot hold one and the NULL below would +# arrive as a zero. +--disable_cursor_protocol +--disable_view_protocol +SELECT JSON_DEPTH('{"a":1,') AS v; +--enable_view_protocol +--enable_cursor_protocol +SELECT JSON_TYPE('{"a":1,') AS v; +SELECT JSON_EXISTS('{"a":1,', '$.a') AS v; +SELECT JSON_CONTAINS('{"a":1,', '1', '$.a') AS v; +SELECT JSON_CONTAINS_PATH('{"a":1,', 'one', '$.a') AS v; +SELECT JSON_OVERLAPS('[1,', '[1]') AS v; +SELECT JSON_EQUALS('{"a":1,', '{"a":1}') AS v; +SELECT JSON_SEARCH('{"a":"x",', 'one', 'x') AS v; +SELECT JSON_NORMALIZE('{"a":1,') AS v; +SELECT JSON_COMPACT('{"a":1,') AS v; +SELECT JSON_LOOSE('{"a":1,') AS v; +SELECT JSON_DETAILED('{"a":1,') AS v; +SELECT JSON_UNQUOTE('"unterminated') AS v; +SELECT JSON_QUOTE('{"a":1,') AS v; + +--echo # +--echo # 3. Invalid path arguments. +--echo # + +SELECT JSON_EXTRACT('{"a":1}', '') AS v; +SELECT JSON_EXTRACT('{"a":1}', 'a') AS v; +SELECT JSON_EXTRACT('{"a":1}', '$..a') AS v; +SELECT JSON_EXTRACT('{"a":1}', '$.') AS v; +SELECT JSON_EXTRACT('{"a":1}', '$[') AS v; +SELECT JSON_SET('{"a":1}', '$.*', 2) AS v; +SELECT JSON_SET('{"a":1}', '$**.a', 2) AS v; +--echo # a path containing a quote, which the key comparison must not accept +SELECT JSON_EXTRACT('{"a\\"b":1}', '$.a"b') AS v; +SELECT JSON_SET('{"a":1}', '$.a"b', 2) AS v; +SELECT JSON_INSERT('{"a":1}', '$.a"b', 2) AS v; + +--echo # +--echo # 4. NULL arguments in every position. +--echo # + +SELECT JSON_SET(NULL, '$.a', 1) AS v; +SELECT JSON_SET('{"a":1}', NULL, 1) AS v; +SELECT JSON_SET('{"a":1}', '$.a', NULL) AS v; +SELECT JSON_INSERT(NULL, '$.a', 1) AS v; +SELECT JSON_REMOVE(NULL, '$.a') AS v; +SELECT JSON_REMOVE('{"a":1}', NULL) AS v; +SELECT JSON_MERGE(NULL, '[1]') AS v; +SELECT JSON_MERGE('[1]', NULL) AS v; +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS v; +SELECT JSON_MERGE_PATCH('{"a":1}', NULL) AS v; +SELECT JSON_EXTRACT(NULL, '$.a') AS v; +SELECT JSON_EXTRACT('{"a":1}', NULL) AS v; +SELECT JSON_ARRAY(NULL) AS v; +SELECT JSON_OBJECT('a', NULL) AS v; +SELECT JSON_OBJECT(NULL, 1) AS v; +SELECT JSON_KEYS(NULL) AS v; +SELECT JSON_LENGTH(NULL) AS v; +SELECT JSON_DEPTH(NULL) AS v; +SELECT JSON_TYPE(NULL) AS v; +SELECT JSON_QUOTE(NULL) AS v; +SELECT JSON_UNQUOTE(NULL) AS v; +SELECT JSON_VALID(NULL) AS v; +SELECT JSON_COMPACT(NULL) AS v; +--echo # a JSON null inside a document is not an SQL NULL +SELECT JSON_EXTRACT('{"a":null}', '$.a') AS v, JSON_TYPE(JSON_EXTRACT('{"a":null}', '$.a')) AS t; +SELECT JSON_VALUE('{"a":null}', '$.a') AS v; +SELECT JSON_VALUE('{"a":null}', '$.a') IS NULL AS is_null; + +--echo # +--echo # 5. Arguments that are not strings. The document argument is taken +--echo # through its string value, so the conversion is part of the result. +--echo # + +SELECT JSON_VALID(1) AS v, JSON_VALID(1.5) AS d, JSON_VALID(TRUE) AS b; +SELECT JSON_TYPE(1) AS v, JSON_TYPE(1.5) AS d; +SELECT JSON_EXTRACT(1, '$') AS v; +SELECT JSON_SET(1, '$', 2) AS v; +SELECT JSON_DEPTH(1) AS v; +SELECT JSON_VALID(DATE'2020-01-01') AS v; +SELECT JSON_EXTRACT(DATE'2020-01-01', '$') AS v; +--echo # numeric and temporal values as inserted values, not documents +SELECT JSON_ARRAY(1, 1.5, -0.0, 1e300, DATE'2020-01-01', TIME'10:20:30') AS v; +SELECT JSON_OBJECT('i', 1, 'd', 1.5, 'dt', DATE'2020-01-01') AS v; +SELECT JSON_SET('{}', '$.i', 1, '$.d', 1.5, '$.dt', DATE'2020-01-01') AS v; +--echo # binary values +SELECT JSON_ARRAY(X'41') AS v, HEX(JSON_ARRAY(X'41')) AS h; +SELECT JSON_ARRAY(X'00') AS v, HEX(JSON_ARRAY(X'00')) AS h; +SELECT JSON_VALID(X'7B7D') AS v; + +--echo # +--echo # 6. Hostile strings: control characters and bytes that JSON requires +--echo # to be escaped. These pin what the constructors emit today for +--echo # values that cannot be carried literally. +--echo # + +--echo # --- control characters through the constructors --- +SELECT HEX(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(9 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(10 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(13 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(27 USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CHAR(31 USING utf8mb4))) AS h; +SELECT HEX(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS h; +--echo # a control character in the KEY position +SELECT HEX(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS h; +--echo # is the constructor output accepted back as a document? +SELECT JSON_VALID(JSON_ARRAY(CHAR(1 USING utf8mb4))) AS v; +SELECT JSON_VALID(JSON_OBJECT('k', CHAR(1 USING utf8mb4))) AS v; +SELECT JSON_VALID(JSON_OBJECT(CONCAT('k', CHAR(1 USING utf8mb4)), 1)) AS v; +--echo # the same values through the mutators +SELECT HEX(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS h; +SELECT JSON_VALID(JSON_SET('{}', '$.a', CHAR(1 USING utf8mb4))) AS v; +SELECT HEX(JSON_ARRAY_APPEND('[]', '$', CHAR(1 USING utf8mb4))) AS h; +--echo # and through JSON_QUOTE, which escapes for exactly this purpose +SELECT HEX(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS h; +SELECT HEX(JSON_QUOTE(CHAR(9 USING utf8mb4))) AS h; +SELECT JSON_VALID(JSON_QUOTE(CHAR(1 USING utf8mb4))) AS v; + +--echo # --- quote and backslash edges --- +SELECT JSON_ARRAY('a"b') AS v, HEX(JSON_ARRAY('a"b')) AS h; +SELECT JSON_ARRAY('a\\b') AS v, HEX(JSON_ARRAY('a\\b')) AS h; +SELECT JSON_ARRAY('a\\"b') AS v; +SELECT JSON_ARRAY('"') AS v; +SELECT JSON_ARRAY('\\') AS v; +SELECT JSON_OBJECT('a"b', 1) AS v; +SELECT JSON_OBJECT('a\\b', 1) AS v; +SELECT JSON_VALID(JSON_ARRAY('a"b')) AS q, JSON_VALID(JSON_ARRAY('a\\b')) AS bs; +SELECT JSON_VALID(JSON_OBJECT('a"b', 1)) AS q, JSON_VALID(JSON_OBJECT('a\\b', 1)) AS bs; +SELECT JSON_QUOTE('a"b') AS v, JSON_QUOTE('a\\b') AS v2; +SELECT JSON_UNQUOTE('"a\\\\b"') AS v; +SELECT JSON_UNQUOTE(JSON_QUOTE('a"b')) AS v; +--echo # a value that already looks like a quoted JSON string +SELECT JSON_ARRAY('"quoted"') AS v; +SELECT JSON_ARRAY('{"a":1}') AS v; +SELECT JSON_VALID(JSON_ARRAY('{"a":1}')) AS v; + +--echo # --- byte sequences that are not valid in the connection charset --- +SET NAMES utf8mb4; +SELECT HEX(JSON_ARRAY(_utf8mb4 X'41')) AS h; +--error ER_INVALID_CHARACTER_STRING +SELECT HEX(JSON_ARRAY(_utf8mb4 X'FF')) AS h; +--error ER_INVALID_CHARACTER_STRING +SELECT HEX(JSON_ARRAY(_utf8mb4 X'C3')) AS h; +--echo # the same bytes carried in as binary and converted +SELECT HEX(JSON_ARRAY(CONVERT(X'FF' USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CONVERT(X'C3' USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CONVERT(X'C328' USING utf8mb4))) AS h; +SELECT HEX(JSON_ARRAY(CONVERT(X'EDA080' USING utf8mb4))) AS h; +SELECT HEX(JSON_QUOTE(CONVERT(X'FF' USING utf8mb4))) AS h; +--echo # invalid bytes inside a document argument +SELECT JSON_VALID(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4)) AS v; +SELECT HEX(JSON_SET(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.b', 1)) AS h; +SELECT HEX(JSON_EXTRACT(CONVERT(CONCAT(X'7B2261223A22', X'FF', X'227D') USING utf8mb4), '$.a')) AS h; +--echo # an escape sequence naming a code point that utf8mb4 cannot encode +SELECT HEX(JSON_UNQUOTE('"\\ud800"')) AS h; +SELECT HEX(JSON_UNQUOTE('"\\ud800\\udc00"')) AS h; +SELECT JSON_VALID('"\\ud800"') AS v; + +--echo # +--echo # 7. Strict and non-strict sql_mode. The document argument never +--echo # depends on sql_mode, but stores of the result do; both are +--echo # recorded so that a later change of the store rule is visible. +--echo # + +CREATE TABLE t1 (a VARCHAR(10)); +SET @@sql_mode=''; +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT a, JSON_VALID(a) AS still_valid FROM t1; +DELETE FROM t1; +SET @@sql_mode='STRICT_ALL_TABLES'; +--error ER_DATA_TOO_LONG +INSERT INTO t1 VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT COUNT(*) AS rows_stored FROM t1; +SET @@sql_mode=DEFAULT; +DROP TABLE t1; + +--echo # +--echo # 8. Errors raised before a document can be built at all, so that +--echo # no partially built result is ever returned as a value. +--echo # + +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_OBJECT('a') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_SET('{}') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_SET('{}', '$.a') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_REMOVE('{}') AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_KEYS() AS v; +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT JSON_MERGE('[1]') AS v; diff --git a/mysql-test/main/func_json_key_span.result b/mysql-test/main/func_json_key_span.result new file mode 100644 index 0000000000000..3ae8b67cf23bb --- /dev/null +++ b/mysql-test/main/func_json_key_span.result @@ -0,0 +1,155 @@ +# +# 1. A plain key, and one holding a quote that makes nonsense of +# the document +# +SELECT JSON_INSERT('{}', '$.b', 1) AS plain; +plain +{"b": 1} +SELECT JSON_INSERT(JSON_OBJECT(), '$.b', 1) AS plain_is_valid; +plain_is_valid +{"b": 1} +SELECT JSON_INSERT('{}', '$.a"b', 1) AS broken; +broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 +SELECT JSON_INSERT(JSON_OBJECT(), '$.a"b', 1) AS broken_is_valid; +broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 +# +# 2. A key that closes its own member and opens another, which +# reads as a document saying something the caller did not +# write and is returned all the same +# +SELECT JSON_INSERT('{}', '$.a":1,"b', 1) AS smuggled; +smuggled +{"a": 1, "b": 1} +SELECT JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1) AS smuggled_is_valid; +smuggled_is_valid +{"a": 1, "b": 1} +# +# 3. A backslash begins an escape, so a key holding one may or +# may not still end where it looks like it ends +# +SELECT JSON_INSERT('{}', '$.a\\b', 1) AS backslash; +backslash +{"a\b": 1} +SELECT JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1) AS backslash_is_valid; +backslash_is_valid +{"a\b": 1} +# +# 4. The same keys in character sets that write a letter in more +# than one byte +# +SET @old_coll= @@collation_connection; +# All four of the keys above run again here, and the last two +# are the reason the section exists: a key that closes its own +# member, and a key holding a backslash, are where a reader +# stepping by bytes and one stepping by characters part +# company. Running only the plain and the quoted key would +# leave that untried. +SET @@collation_connection='ucs2_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS ucs2_plain; +ucs2_plain +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS ucs2_plain_is_valid; +ucs2_plain_is_valid +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS ucs2_broken; +ucs2_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS ucs2_broken_is_valid; +ucs2_broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS ucs2_smuggled; +ucs2_smuggled +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) +AS ucs2_smuggled_is_valid; +ucs2_smuggled_is_valid +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS ucs2_backslash; +ucs2_backslash +007B00220061005C00620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) +AS ucs2_backslash_is_valid; +ucs2_backslash_is_valid +007B00220061005C00620022003A00200031007D +SET @@collation_connection='utf16_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf16_plain; +utf16_plain +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf16_plain_is_valid; +utf16_plain_is_valid +007B002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf16_broken; +utf16_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf16_broken_is_valid; +utf16_broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 8 +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf16_smuggled; +utf16_smuggled +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) +AS utf16_smuggled_is_valid; +utf16_smuggled_is_valid +007B002200610022003A00200031002C0020002200620022003A00200031007D +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf16_backslash; +utf16_backslash +007B00220061005C00620022003A00200031007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) +AS utf16_backslash_is_valid; +utf16_backslash_is_valid +007B00220061005C00620022003A00200031007D +SET @@collation_connection='utf32_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf32_plain; +utf32_plain +0000007B0000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf32_plain_is_valid; +utf32_plain_is_valid +0000007B0000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf32_broken; +utf32_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 16 +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf32_broken_is_valid; +utf32_broken_is_valid +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 16 +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf32_smuggled; +utf32_smuggled +0000007B0000002200000061000000220000003A00000020000000310000002C000000200000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) +AS utf32_smuggled_is_valid; +utf32_smuggled_is_valid +0000007B0000002200000061000000220000003A00000020000000310000002C000000200000002200000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf32_backslash; +utf32_backslash +0000007B00000022000000610000005C00000062000000220000003A00000020000000310000007D +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) +AS utf32_backslash_is_valid; +utf32_backslash_is_valid +0000007B00000022000000610000005C00000062000000220000003A00000020000000310000007D +SET @@collation_connection= @old_coll; +# +# 5. Removing a key reads the path the same way +# +SELECT JSON_REMOVE('{"a":1,"b":2}', '$.a') AS removed; +removed +{"b": 2} +SELECT JSON_REMOVE(JSON_INSERT('{"b":2}', '$.a', 1), '$.a') AS removed_is_valid; +removed_is_valid +{"b": 2} diff --git a/mysql-test/main/func_json_key_span.test b/mysql-test/main/func_json_key_span.test new file mode 100644 index 0000000000000..7f9757dc61422 --- /dev/null +++ b/mysql-test/main/func_json_key_span.test @@ -0,0 +1,108 @@ +# +# What a path's last step is allowed to be, when it becomes a key in +# the document being written. +# +# The step is copied into the result between two quotes. A step that +# holds a quote of its own therefore closes the key early, and what +# follows it lands in the document as punctuation rather than as part +# of the name. Sometimes that makes nonsense of the document and +# sometimes it makes a perfectly good one saying something else, and +# which of the two it is depends on the whole result and not on the +# step - so a step that reaches past its quotes is not turned down, it +# is only left to the reading back to judge, exactly as before. +# +# The cases below are here because the reading back is now skipped for +# a document answering is_valid and is_nice. Each key is put to such a +# document and to one answering is_valid false, and the two must give +# the same answer. +# +# The wide character sets are the second half of it. A key is written +# in characters and not in bytes: 'b' in ucs2 is the two bytes 00 62, +# and anything reading those one at a time finds a control character +# where there is none. +# + +--echo # +--echo # 1. A plain key, and one holding a quote that makes nonsense of +--echo # the document +--echo # +SELECT JSON_INSERT('{}', '$.b', 1) AS plain; +SELECT JSON_INSERT(JSON_OBJECT(), '$.b', 1) AS plain_is_valid; +SELECT JSON_INSERT('{}', '$.a"b', 1) AS broken; +SELECT JSON_INSERT(JSON_OBJECT(), '$.a"b', 1) AS broken_is_valid; + +--echo # +--echo # 2. A key that closes its own member and opens another, which +--echo # reads as a document saying something the caller did not +--echo # write and is returned all the same +--echo # +SELECT JSON_INSERT('{}', '$.a":1,"b', 1) AS smuggled; +SELECT JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1) AS smuggled_is_valid; + +--echo # +--echo # 3. A backslash begins an escape, so a key holding one may or +--echo # may not still end where it looks like it ends +--echo # +SELECT JSON_INSERT('{}', '$.a\\b', 1) AS backslash; +SELECT JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1) AS backslash_is_valid; + +--echo # +--echo # 4. The same keys in character sets that write a letter in more +--echo # than one byte +--echo # +SET @old_coll= @@collation_connection; +# A wrapping view is created on a connection of its own, which does not +# carry the connection collation set below, so the answers would come back +# encoded in the default set instead of the wide one being tested. +--disable_view_protocol + +--echo # All four of the keys above run again here, and the last two +--echo # are the reason the section exists: a key that closes its own +--echo # member, and a key holding a backslash, are where a reader +--echo # stepping by bytes and one stepping by characters part +--echo # company. Running only the plain and the quoted key would +--echo # leave that untried. +SET @@collation_connection='ucs2_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS ucs2_plain; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS ucs2_plain_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS ucs2_broken; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS ucs2_broken_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS ucs2_smuggled; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) + AS ucs2_smuggled_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS ucs2_backslash; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) + AS ucs2_backslash_is_valid; + +SET @@collation_connection='utf16_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf16_plain; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf16_plain_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf16_broken; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf16_broken_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf16_smuggled; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) + AS utf16_smuggled_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf16_backslash; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) + AS utf16_backslash_is_valid; + +SET @@collation_connection='utf32_general_ci'; +SELECT HEX(JSON_INSERT('{}', '$.b', 1)) AS utf32_plain; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.b', 1)) AS utf32_plain_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a"b', 1)) AS utf32_broken; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a"b', 1)) AS utf32_broken_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a":1,"b', 1)) AS utf32_smuggled; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a":1,"b', 1)) + AS utf32_smuggled_is_valid; +SELECT HEX(JSON_INSERT('{}', '$.a\\b', 1)) AS utf32_backslash; +SELECT HEX(JSON_INSERT(JSON_OBJECT(), '$.a\\b', 1)) + AS utf32_backslash_is_valid; + +SET @@collation_connection= @old_coll; +--enable_view_protocol + +--echo # +--echo # 5. Removing a key reads the path the same way +--echo # +SELECT JSON_REMOVE('{"a":1,"b":2}', '$.a') AS removed; +SELECT JSON_REMOVE(JSON_INSERT('{"b":2}', '$.a', 1), '$.a') AS removed_is_valid; diff --git a/mysql-test/main/func_json_keys_typed.result b/mysql-test/main/func_json_keys_typed.result new file mode 100644 index 0000000000000..d422b0c26e77a --- /dev/null +++ b/mysql-test/main/func_json_keys_typed.result @@ -0,0 +1,327 @@ +# +# JSON_KEYS returns a JSON array, so a function given it in value +# position puts an array in rather than a string that reads as one. +# +SET NAMES utf8mb4; +# +# 1. Value position, in the functions that build a document. +# +SELECT JSON_ARRAY(JSON_KEYS('{"a":1,"b":2}')) AS v; +v +[["a", "b"]] +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"k": ["a", "b"]} +SELECT JSON_ARRAY(JSON_KEYS('{"a":1}'), JSON_KEYS('{"b":2,"c":3}')) AS v; +v +[["a"], ["b", "c"]] +SELECT JSON_OBJECT('one', JSON_KEYS('{"a":1}'), +'two', JSON_KEYS('{"b":2,"c":3}')) AS v; +v +{"one": ["a"], "two": ["b", "c"]} +# +# 2. Value position, in the functions that edit one. +# +SELECT JSON_SET('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"x": 1, "y": ["a", "b"]} +SELECT JSON_INSERT('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"x": 1, "y": ["a", "b"]} +SELECT JSON_REPLACE('{"x":1}', '$.x', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +{"x": ["a", "b"]} +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +[1, ["a", "b"]] +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_KEYS('{"a":1,"b":2}')) AS v; +v +[["a", "b"], 1] +# +# 3. The same shape from the functions that were already typed as +# returning a document, for comparison. +# +SELECT JSON_ARRAY(JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; +v +[[1, 2]] +SELECT JSON_ARRAY(JSON_QUERY('{"a":[1,2]}','$.a')) AS v; +v +[[1,2]] +SELECT JSON_SET('{"x":1}', '$.y', JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; +v +{"x": 1, "y": [1, 2]} +# +# 4. Document position is unchanged: the value was always a document +# when it was read as one. +# +SELECT JSON_VALID(JSON_KEYS('{"a":1,"b":2}')) AS valid, +JSON_LENGTH(JSON_KEYS('{"a":1,"b":2}')) AS len, +JSON_TYPE(JSON_KEYS('{"a":1,"b":2}')) AS type; +valid len type +1 2 ARRAY +SELECT JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$[0]') AS v; +v +"a" +SELECT JSON_DEPTH(JSON_KEYS('{"a":{"b":1},"c":2}')) AS v; +v +2 +# the workaround that used to be needed still gives the same document +SELECT JSON_ARRAY(JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$')) AS v; +v +[["a", "b"]] +# +# 5. Keys that have to be written with an escape. The key goes in as +# it was written in the document it came from, so the array is +# still one however the key is written. +# +SELECT JSON_KEYS('{"a\\"b":1,"c\\\\d":2}') AS v; +v +["a\"b", "c\\d"] +SELECT JSON_VALID(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS ok; +ok +1 +SELECT JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS v; +v +[["a\"b", "c\\d"]] +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}'))) AS ok; +ok +1 +# an escaped control character and an escaped code point +SELECT JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}')) AS v; +v +[["a\nb", "c\u00e4d"]] +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}'))) AS ok; +ok +1 +# a key written in characters of its own rather than in escapes +SELECT HEX(JSON_ARRAY(JSON_KEYS(_utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) +AS h; +h +5B5B22C3A4222C2022C3B6225D5D +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS( +_utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) AS ok; +ok +1 +# an empty key +SELECT JSON_ARRAY(JSON_KEYS('{"":1,"b":2}')) AS v; +v +[["", "b"]] +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"":1,"b":2}'))) AS ok; +ok +1 +# +# 6. The answers that are not arrays. An object with no keys is an +# empty array; anything that is not an object is NULL, and a NULL +# goes in as a null. +# +SELECT JSON_ARRAY(JSON_KEYS('{}')) AS v; +v +[[]] +SELECT JSON_ARRAY(JSON_KEYS('[1,2]')) AS v; +v +[null] +SELECT JSON_ARRAY(JSON_KEYS('7')) AS v; +v +[null] +SELECT JSON_ARRAY(JSON_KEYS(NULL)) AS v; +v +[null] +SELECT JSON_OBJECT('k', JSON_KEYS('[1,2]')) AS v; +v +{"k": null} +# input that does not read as a document at all +SELECT JSON_ARRAY(JSON_KEYS('{"a":')) AS v; +v +[null] +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +# +# 7. The two-argument form, which reaches an object further in. +# +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +v +[["b", "c"]] +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +v +{"k": ["b", "c"]} +# a path that reaches nothing +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1}}', '$.zz')) AS v; +v +[null] +# +# 8. Nested one inside the other. +# +SELECT JSON_ARRAY(JSON_KEYS(JSON_OBJECT('a', 1, 'b', 2))) AS v; +v +[["a", "b"]] +SELECT JSON_KEYS(JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}'))) AS v; +v +["k"] +# +# 9. Over rows, including through an aggregate and through a +# temporary table. +# +CREATE TABLE t1 (id INT, j VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2,"c":3}'); +SELECT id, JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY id; +id v +1 [["a"]] +2 [["b", "c"]] +SELECT JSON_ARRAYAGG(JSON_KEYS(j)) AS v FROM t1; +v +[["a"],["b", "c"]] +SELECT JSON_OBJECTAGG(id, JSON_KEYS(j)) AS v FROM t1; +v +{"1":["a"], "2":["b", "c"]} +# materialised into a temporary table on the way out +SELECT JSON_ARRAY(k) AS v FROM (SELECT JSON_KEYS(j) AS k FROM t1) d ORDER BY v; +v +[["a"]] +[["b", "c"]] +# MAX() is not typed by what it is taken over, so its result is +# quoted here as it is for any other document producing function +SELECT JSON_ARRAY(MAX(JSON_KEYS(j))) AS v FROM t1 GROUP BY id ORDER BY v; +v +["[\"a\"]"] +["[\"b\", \"c\"]"] +SELECT JSON_ARRAY(MAX(JSON_EXTRACT(j, '$'))) AS v FROM t1 GROUP BY id ORDER BY v; +v +["{\"a\": 1}"] +["{\"b\": 2, \"c\": 3}"] +SELECT DISTINCT JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY v; +v +[["a"]] +[["b", "c"]] +# the column a result set of it makes, and the column the already +# typed function makes beside it +CREATE TABLE t2 AS SELECT JSON_KEYS(j) AS k FROM t1; +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `k` varchar(64) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT JSON_ARRAY(k) AS v FROM t2 ORDER BY v; +v +["[\"a\"]"] +["[\"b\", \"c\"]"] +DROP TABLE t2; +CREATE TABLE t3 AS SELECT JSON_EXTRACT(j, '$') AS k FROM t1; +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `k` varchar(128) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE t3; +DROP TABLE t1; +# +# 10. The result metadata, which is what decides all of the above. +# +SELECT JSON_KEYS('{"a":1,"b":2}') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 52 10 Y 0 39 45 +v +["a", "b"] +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 (format=json) 88 6 Y 0 39 45 +v +[1, 2] +SELECT JSON_QUOTE('a') AS v; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def v 253 32 3 Y 128 39 45 +v +"a" +# +# 11. JSON_QUOTE is documented to return a string and still does. +# +SELECT JSON_ARRAY(JSON_QUOTE('a')) AS v; +v +["\"a\""] +SELECT JSON_OBJECT('k', JSON_QUOTE('a')) AS v; +v +{"k": "\"a\""} +# +# 12. The array is attested as well as typed, so a splice of it +# goes in without being read again - which no result here can +# show and Json_scans can, and does in func_json_scan_count. +# +# What stands in the way of that is a character set this +# cannot attest to, and there are two kinds of those. +# +# A set that cannot encode a bracket cannot encode a brace +# either, so no array is ever answered in it and the question +# never comes up - swe7, below. +# +# But the question actually asked is whether the set is +# ASCII-compatible, and a set can decode every ASCII byte to +# itself and still be flagged as not. sjis is one: it parses +# a document, answers a real array, and has the mark withheld +# from it all the same, so a splice reads that array again. +# The array is a perfectly good document either way; what is +# lost is the reading it was meant to save. +# +# The counts that say which of the two happened are in +# func_json_keys_typed_scans, which a debug build runs. +# +SELECT JSON_KEYS(CONVERT('{"a":1}' USING swe7)) AS no_answer; +no_answer +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 1 +SELECT JSON_VALID(CONVERT('{"a":1}' USING swe7)) AS nor_a_document; +nor_a_document +0 +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# sjis: a real answer, and one the mark is withheld from +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +SELECT JSON_KEYS(j) AS answered, JSON_VALID(JSON_KEYS(j)) AS ok FROM ts; +answered ok +["a", "b"] 1 +DROP TABLE ts; +# +# 13. What the answer is attested does not say the argument was +# a document all the way through. This reads an object and +# stops at its end, so a break past that end is never met - and +# one it does meet refuses the answer whole. +# +# never met: the break is past the object, or past the sub-object +# the path chose +SELECT JSON_KEYS('{"a":1} rubbish') AS trailing_text; +trailing_text +["a"] +SELECT JSON_KEYS('{"a":1}{"b":2}') AS second_document; +second_document +["a"] +SELECT JSON_KEYS('{"a":1}}') AS trailing_brace; +trailing_brace +["a"] +SELECT JSON_KEYS('{"a":{"z":1} "b":2}', '$.a') AS break_after_the_step; +break_after_the_step +["z"] +# and the answers are documents, which is what the mark says +SELECT JSON_VALID(JSON_KEYS('{"a":1} rubbish')) AS ok; +ok +1 +SELECT JSON_ARRAY(JSON_KEYS('{"a":1} rubbish')) AS spliced; +spliced +[["a"]] +# met: refused whole, and reported where it happened +SELECT JSON_KEYS('{"a":1 "b":2}') AS break_inside; +break_inside +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 8 +SELECT JSON_KEYS('{"a":') AS ends_early; +ends_early +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_keys' +SELECT JSON_KEYS('{"a":{"z":1 "y":2}}', '$.a') AS break_inside_the_step; +break_inside_the_step +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_keys' at position 13 diff --git a/mysql-test/main/func_json_keys_typed.test b/mysql-test/main/func_json_keys_typed.test new file mode 100644 index 0000000000000..4d1c9034e02e5 --- /dev/null +++ b/mysql-test/main/func_json_keys_typed.test @@ -0,0 +1,212 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # JSON_KEYS returns a JSON array, so a function given it in value +--echo # position puts an array in rather than a string that reads as one. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Value position, in the functions that build a document. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_ARRAY(JSON_KEYS('{"a":1}'), JSON_KEYS('{"b":2,"c":3}')) AS v; +SELECT JSON_OBJECT('one', JSON_KEYS('{"a":1}'), + 'two', JSON_KEYS('{"b":2,"c":3}')) AS v; + +--echo # +--echo # 2. Value position, in the functions that edit one. +--echo # + +SELECT JSON_SET('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_INSERT('{"x":1}', '$.y', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_REPLACE('{"x":1}', '$.x', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_KEYS('{"a":1,"b":2}')) AS v; +SELECT JSON_ARRAY_INSERT('[1]', '$[0]', JSON_KEYS('{"a":1,"b":2}')) AS v; + +--echo # +--echo # 3. The same shape from the functions that were already typed as +--echo # returning a document, for comparison. +--echo # + +SELECT JSON_ARRAY(JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; +SELECT JSON_ARRAY(JSON_QUERY('{"a":[1,2]}','$.a')) AS v; +SELECT JSON_SET('{"x":1}', '$.y', JSON_EXTRACT('{"a":[1,2]}','$.a')) AS v; + +--echo # +--echo # 4. Document position is unchanged: the value was always a document +--echo # when it was read as one. +--echo # + +SELECT JSON_VALID(JSON_KEYS('{"a":1,"b":2}')) AS valid, + JSON_LENGTH(JSON_KEYS('{"a":1,"b":2}')) AS len, + JSON_TYPE(JSON_KEYS('{"a":1,"b":2}')) AS type; +SELECT JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$[0]') AS v; +SELECT JSON_DEPTH(JSON_KEYS('{"a":{"b":1},"c":2}')) AS v; +--echo # the workaround that used to be needed still gives the same document +SELECT JSON_ARRAY(JSON_EXTRACT(JSON_KEYS('{"a":1,"b":2}'), '$')) AS v; + +--echo # +--echo # 5. Keys that have to be written with an escape. The key goes in as +--echo # it was written in the document it came from, so the array is +--echo # still one however the key is written. +--echo # + +SELECT JSON_KEYS('{"a\\"b":1,"c\\\\d":2}') AS v; +SELECT JSON_VALID(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS ok; +SELECT JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}')) AS v; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\"b":1,"c\\\\d":2}'))) AS ok; +--echo # an escaped control character and an escaped code point +SELECT JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}')) AS v; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"a\\nb":1,"c\\u00e4d":2}'))) AS ok; +--echo # a key written in characters of its own rather than in escapes +SELECT HEX(JSON_ARRAY(JSON_KEYS(_utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) + AS h; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS( + _utf8mb4 X'7B22C3A4223A312C22C3B6223A327D'))) AS ok; +--echo # an empty key +SELECT JSON_ARRAY(JSON_KEYS('{"":1,"b":2}')) AS v; +SELECT JSON_VALID(JSON_ARRAY(JSON_KEYS('{"":1,"b":2}'))) AS ok; + +--echo # +--echo # 6. The answers that are not arrays. An object with no keys is an +--echo # empty array; anything that is not an object is NULL, and a NULL +--echo # goes in as a null. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS('{}')) AS v; +SELECT JSON_ARRAY(JSON_KEYS('[1,2]')) AS v; +SELECT JSON_ARRAY(JSON_KEYS('7')) AS v; +SELECT JSON_ARRAY(JSON_KEYS(NULL)) AS v; +SELECT JSON_OBJECT('k', JSON_KEYS('[1,2]')) AS v; +--echo # input that does not read as a document at all +SELECT JSON_ARRAY(JSON_KEYS('{"a":')) AS v; + +--echo # +--echo # 7. The two-argument form, which reaches an object further in. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +SELECT JSON_OBJECT('k', JSON_KEYS('{"a":{"b":1,"c":2}}', '$.a')) AS v; +--echo # a path that reaches nothing +SELECT JSON_ARRAY(JSON_KEYS('{"a":{"b":1}}', '$.zz')) AS v; + +--echo # +--echo # 8. Nested one inside the other. +--echo # + +SELECT JSON_ARRAY(JSON_KEYS(JSON_OBJECT('a', 1, 'b', 2))) AS v; +SELECT JSON_KEYS(JSON_OBJECT('k', JSON_KEYS('{"a":1,"b":2}'))) AS v; + +--echo # +--echo # 9. Over rows, including through an aggregate and through a +--echo # temporary table. +--echo # + +CREATE TABLE t1 (id INT, j VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2,"c":3}'); + +SELECT id, JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY id; +SELECT JSON_ARRAYAGG(JSON_KEYS(j)) AS v FROM t1; +SELECT JSON_OBJECTAGG(id, JSON_KEYS(j)) AS v FROM t1; +--echo # materialised into a temporary table on the way out +SELECT JSON_ARRAY(k) AS v FROM (SELECT JSON_KEYS(j) AS k FROM t1) d ORDER BY v; +--echo # MAX() is not typed by what it is taken over, so its result is +--echo # quoted here as it is for any other document producing function +SELECT JSON_ARRAY(MAX(JSON_KEYS(j))) AS v FROM t1 GROUP BY id ORDER BY v; +SELECT JSON_ARRAY(MAX(JSON_EXTRACT(j, '$'))) AS v FROM t1 GROUP BY id ORDER BY v; +SELECT DISTINCT JSON_ARRAY(JSON_KEYS(j)) AS v FROM t1 ORDER BY v; + +--echo # the column a result set of it makes, and the column the already +--echo # typed function makes beside it +CREATE TABLE t2 AS SELECT JSON_KEYS(j) AS k FROM t1; +SHOW CREATE TABLE t2; +SELECT JSON_ARRAY(k) AS v FROM t2 ORDER BY v; +DROP TABLE t2; +CREATE TABLE t3 AS SELECT JSON_EXTRACT(j, '$') AS k FROM t1; +SHOW CREATE TABLE t3; +DROP TABLE t3; + +DROP TABLE t1; + +--echo # +--echo # 10. The result metadata, which is what decides all of the above. +--echo # + +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_KEYS('{"a":1,"b":2}') AS v; +SELECT JSON_EXTRACT('{"a":[1,2]}','$.a') AS v; +SELECT JSON_QUOTE('a') AS v; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +--echo # +--echo # 11. JSON_QUOTE is documented to return a string and still does. +--echo # + +SELECT JSON_ARRAY(JSON_QUOTE('a')) AS v; +SELECT JSON_OBJECT('k', JSON_QUOTE('a')) AS v; + +--echo # +--echo # 12. The array is attested as well as typed, so a splice of it +--echo # goes in without being read again - which no result here can +--echo # show and Json_scans can, and does in func_json_scan_count. +--echo # +--echo # What stands in the way of that is a character set this +--echo # cannot attest to, and there are two kinds of those. +--echo # +--echo # A set that cannot encode a bracket cannot encode a brace +--echo # either, so no array is ever answered in it and the question +--echo # never comes up - swe7, below. +--echo # +--echo # But the question actually asked is whether the set is +--echo # ASCII-compatible, and a set can decode every ASCII byte to +--echo # itself and still be flagged as not. sjis is one: it parses +--echo # a document, answers a real array, and has the mark withheld +--echo # from it all the same, so a splice reads that array again. +--echo # The array is a perfectly good document either way; what is +--echo # lost is the reading it was meant to save. +--echo # +--echo # The counts that say which of the two happened are in +--echo # func_json_keys_typed_scans, which a debug build runs. +--echo # + +SELECT JSON_KEYS(CONVERT('{"a":1}' USING swe7)) AS no_answer; +SELECT JSON_VALID(CONVERT('{"a":1}' USING swe7)) AS nor_a_document; + +--echo # sjis: a real answer, and one the mark is withheld from +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +SELECT JSON_KEYS(j) AS answered, JSON_VALID(JSON_KEYS(j)) AS ok FROM ts; +DROP TABLE ts; + +--echo # +--echo # 13. What the answer is attested does not say the argument was +--echo # a document all the way through. This reads an object and +--echo # stops at its end, so a break past that end is never met - and +--echo # one it does meet refuses the answer whole. +--echo # + +--echo # never met: the break is past the object, or past the sub-object +--echo # the path chose +SELECT JSON_KEYS('{"a":1} rubbish') AS trailing_text; +SELECT JSON_KEYS('{"a":1}{"b":2}') AS second_document; +SELECT JSON_KEYS('{"a":1}}') AS trailing_brace; +SELECT JSON_KEYS('{"a":{"z":1} "b":2}', '$.a') AS break_after_the_step; +--echo # and the answers are documents, which is what the mark says +SELECT JSON_VALID(JSON_KEYS('{"a":1} rubbish')) AS ok; +SELECT JSON_ARRAY(JSON_KEYS('{"a":1} rubbish')) AS spliced; + +--echo # met: refused whole, and reported where it happened +SELECT JSON_KEYS('{"a":1 "b":2}') AS break_inside; +SELECT JSON_KEYS('{"a":') AS ends_early; +SELECT JSON_KEYS('{"a":{"z":1 "y":2}}', '$.a') AS break_inside_the_step; diff --git a/mysql-test/main/func_json_keys_typed_scans.result b/mysql-test/main/func_json_keys_typed_scans.result new file mode 100644 index 0000000000000..24a80ccc5f4f7 --- /dev/null +++ b/mysql-test/main/func_json_keys_typed_scans.result @@ -0,0 +1,34 @@ +# +# The reading a JSON_KEYS array saves where it is spliced, and the +# reading it does not save where the set it is written in cannot +# be attested. Nothing but the count shows either of them. +# +# sjis decodes every ASCII byte to itself and is flagged as not +# ASCII-compatible all the same, so the array it answers is a +# perfectly good document with no mark on it, and the splice reads +# it again. The constructor round it then reads its own answer +# once more, that flag being the only warning it has that a set +# might not write the brackets as brackets - a set that says +# nothing against itself is not read at all. utf8mb4 stands +# beside it, over the same document, where no reading happens on +# either count. +# +# What the answers themselves look like is in +# func_json_keys_typed. +# +SET NAMES utf8mb4; +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +CREATE TABLE tu (j TEXT) CHARACTER SET utf8mb4; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +INSERT INTO tu VALUES ('{"a":1,"b":2}'); +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM ts; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM tu; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +DROP TABLE ts, tu; diff --git a/mysql-test/main/func_json_keys_typed_scans.test b/mysql-test/main/func_json_keys_typed_scans.test new file mode 100644 index 0000000000000..90901e94fc7ad --- /dev/null +++ b/mysql-test/main/func_json_keys_typed_scans.test @@ -0,0 +1,46 @@ +--source include/have_debug.inc +--source include/have_utf8mb4.inc + +--echo # +--echo # The reading a JSON_KEYS array saves where it is spliced, and the +--echo # reading it does not save where the set it is written in cannot +--echo # be attested. Nothing but the count shows either of them. +--echo # +--echo # sjis decodes every ASCII byte to itself and is flagged as not +--echo # ASCII-compatible all the same, so the array it answers is a +--echo # perfectly good document with no mark on it, and the splice reads +--echo # it again. The constructor round it then reads its own answer +--echo # once more, that flag being the only warning it has that a set +--echo # might not write the brackets as brackets - a set that says +--echo # nothing against itself is not read at all. utf8mb4 stands +--echo # beside it, over the same document, where no reading happens on +--echo # either count. +--echo # +--echo # What the answers themselves look like is in +--echo # func_json_keys_typed. +--echo # + +SET NAMES utf8mb4; + +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol + +CREATE TABLE ts (j TEXT) CHARACTER SET sjis; +CREATE TABLE tu (j TEXT) CHARACTER SET utf8mb4; +INSERT INTO ts VALUES ('{"a":1,"b":2}'); +INSERT INTO tu VALUES ('{"a":1,"b":2}'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM ts; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(j)) FROM tu; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE ts, tu; + +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_marks.result b/mysql-test/main/func_json_marks.result new file mode 100644 index 0000000000000..7dbf5843a2cec --- /dev/null +++ b/mysql-test/main/func_json_marks.result @@ -0,0 +1,380 @@ +# +# 1. Building a document out of values answering is_valid +# +SELECT JSON_ARRAY() AS empty_array, +JSON_OBJECT() AS empty_object; +empty_array empty_object +[] {} +SELECT JSON_ARRAY(1, 'two', NULL, TRUE, 3.5) AS scalars; +scalars +[1, "two", null, true, 3.5] +SELECT JSON_OBJECT('a', 1, 'b', 'two', 'c', NULL) AS pairs; +pairs +{"a": 1, "b": "two", "c": null} +SELECT JSON_ARRAY(JSON_ARRAY(1, 2), JSON_OBJECT('a', 1)) AS nested; +nested +[[1, 2], {"a": 1}] +SELECT JSON_OBJECT('k', JSON_ARRAY(1, 2)) AS nested_in_object; +nested_in_object +{"k": [1, 2]} +SELECT JSON_ARRAY(JSON_SET('{"a": 1}', '$.b', 2)) AS edited_then_built; +edited_then_built +[{"a": 1, "b": 2}] +SELECT JSON_ARRAY(JSON_EXTRACT('{"a": [1, 2]}', '$.a')) AS extracted_then_built; +extracted_then_built +[[1, 2]] +# +# 2. Building a document out of values that cannot be +# +# A JSON column holds whatever was in it when the check was off, +# and nothing between the column and here reads it. The value +# goes in as it stands, as it always has, and the note is what +# says so. +# +CREATE TABLE tj (id INT, j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tj VALUES (1, '{"a": 1}'), (2, '{"a":1,'), (3, ''), (4, '{"a":1}'); +SET SESSION check_constraint_checks = ON; +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 1)) AS from_good; +from_good +[{"a": 1}] +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 2)) AS from_broken; +from_broken +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 3)) AS from_empty; +from_empty +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +SELECT JSON_OBJECT('k', (SELECT j FROM tj WHERE id = 2)) AS from_broken_object; +from_broken_object +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_object' +# +# The compact one reads perfectly well and is simply not +# written the way this function writes. +# +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 4)) AS from_compact; +from_compact +[{"a":1}] +# +# 3. Editing a document +# +SELECT JSON_SET('{"a": 1}', '$.b', 2) AS set_done, +JSON_INSERT('{"a": 1}', '$.b', 2) AS insert_done, +JSON_REPLACE('{"a": 1}', '$.a', 2) AS replace_done; +set_done insert_done replace_done +{"a": 1, "b": 2} {"a": 1, "b": 2} {"a": 2} +SELECT JSON_REMOVE('{"a": 1, "b": 2}', '$.a') AS removed, +JSON_ARRAY_APPEND('[1]', '$', 2) AS appended, +JSON_ARRAY_INSERT('[1, 3]', '$[1]', 2) AS inserted; +removed appended inserted +{"b": 2} [1, 2] [1, 2, 3] +SELECT JSON_MERGE('[1]', '[2]') AS merged, +JSON_MERGE_PATCH('{"a": 1}', '{"b": 2}') AS patched; +merged patched +[1, 2] {"a": 1, "b": 2} +SELECT JSON_SET((SELECT j FROM tj WHERE id = 2), '$.b', 2) AS set_over_broken; +set_over_broken +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SELECT JSON_SET('{"a": 1}', '$.b', (SELECT j FROM tj WHERE id = 2)) +AS set_with_broken; +set_with_broken +NULL +Warnings: +Note 4037 Unexpected end of JSON text in argument 3 to function 'json_set' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_set' at position 21 +# +# 4. Reading a document out of another one +# +SELECT JSON_EXTRACT('{"a": [1, 2], "b": 3}', '$.a') AS extracted, +JSON_EXTRACT('{"a": 1, "b": 2}', '$.a', '$.b') AS extracted_two; +extracted extracted_two +[1, 2] [1, 2] +SELECT JSON_QUERY('{"a": [1,2]}', '$.a') AS queried; +queried +[1,2] +SELECT JSON_KEYS('{"a": 1, "b": 2}') AS keys_of; +keys_of +["a", "b"] +SELECT JSON_SEARCH('{"a": "x", "b": "x"}', 'all', 'x') AS searched_all, +JSON_SEARCH('{"a": "x"}', 'one', 'x') AS searched_one; +searched_all searched_one +["$.a", "$.b"] "$.a" +SELECT JSON_NORMALIZE('{"b": 1, "a": 2}') AS normalized; +normalized +{"a":2.0E0,"b":1.0E0} +SELECT JSON_COMPACT('{"a": [1, 2]}') AS compacted, +JSON_LOOSE('{"a":[1,2]}') AS loosened; +compacted loosened +{"a":[1,2]} {"a": [1, 2]} +SELECT JSON_DETAILED('{"a": 1}') AS detailed; +detailed +{ + "a": 1 +} +# +# JSON_VALID reads its argument through val_json(), where the +# format functions pass the value straight through without touching +# it, so nothing here is written out afresh. +# +SELECT JSON_VALID(JSON_LOOSE('{"a":1}')) AS loose_forwarded, +JSON_VALID(JSON_COMPACT('{"a": 1}')) AS compact_forwarded, +JSON_VALID(JSON_LOOSE((SELECT j FROM tj WHERE id = 2))) +AS broken_forwarded; +loose_forwarded compact_forwarded broken_forwarded +1 1 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# 5. Building a document a row at a time +# +CREATE TABLE tg (g INT, v INT, s VARCHAR(16)); +INSERT INTO tg VALUES (1, 1, 'a'), (1, 2, 'b'), (2, 3, 'c'), (2, 4, 'd'); +SELECT g, JSON_ARRAYAGG(v) AS arr, JSON_OBJECTAGG(s, v) AS obj +FROM tg GROUP BY g ORDER BY g; +g arr obj +1 [1,2] {"a":1, "b":2} +2 [3,4] {"c":3, "d":4} +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS agg_then_built FROM tg; +agg_then_built +[[1,2,3,4]] +SELECT g, JSON_ARRAYAGG(v) AS arr FROM tg GROUP BY g WITH ROLLUP; +g arr +1 [1,2] +2 [3,4] +NULL [1,2,3,4] +# +# A group cut to fit group_concat_max_len is cut back to the +# last whole element, so every element in it is one a row of +# the group held and the row it stopped at is reported. What +# comes out is a document, but not the value the group has. +# A group built out of a broken column is complete too, and is +# not a document at all. +# +SET @save_len = @@group_concat_max_len; +SET SESSION group_concat_max_len = 10; +SELECT JSON_ARRAYAGG(s) AS cut FROM tg; +cut +["a","b"] +Warnings: +Warning 1260 Row 3 was cut by JSON_ARRAYAGG() +SET SESSION group_concat_max_len = @save_len; +SELECT JSON_ARRAYAGG(j) AS agg_broken FROM tj; +agg_broken +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_arrayagg' +SELECT JSON_OBJECTAGG(id, j) AS objagg_broken FROM tj; +objagg_broken +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_objectagg' +# +# 6. Character sets +# +# In swe7 the bytes that encode brackets and braces elsewhere +# encode national letters, so a document cannot be written in it +# at all. The bytes returned are the same ones as always. +# +SELECT HEX(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7, +JSON_VALID(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7_valid; +array_in_swe7 array_in_swe7_valid +NULL NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +SELECT HEX(JSON_ARRAYAGG(CONVERT(s USING swe7))) AS agg_in_swe7 FROM tg; +agg_in_swe7 +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_arrayagg' at position 1 +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) AS array_in_ucs2, +JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) +AS array_in_ucs2_valid; +array_in_ucs2 array_in_ucs2_valid +005B002200E90022005D 1 +SELECT JSON_VALID(JSON_OBJECT('k', CONVERT(_utf8mb4 0xC3A8 USING utf16))) +AS object_in_utf16_valid; +object_in_utf16_valid +1 +SELECT HEX(JSON_ARRAY(_binary 0x61)) AS array_in_binary; +array_in_binary +5B2261225D +# +# A character set whose characters are never a single byte +# encodes the punctuation properly, so a document can be written +# in it and these results really are documents. sjis is not +# such a set: it stores ASCII a byte at a time and puts a yen +# sign where the backslash belongs. +# +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32)) ) AS array_in_utf32, +JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32))) +AS array_in_utf32_valid; +array_in_utf32 array_in_utf32_valid +0000005B00000022000000E9000000220000005D 1 +SELECT JSON_VALID(JSON_ARRAY(CONVERT(s USING ucs2))) AS array_of_col_ucs2 +FROM tg ORDER BY v; +array_of_col_ucs2 +1 +1 +1 +1 +SELECT JSON_VALID(JSON_OBJECT(CONVERT(s USING utf16), v)) AS object_in_utf16 +FROM tg ORDER BY v; +object_in_utf16 +1 +1 +1 +1 +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(s USING ucs2))) AS agg_in_ucs2 +FROM tg GROUP BY g ORDER BY g; +agg_in_ucs2 +1 +1 +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(s USING utf16), v)) AS objectagg_utf16 +FROM tg GROUP BY g ORDER BY g; +objectagg_utf16 +1 +1 +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) AS backslash_in_sjis, +JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) +AS backslash_in_sjis_valid; +backslash_in_sjis backslash_in_sjis_valid +5B22815F815F225D 1 +# +# 7. Values that arrive through a conversion +# +# A conversion that lost nothing kept the characters it was +# given; one that had to put a question mark in its place did +# not, and what it produced is not the value that went in. +# +SELECT JSON_VALID(CONVERT(JSON_OBJECT('a', 1) USING ucs2)) AS converted_valid; +converted_valid +1 +SELECT JSON_ARRAY(CONVERT(JSON_OBJECT('a', 1) USING utf8mb4)) AS converted_built; +converted_built +[{"a": 1}] +SELECT HEX(CONVERT(JSON_ARRAY(_utf8mb4 0xC3A9) USING swe7)) AS converted_lossy; +converted_lossy +3F2260223F +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x5B to 'swe7' +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), '$.b', +_utf8mb4 0xC3A9)) AS mixed_charsets, +JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), +'$.b', _utf8mb4 0xC3A9)) AS mixed_charsets_valid; +mixed_charsets mixed_charsets_valid +7B2261223A20312C202262223A2022E9227D 1 +# +# Conversion to binary keeps the bytes and relabels them, so a +# document written two bytes to the character comes out as twice +# as many characters, and is no longer one. Nothing is lost, so +# the copier has nothing to report about it. +# +SELECT HEX(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) +AS relabelled_bytes, +JSON_VALID(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) +AS relabelled_valid; +relabelled_bytes relabelled_valid +005B0031005D 0 +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +SELECT JSON_ARRAY(CONVERT(CONVERT(JSON_OBJECT('a', 1) USING ucs2) +USING binary)) AS built_from_relabelled; +built_from_relabelled +NULL +Warnings: +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +# +# 8. Values that arrive through a reference +# +SELECT JSON_ARRAY(x) AS through_derived +FROM (SELECT JSON_OBJECT('a', 1) AS x) d; +through_derived +[{"a": 1}] +SELECT JSON_ARRAY(x) AS through_broken_derived +FROM (SELECT j AS x FROM tj WHERE id = 2) d; +through_broken_derived +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_array' +CREATE VIEW v1 AS SELECT JSON_OBJECT('a', 1) AS x; +SELECT JSON_ARRAY(x) AS through_view FROM v1; +through_view +[{"a": 1}] +DROP VIEW v1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS through_aggregate_ref +FROM tg GROUP BY g ORDER BY g; +through_aggregate_ref +[[1,2]] +[[3,4]] +# +# A reference into a merged view on the inner side of an outer +# join returns NULL for a row the join filled in, without asking +# the item it refers to. The second row below is such a row and +# follows one that matched, so anything remembered from the +# first would still be there. +# +CREATE TABLE touter (id INT); +CREATE TABLE tinner (id INT); +INSERT INTO touter VALUES (1),(2); +INSERT INTO tinner VALUES (1); +CREATE VIEW vinner AS SELECT id, JSON_OBJECT('a', id) AS x FROM tinner; +SELECT touter.id, JSON_ARRAY(vinner.x) AS through_outer_join_view +FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +id through_outer_join_view +1 [{"a": 1}] +2 [null] +SELECT touter.id, JSON_VALID(vinner.x) AS through_outer_join_view_valid +FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +id through_outer_join_view_valid +1 1 +2 NULL +DROP VIEW vinner; +DROP TABLE tinner; +DROP TABLE touter; +DROP TABLE tg; +DROP TABLE tj; +# +# 9. Items that are copied +# +# Pushing a condition into a derived table copies the expression +# that produces the column, leaving two items in two places of +# the plan. A copy has produced no value of its own yet. +# +CREATE TABLE tc (a INT, b VARCHAR(32)); +INSERT INTO tc VALUES (1, 'x'), (2, 'y'); +SELECT * FROM (SELECT a, JSON_ARRAY(1, 2) AS j FROM tc) d WHERE d.j LIKE '[%'; +a j +1 [1, 2] +2 [1, 2] +SELECT * FROM (SELECT a, JSON_ARRAY(a, b) AS j FROM tc) d WHERE d.j LIKE '[%'; +a j +1 [1, "x"] +2 [2, "y"] +SELECT * FROM (SELECT a, JSON_OBJECT('k', a) AS j FROM tc) d +WHERE d.j = '{"k": 1}'; +a j +1 {"k": 1} +# +# A table definition holding a JSON expression is copied whole +# when the table is altered. +# +CREATE TABLE tv (a INT, j VARCHAR(64) AS (JSON_ARRAY(a)) VIRTUAL, +CHECK (JSON_VALID(JSON_OBJECT('a', a)))); +INSERT INTO tv (a) VALUES (1), (2); +ALTER TABLE tv ADD COLUMN c INT; +SELECT * FROM tv ORDER BY a; +a j c +1 [1] NULL +2 [2] NULL +DROP TABLE tv; +DROP TABLE tc; diff --git a/mysql-test/main/func_json_marks.test b/mysql-test/main/func_json_marks.test new file mode 100644 index 0000000000000..a49753fbdeb20 --- /dev/null +++ b/mysql-test/main/func_json_marks.test @@ -0,0 +1,251 @@ +# +# What a JSON function is able to say about the value it has just +# produced is recorded on the function and appears in no result, so +# there is nothing in this file for a .result to assert about it +# directly. What the file is for is to walk every one of those +# functions past the point where it says something, in a build that +# reads the value back and stops the server if the two disagree. +# +# The recorded output is the other half of it, and the stricter half: +# none of it may move. Nothing here is a new answer - every statement +# below is answered exactly as it was before any of this was written. +# +# Values that are not ASCII are written as hex literals so that this +# file does not depend on its own encoding. The ones used below are +# C3A9 = e with an acute accent, and C3A8 = e with a grave accent. +# + +--echo # +--echo # 1. Building a document out of values answering is_valid +--echo # +SELECT JSON_ARRAY() AS empty_array, + JSON_OBJECT() AS empty_object; +# A view keeps its body as printed text, and a boolean literal prints as the +# number it equals, so read through one this array would hold a 1 rather +# than a true. +--disable_view_protocol +SELECT JSON_ARRAY(1, 'two', NULL, TRUE, 3.5) AS scalars; +--enable_view_protocol +SELECT JSON_OBJECT('a', 1, 'b', 'two', 'c', NULL) AS pairs; +SELECT JSON_ARRAY(JSON_ARRAY(1, 2), JSON_OBJECT('a', 1)) AS nested; +SELECT JSON_OBJECT('k', JSON_ARRAY(1, 2)) AS nested_in_object; +SELECT JSON_ARRAY(JSON_SET('{"a": 1}', '$.b', 2)) AS edited_then_built; +SELECT JSON_ARRAY(JSON_EXTRACT('{"a": [1, 2]}', '$.a')) AS extracted_then_built; + +--echo # +--echo # 2. Building a document out of values that cannot be +--echo # +--echo # A JSON column holds whatever was in it when the check was off, +--echo # and nothing between the column and here reads it. The value +--echo # goes in as it stands, as it always has, and the note is what +--echo # says so. +--echo # +CREATE TABLE tj (id INT, j JSON); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tj VALUES (1, '{"a": 1}'), (2, '{"a":1,'), (3, ''), (4, '{"a":1}'); +SET SESSION check_constraint_checks = ON; + +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 1)) AS from_good; +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 2)) AS from_broken; +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 3)) AS from_empty; +SELECT JSON_OBJECT('k', (SELECT j FROM tj WHERE id = 2)) AS from_broken_object; + +--echo # +--echo # The compact one reads perfectly well and is simply not +--echo # written the way this function writes. +--echo # +SELECT JSON_ARRAY((SELECT j FROM tj WHERE id = 4)) AS from_compact; + +--echo # +--echo # 3. Editing a document +--echo # +SELECT JSON_SET('{"a": 1}', '$.b', 2) AS set_done, + JSON_INSERT('{"a": 1}', '$.b', 2) AS insert_done, + JSON_REPLACE('{"a": 1}', '$.a', 2) AS replace_done; +SELECT JSON_REMOVE('{"a": 1, "b": 2}', '$.a') AS removed, + JSON_ARRAY_APPEND('[1]', '$', 2) AS appended, + JSON_ARRAY_INSERT('[1, 3]', '$[1]', 2) AS inserted; +SELECT JSON_MERGE('[1]', '[2]') AS merged, + JSON_MERGE_PATCH('{"a": 1}', '{"b": 2}') AS patched; +SELECT JSON_SET((SELECT j FROM tj WHERE id = 2), '$.b', 2) AS set_over_broken; +SELECT JSON_SET('{"a": 1}', '$.b', (SELECT j FROM tj WHERE id = 2)) + AS set_with_broken; + +--echo # +--echo # 4. Reading a document out of another one +--echo # +SELECT JSON_EXTRACT('{"a": [1, 2], "b": 3}', '$.a') AS extracted, + JSON_EXTRACT('{"a": 1, "b": 2}', '$.a', '$.b') AS extracted_two; +SELECT JSON_QUERY('{"a": [1,2]}', '$.a') AS queried; +SELECT JSON_KEYS('{"a": 1, "b": 2}') AS keys_of; +SELECT JSON_SEARCH('{"a": "x", "b": "x"}', 'all', 'x') AS searched_all, + JSON_SEARCH('{"a": "x"}', 'one', 'x') AS searched_one; +SELECT JSON_NORMALIZE('{"b": 1, "a": 2}') AS normalized; +SELECT JSON_COMPACT('{"a": [1, 2]}') AS compacted, + JSON_LOOSE('{"a":[1,2]}') AS loosened; +SELECT JSON_DETAILED('{"a": 1}') AS detailed; + +--echo # +--echo # JSON_VALID reads its argument through val_json(), where the +--echo # format functions pass the value straight through without touching +--echo # it, so nothing here is written out afresh. +--echo # +SELECT JSON_VALID(JSON_LOOSE('{"a":1}')) AS loose_forwarded, + JSON_VALID(JSON_COMPACT('{"a": 1}')) AS compact_forwarded, + JSON_VALID(JSON_LOOSE((SELECT j FROM tj WHERE id = 2))) + AS broken_forwarded; + +--echo # +--echo # 5. Building a document a row at a time +--echo # +CREATE TABLE tg (g INT, v INT, s VARCHAR(16)); +INSERT INTO tg VALUES (1, 1, 'a'), (1, 2, 'b'), (2, 3, 'c'), (2, 4, 'd'); +SELECT g, JSON_ARRAYAGG(v) AS arr, JSON_OBJECTAGG(s, v) AS obj + FROM tg GROUP BY g ORDER BY g; +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS agg_then_built FROM tg; +SELECT g, JSON_ARRAYAGG(v) AS arr FROM tg GROUP BY g WITH ROLLUP; + +--echo # +--echo # A group cut to fit group_concat_max_len is cut back to the +--echo # last whole element, so every element in it is one a row of +--echo # the group held and the row it stopped at is reported. What +--echo # comes out is a document, but not the value the group has. +--echo # A group built out of a broken column is complete too, and is +--echo # not a document at all. +--echo # +SET @save_len = @@group_concat_max_len; +SET SESSION group_concat_max_len = 10; +SELECT JSON_ARRAYAGG(s) AS cut FROM tg; +SET SESSION group_concat_max_len = @save_len; +SELECT JSON_ARRAYAGG(j) AS agg_broken FROM tj; +SELECT JSON_OBJECTAGG(id, j) AS objagg_broken FROM tj; + +--echo # +--echo # 6. Character sets +--echo # +--echo # In swe7 the bytes that encode brackets and braces elsewhere +--echo # encode national letters, so a document cannot be written in it +--echo # at all. The bytes returned are the same ones as always. +--echo # +SELECT HEX(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7, + JSON_VALID(JSON_ARRAY(_swe7 0x61)) AS array_in_swe7_valid; +SELECT HEX(JSON_ARRAYAGG(CONVERT(s USING swe7))) AS agg_in_swe7 FROM tg; +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) AS array_in_ucs2, + JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING ucs2))) + AS array_in_ucs2_valid; +SELECT JSON_VALID(JSON_OBJECT('k', CONVERT(_utf8mb4 0xC3A8 USING utf16))) + AS object_in_utf16_valid; +SELECT HEX(JSON_ARRAY(_binary 0x61)) AS array_in_binary; + +--echo # +--echo # A character set whose characters are never a single byte +--echo # encodes the punctuation properly, so a document can be written +--echo # in it and these results really are documents. sjis is not +--echo # such a set: it stores ASCII a byte at a time and puts a yen +--echo # sign where the backslash belongs. +--echo # +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32)) ) AS array_in_utf32, + JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0xC3A9 USING utf32))) + AS array_in_utf32_valid; +SELECT JSON_VALID(JSON_ARRAY(CONVERT(s USING ucs2))) AS array_of_col_ucs2 + FROM tg ORDER BY v; +SELECT JSON_VALID(JSON_OBJECT(CONVERT(s USING utf16), v)) AS object_in_utf16 + FROM tg ORDER BY v; +SELECT JSON_VALID(JSON_ARRAYAGG(CONVERT(s USING ucs2))) AS agg_in_ucs2 + FROM tg GROUP BY g ORDER BY g; +SELECT JSON_VALID(JSON_OBJECTAGG(CONVERT(s USING utf16), v)) AS objectagg_utf16 + FROM tg GROUP BY g ORDER BY g; +SELECT HEX(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) AS backslash_in_sjis, + JSON_VALID(JSON_ARRAY(CONVERT(_utf8mb4 0x5C USING sjis))) + AS backslash_in_sjis_valid; + +--echo # +--echo # 7. Values that arrive through a conversion +--echo # +--echo # A conversion that lost nothing kept the characters it was +--echo # given; one that had to put a question mark in its place did +--echo # not, and what it produced is not the value that went in. +--echo # +SELECT JSON_VALID(CONVERT(JSON_OBJECT('a', 1) USING ucs2)) AS converted_valid; +SELECT JSON_ARRAY(CONVERT(JSON_OBJECT('a', 1) USING utf8mb4)) AS converted_built; +SELECT HEX(CONVERT(JSON_ARRAY(_utf8mb4 0xC3A9) USING swe7)) AS converted_lossy; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), '$.b', + _utf8mb4 0xC3A9)) AS mixed_charsets, + JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B2261223A317D USING latin1), + '$.b', _utf8mb4 0xC3A9)) AS mixed_charsets_valid; + +--echo # +--echo # Conversion to binary keeps the bytes and relabels them, so a +--echo # document written two bytes to the character comes out as twice +--echo # as many characters, and is no longer one. Nothing is lost, so +--echo # the copier has nothing to report about it. +--echo # +SELECT HEX(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) + AS relabelled_bytes, + JSON_VALID(CONVERT(CONVERT(JSON_ARRAY(1) USING ucs2) USING binary)) + AS relabelled_valid; +SELECT JSON_ARRAY(CONVERT(CONVERT(JSON_OBJECT('a', 1) USING ucs2) + USING binary)) AS built_from_relabelled; + +--echo # +--echo # 8. Values that arrive through a reference +--echo # +SELECT JSON_ARRAY(x) AS through_derived + FROM (SELECT JSON_OBJECT('a', 1) AS x) d; +SELECT JSON_ARRAY(x) AS through_broken_derived + FROM (SELECT j AS x FROM tj WHERE id = 2) d; +CREATE VIEW v1 AS SELECT JSON_OBJECT('a', 1) AS x; +SELECT JSON_ARRAY(x) AS through_view FROM v1; +DROP VIEW v1; +SELECT JSON_ARRAY(JSON_ARRAYAGG(v)) AS through_aggregate_ref + FROM tg GROUP BY g ORDER BY g; + +--echo # +--echo # A reference into a merged view on the inner side of an outer +--echo # join returns NULL for a row the join filled in, without asking +--echo # the item it refers to. The second row below is such a row and +--echo # follows one that matched, so anything remembered from the +--echo # first would still be there. +--echo # +CREATE TABLE touter (id INT); +CREATE TABLE tinner (id INT); +INSERT INTO touter VALUES (1),(2); +INSERT INTO tinner VALUES (1); +CREATE VIEW vinner AS SELECT id, JSON_OBJECT('a', id) AS x FROM tinner; +SELECT touter.id, JSON_ARRAY(vinner.x) AS through_outer_join_view + FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +SELECT touter.id, JSON_VALID(vinner.x) AS through_outer_join_view_valid + FROM touter LEFT JOIN vinner ON touter.id = vinner.id ORDER BY touter.id; +DROP VIEW vinner; +DROP TABLE tinner; +DROP TABLE touter; + +DROP TABLE tg; +DROP TABLE tj; + +--echo # +--echo # 9. Items that are copied +--echo # +--echo # Pushing a condition into a derived table copies the expression +--echo # that produces the column, leaving two items in two places of +--echo # the plan. A copy has produced no value of its own yet. +--echo # +CREATE TABLE tc (a INT, b VARCHAR(32)); +INSERT INTO tc VALUES (1, 'x'), (2, 'y'); +SELECT * FROM (SELECT a, JSON_ARRAY(1, 2) AS j FROM tc) d WHERE d.j LIKE '[%'; +SELECT * FROM (SELECT a, JSON_ARRAY(a, b) AS j FROM tc) d WHERE d.j LIKE '[%'; +SELECT * FROM (SELECT a, JSON_OBJECT('k', a) AS j FROM tc) d + WHERE d.j = '{"k": 1}'; + +--echo # +--echo # A table definition holding a JSON expression is copied whole +--echo # when the table is altered. +--echo # +CREATE TABLE tv (a INT, j VARCHAR(64) AS (JSON_ARRAY(a)) VIRTUAL, + CHECK (JSON_VALID(JSON_OBJECT('a', a)))); +INSERT INTO tv (a) VALUES (1), (2); +ALTER TABLE tv ADD COLUMN c INT; +SELECT * FROM tv ORDER BY a; + +DROP TABLE tv; +DROP TABLE tc; diff --git a/mysql-test/main/func_json_merge_broken.result b/mysql-test/main/func_json_merge_broken.result new file mode 100644 index 0000000000000..d33dca675bba2 --- /dev/null +++ b/mysql-test/main/func_json_merge_broken.result @@ -0,0 +1,286 @@ +# --------------------------------------------------------------- +# what every other JSON function says about these documents +# --------------------------------------------------------------- +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS no_comma; +no_comma +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +SELECT JSON_EXTRACT('{"a":1,', '$') AS stops_after_comma; +stops_after_comma +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{', '$') AS stops_at_once; +stops_at_once +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS no_comma_nested; +no_comma_nested +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +# --------------------------------------------------------------- +# JSON_MERGE - an argument that never completes a value +# --------------------------------------------------------------- +# argument 2 breaks, argument 1 being an empty object. Its keys +# are what is walked first, and an empty object has none, so the +# walk that meets the breakage is the one that was reached second +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS empty_lhs; +empty_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{}', '{"a":1,') AS empty_lhs_stops; +empty_lhs_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('{}', '{') AS empty_lhs_at_once; +empty_lhs_at_once +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS empty_lhs_nested; +empty_lhs_nested +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 13 +# argument 1 breaks, argument 2 being an empty object +SELECT JSON_MERGE('{"a":1 "b":2}', '{}') AS empty_rhs; +empty_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{"a":1,', '{}') AS empty_rhs_stops; +empty_rhs_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +# the same breakage where the other argument has keys of its own, +# which is the case that has always been refused +SELECT JSON_MERGE('{"z":0}', '{"a":1 "b":2}') AS keyed_lhs; +keyed_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{"a":1 "b":2}', '{"z":0}') AS keyed_rhs; +keyed_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 8 +# three arguments, so a composed document is on the left by the +# time the broken one is reached +SELECT JSON_MERGE('{}', '{"a":1 "b":2}', '{"c":3}') AS three_args; +three_args +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE('{}', '{"c":3}', '{"a":1 "b":2}') AS three_args_last; +three_args_last +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 3 to function 'json_merge_preserve' at position 8 +# --------------------------------------------------------------- +# JSON_MERGE_PATCH, which walks the same two loops +# --------------------------------------------------------------- +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_empty_lhs; +patch_empty_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('{}', '{"a":1,') AS patch_empty_lhs_stops; +patch_empty_lhs_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +SELECT JSON_MERGE_PATCH('{"a":1 "b":2}', '{}') AS patch_empty_rhs; +patch_empty_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('{"z":0}', '{"a":1 "b":2}') AS patch_keyed_lhs; +patch_keyed_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}', '{"c":3}') AS patch_three; +patch_three +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +# merging onto SQL NULL, where the argument is taken over whole +SELECT JSON_MERGE_PATCH(NULL, '{"a":1 "b":2}', '[3,4]') AS patch_null_obj; +patch_null_obj +[3, 4] +SELECT JSON_MERGE_PATCH(NULL, '[1,2 3]', '[3,4]') AS patch_null_arr; +patch_null_arr +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 6 +# --------------------------------------------------------------- +# arrays and scalars, which have always refused this +# --------------------------------------------------------------- +SELECT JSON_MERGE('[]', '[1,2') AS arr_empty_lhs; +arr_empty_lhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('[1]', '[2,3') AS arr_keyed_lhs; +arr_keyed_lhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +SELECT JSON_MERGE('[]', '[1,2 3]') AS arr_no_comma; +arr_no_comma +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 6 +SELECT JSON_MERGE_PATCH('[]', '[1,2') AS patch_arr; +patch_arr +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +# --------------------------------------------------------------- +# controls - a complete value with text after it is not this, and +# must keep the answer it has always had +# --------------------------------------------------------------- +SELECT JSON_MERGE('{}', '{"a":1} rubbish') AS trailing_rhs; +trailing_rhs +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2}') AS trailing_lhs; +trailing_lhs +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE_PATCH('{}', '{"a":1} rubbish') AS patch_trailing; +patch_trailing +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE('{}', '{"a":1} {"zzz":9}') AS trailing_whole_doc; +trailing_whole_doc +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +# and documents that are simply documents +SELECT JSON_MERGE('{}', '{"a":1, "b":2}') AS whole; +whole +{"a": 1, "b": 2} +SELECT JSON_MERGE('{"z":0}', '{"a":1, "b":2}') AS whole_keyed; +whole_keyed +{"z": 0, "a": 1, "b": 2} +SELECT JSON_MERGE_PATCH('{}', '{"a":1, "b":2}') AS patch_whole; +patch_whole +{"a": 1, "b": 2} +# --------------------------------------------------------------- +# out of a table, so one row does not settle what another says +# --------------------------------------------------------------- +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES +(1, '{}', '{"a":1 "b":2}'), +(2, '{"z":0}', '{"a":1 "b":2}'), +(3, '{}', '{"a":1, "b":2}'), +(4, '{}', '{"a":1} rubbish'); +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +id v +1 NULL +2 NULL +3 {"a": 1, "b": 2} +4 {"a": 1} +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +id v +1 NULL +2 NULL +3 {"a": 1, "b": 2} +4 {"a": 1} +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +DROP TABLE t; +# --------------------------------------------------------------- +# nothing downstream is handed a value the merging could not read +# --------------------------------------------------------------- +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1 "b":2}')) AS valid_broken; +valid_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_TYPE(JSON_MERGE('{}', '{"a":1 "b":2}')) AS type_broken; +type_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1 "b":2}')) AS length_broken; +length_broken +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1, "b":2}')) AS valid_whole; +valid_whole +1 +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1, "b":2}')) AS length_whole; +length_whole +2 +# --------------------------------------------------------------- +# JSON_MERGE_PATCH where argument 1 is not an object +# --------------------------------------------------------------- +# Merging onto something that is not an object does not walk the +# two objects together; it takes argument 2 whole. That is a +# different walk over the same shape, and the argument has to be +# a document there too. +SELECT JSON_MERGE_PATCH('[]', '{"a":1 "b":2}') AS array_lhs; +array_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('1', '{"a":1 "b":2}') AS scalar_lhs; +scalar_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +SELECT JSON_MERGE_PATCH('"s"', '{"a":1,') AS string_lhs; +string_lhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +SELECT JSON_MERGE_PATCH('true', '{') AS opening_brace_only; +opening_brace_only +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_patch' +# the break is inside a member value rather than at the top +SELECT JSON_MERGE_PATCH('[]', '{"a":{"p":1 "q":2}}') AS nested_break; +nested_break +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 13 +# and out of a table, so one row does not settle another +CREATE TABLE tm (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO tm VALUES (1, '[]', '{"a":1 "b":2}'), +(2, '[]', '{"a":1, "b":2}'), +(3, '{}', '{"a":1 "b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM tm ORDER BY id; +id v +1 NULL +2 {"a": 1, "b": 2} +3 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +DROP TABLE tm; +# a whole document down that same walk keeps everything it had +SELECT JSON_MERGE_PATCH('[]', '{"a":1, "b":2}') AS array_lhs_whole; +array_lhs_whole +{"a": 1, "b": 2} +SELECT JSON_MERGE_PATCH('1', '{"a":{"p":1, "q":2}}') AS scalar_lhs_whole; +scalar_lhs_whole +{"a": {"p": 1, "q": 2}} diff --git a/mysql-test/main/func_json_merge_broken.test b/mysql-test/main/func_json_merge_broken.test new file mode 100644 index 0000000000000..e15d571c8bff5 --- /dev/null +++ b/mysql-test/main/func_json_merge_broken.test @@ -0,0 +1,133 @@ +# +# JSON_MERGE and JSON_MERGE_PATCH are documented to merge JSON documents, +# and to raise an error when an argument is not one. An argument that +# breaks before its first value is complete is not one. +# +# That is a different thing from text standing after a value that IS +# complete, which func_json_merge_trailing covers: there the value the +# merging works from is whole, and what stands after it has never reached +# the answer. Here there is no whole value to work from, and what the +# merging composes is part of an object nobody wrote. +# +# The refusal is not new either. It is what these functions already do +# for the same breakage met one loop further in, and what every other +# JSON function does for the same characters. What was escaping it was +# every loop that walks an object's keys and asks only whether the +# scanner moved - one per argument in each function, and one more where +# an argument is copied whole instead of merged with anything. +# + +--echo # --------------------------------------------------------------- +--echo # what every other JSON function says about these documents +--echo # --------------------------------------------------------------- +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS no_comma; +SELECT JSON_EXTRACT('{"a":1,', '$') AS stops_after_comma; +SELECT JSON_EXTRACT('{', '$') AS stops_at_once; +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS no_comma_nested; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE - an argument that never completes a value +--echo # --------------------------------------------------------------- + +--echo # argument 2 breaks, argument 1 being an empty object. Its keys +--echo # are what is walked first, and an empty object has none, so the +--echo # walk that meets the breakage is the one that was reached second +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS empty_lhs; +SELECT JSON_MERGE('{}', '{"a":1,') AS empty_lhs_stops; +SELECT JSON_MERGE('{}', '{') AS empty_lhs_at_once; +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS empty_lhs_nested; + +--echo # argument 1 breaks, argument 2 being an empty object +SELECT JSON_MERGE('{"a":1 "b":2}', '{}') AS empty_rhs; +SELECT JSON_MERGE('{"a":1,', '{}') AS empty_rhs_stops; + +--echo # the same breakage where the other argument has keys of its own, +--echo # which is the case that has always been refused +SELECT JSON_MERGE('{"z":0}', '{"a":1 "b":2}') AS keyed_lhs; +SELECT JSON_MERGE('{"a":1 "b":2}', '{"z":0}') AS keyed_rhs; + +--echo # three arguments, so a composed document is on the left by the +--echo # time the broken one is reached +SELECT JSON_MERGE('{}', '{"a":1 "b":2}', '{"c":3}') AS three_args; +SELECT JSON_MERGE('{}', '{"c":3}', '{"a":1 "b":2}') AS three_args_last; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH, which walks the same two loops +--echo # --------------------------------------------------------------- +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_empty_lhs; +SELECT JSON_MERGE_PATCH('{}', '{"a":1,') AS patch_empty_lhs_stops; +SELECT JSON_MERGE_PATCH('{"a":1 "b":2}', '{}') AS patch_empty_rhs; +SELECT JSON_MERGE_PATCH('{"z":0}', '{"a":1 "b":2}') AS patch_keyed_lhs; +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}', '{"c":3}') AS patch_three; + +--echo # merging onto SQL NULL, where the argument is taken over whole +SELECT JSON_MERGE_PATCH(NULL, '{"a":1 "b":2}', '[3,4]') AS patch_null_obj; +SELECT JSON_MERGE_PATCH(NULL, '[1,2 3]', '[3,4]') AS patch_null_arr; + +--echo # --------------------------------------------------------------- +--echo # arrays and scalars, which have always refused this +--echo # --------------------------------------------------------------- +SELECT JSON_MERGE('[]', '[1,2') AS arr_empty_lhs; +SELECT JSON_MERGE('[1]', '[2,3') AS arr_keyed_lhs; +SELECT JSON_MERGE('[]', '[1,2 3]') AS arr_no_comma; +SELECT JSON_MERGE_PATCH('[]', '[1,2') AS patch_arr; + +--echo # --------------------------------------------------------------- +--echo # controls - a complete value with text after it is not this, and +--echo # must keep the answer it has always had +--echo # --------------------------------------------------------------- +SELECT JSON_MERGE('{}', '{"a":1} rubbish') AS trailing_rhs; +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2}') AS trailing_lhs; +SELECT JSON_MERGE_PATCH('{}', '{"a":1} rubbish') AS patch_trailing; +SELECT JSON_MERGE('{}', '{"a":1} {"zzz":9}') AS trailing_whole_doc; + +--echo # and documents that are simply documents +SELECT JSON_MERGE('{}', '{"a":1, "b":2}') AS whole; +SELECT JSON_MERGE('{"z":0}', '{"a":1, "b":2}') AS whole_keyed; +SELECT JSON_MERGE_PATCH('{}', '{"a":1, "b":2}') AS patch_whole; + +--echo # --------------------------------------------------------------- +--echo # out of a table, so one row does not settle what another says +--echo # --------------------------------------------------------------- +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES + (1, '{}', '{"a":1 "b":2}'), + (2, '{"z":0}', '{"a":1 "b":2}'), + (3, '{}', '{"a":1, "b":2}'), + (4, '{}', '{"a":1} rubbish'); +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +DROP TABLE t; + +--echo # --------------------------------------------------------------- +--echo # nothing downstream is handed a value the merging could not read +--echo # --------------------------------------------------------------- +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1 "b":2}')) AS valid_broken; +SELECT JSON_TYPE(JSON_MERGE('{}', '{"a":1 "b":2}')) AS type_broken; +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1 "b":2}')) AS length_broken; +SELECT JSON_VALID(JSON_MERGE('{}', '{"a":1, "b":2}')) AS valid_whole; +SELECT JSON_LENGTH(JSON_MERGE('{}', '{"a":1, "b":2}')) AS length_whole; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH where argument 1 is not an object +--echo # --------------------------------------------------------------- +--echo # Merging onto something that is not an object does not walk the +--echo # two objects together; it takes argument 2 whole. That is a +--echo # different walk over the same shape, and the argument has to be +--echo # a document there too. +SELECT JSON_MERGE_PATCH('[]', '{"a":1 "b":2}') AS array_lhs; +SELECT JSON_MERGE_PATCH('1', '{"a":1 "b":2}') AS scalar_lhs; +SELECT JSON_MERGE_PATCH('"s"', '{"a":1,') AS string_lhs; +SELECT JSON_MERGE_PATCH('true', '{') AS opening_brace_only; +--echo # the break is inside a member value rather than at the top +SELECT JSON_MERGE_PATCH('[]', '{"a":{"p":1 "q":2}}') AS nested_break; +--echo # and out of a table, so one row does not settle another +CREATE TABLE tm (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO tm VALUES (1, '[]', '{"a":1 "b":2}'), + (2, '[]', '{"a":1, "b":2}'), + (3, '{}', '{"a":1 "b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM tm ORDER BY id; +DROP TABLE tm; +--echo # a whole document down that same walk keeps everything it had +SELECT JSON_MERGE_PATCH('[]', '{"a":1, "b":2}') AS array_lhs_whole; +SELECT JSON_MERGE_PATCH('1', '{"a":{"p":1, "q":2}}') AS scalar_lhs_whole; diff --git a/mysql-test/main/func_json_merge_patch_adopt.result b/mysql-test/main/func_json_merge_patch_adopt.result new file mode 100644 index 0000000000000..334a1e5e5a77d --- /dev/null +++ b/mysql-test/main/func_json_merge_patch_adopt.result @@ -0,0 +1,77 @@ +# the answer must not depend on how the arguments are produced +SELECT JSON_MERGE_PATCH(NULL, '[1]', '{"aaaaaaaaaaaaaaaa":1}') AS literal; +literal +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS built; +built +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, CONCAT('[', '1]'), +CONCAT('{"aaaaaaaaaaaaaaaa"', ':1}')) AS concatenated; +concatenated +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, REVERSE(']1['), +REVERSE('}1:"aaaaaaaaaaaaaaaa"{')) AS reversed; +reversed +{"aaaaaaaaaaaaaaaa": 1} +# a taken-over document of every kind that is not an object, since +# an object is merged rather than taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_number; +was_number +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('"S"'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_string; +was_string +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('TRUE'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_true; +was_true +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2]'), +LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_array; +was_array +{"aaaaaaaaaaaaaaaa": 1} +# room that has to be found rather than grown into +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), +LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}'))) +IS NULL AS big_jump_null; +big_jump_null +0 +SELECT JSON_LENGTH(JSON_MERGE_PATCH(NULL, LOWER('1'), +LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}')))) AS big_len; +big_len +1 +# more arguments after the one that was taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), +LOWER('{"BBBBBBBBBBBBBBBB":2}')) AS three; +three +{"aaaaaaaaaaaaaaaa": 1, "bbbbbbbbbbbbbbbb": 2} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), +LOWER('{"AAAAAAAAAAAAAAAA":9}')) AS three_same_key; +three_same_key +{"aaaaaaaaaaaaaaaa": 9} +# out of a table, so the arguments change from row to row +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES +(1, '[1]', '{"aaaaaaaaaaaaaaaa":1}'), +(2, '[2]', '{"bbbbbbbbbbbbbbbb":2}'); +SELECT id, JSON_MERGE_PATCH(NULL, a, b) AS plain FROM t ORDER BY id; +id plain +1 {"aaaaaaaaaaaaaaaa": 1} +2 {"bbbbbbbbbbbbbbbb": 2} +SELECT id, JSON_MERGE_PATCH(NULL, LOWER(a), LOWER(b)) AS built FROM t +ORDER BY id; +id built +1 {"aaaaaaaaaaaaaaaa": 1} +2 {"bbbbbbbbbbbbbbbb": 2} +DROP TABLE t; +# controls: the same second argument reached without a taken-over +# first one, and a second argument no longer than the first +SELECT JSON_MERGE_PATCH('{}', LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS no_adoption; +no_adoption +{"aaaaaaaaaaaaaaaa": 1} +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2,3,4,5,6,7,8,9]'), +LOWER('{"A":1}')) AS shorter_after; +shorter_after +{"a": 1} diff --git a/mysql-test/main/func_json_merge_patch_adopt.test b/mysql-test/main/func_json_merge_patch_adopt.test new file mode 100644 index 0000000000000..0dde50f0136d7 --- /dev/null +++ b/mysql-test/main/func_json_merge_patch_adopt.test @@ -0,0 +1,65 @@ +# +# JSON_MERGE_PATCH taking a whole document over when everything before it +# was NULL. +# +# The document is taken over from the buffer the argument was read into, +# and the argument after it is read into that same buffer. An argument +# that needs more room than the one before leaves that buffer somewhere +# else, so a document taken over by being pointed at is gone by the time +# it comes to be read. +# +# Whether an argument is read into the caller's buffer at all depends on +# the function that produced it - one that returns a piece of its own +# argument does not, one that builds its answer does. So the cases below +# come in pairs which differ only in how the same value is produced, and +# both have to give the same answer. +# + +--echo # the answer must not depend on how the arguments are produced +SELECT JSON_MERGE_PATCH(NULL, '[1]', '{"aaaaaaaaaaaaaaaa":1}') AS literal; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS built; +SELECT JSON_MERGE_PATCH(NULL, CONCAT('[', '1]'), + CONCAT('{"aaaaaaaaaaaaaaaa"', ':1}')) AS concatenated; +SELECT JSON_MERGE_PATCH(NULL, REVERSE(']1['), + REVERSE('}1:"aaaaaaaaaaaaaaaa"{')) AS reversed; + +--echo # a taken-over document of every kind that is not an object, since +--echo # an object is merged rather than taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_number; +SELECT JSON_MERGE_PATCH(NULL, LOWER('"S"'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_string; +SELECT JSON_MERGE_PATCH(NULL, LOWER('TRUE'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_true; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2]'), + LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS was_array; + +--echo # room that has to be found rather than grown into +SELECT JSON_MERGE_PATCH(NULL, LOWER('1'), + LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}'))) + IS NULL AS big_jump_null; +SELECT JSON_LENGTH(JSON_MERGE_PATCH(NULL, LOWER('1'), + LOWER(CONCAT('{"k":"', REPEAT('x', 500), '"}')))) AS big_len; + +--echo # more arguments after the one that was taken over +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), + LOWER('{"BBBBBBBBBBBBBBBB":2}')) AS three; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1]'), LOWER('{"AAAAAAAAAAAAAAAA":1}'), + LOWER('{"AAAAAAAAAAAAAAAA":9}')) AS three_same_key; + +--echo # out of a table, so the arguments change from row to row +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES + (1, '[1]', '{"aaaaaaaaaaaaaaaa":1}'), + (2, '[2]', '{"bbbbbbbbbbbbbbbb":2}'); +SELECT id, JSON_MERGE_PATCH(NULL, a, b) AS plain FROM t ORDER BY id; +SELECT id, JSON_MERGE_PATCH(NULL, LOWER(a), LOWER(b)) AS built FROM t + ORDER BY id; +DROP TABLE t; + +--echo # controls: the same second argument reached without a taken-over +--echo # first one, and a second argument no longer than the first +SELECT JSON_MERGE_PATCH('{}', LOWER('{"AAAAAAAAAAAAAAAA":1}')) AS no_adoption; +SELECT JSON_MERGE_PATCH(NULL, LOWER('[1,2,3,4,5,6,7,8,9]'), + LOWER('{"A":1}')) AS shorter_after; diff --git a/mysql-test/main/func_json_merge_trailing.result b/mysql-test/main/func_json_merge_trailing.result new file mode 100644 index 0000000000000..841d75713b2ad --- /dev/null +++ b/mysql-test/main/func_json_merge_trailing.result @@ -0,0 +1,277 @@ +# --------------------------------------------------------------- +# JSON_MERGE_PATCH, ordinary merge - nothing is taken over here +# --------------------------------------------------------------- +# text after the value of argument 1 +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}') AS obj_lhs; +obj_lhs +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE_PATCH('[1,2] rubbish', '{"a":1}') AS arr_lhs; +arr_lhs +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 7 +# text after the value of argument 2 +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish') AS obj_rhs; +obj_rhs +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE_PATCH('{"a":1}', '[1,2] rubbish') AS arr_rhs; +arr_rhs +[1, 2] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +# both arguments, so both are named +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2} nonsense') AS both_args; +both_args +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# three arguments, the middle one carrying it +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish', '{"c":3}') AS middle; +middle +{"a": 1, "b": 2, "c": 3} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# what stands after the value is a whole document of its own, which +# goes nowhere +SELECT JSON_MERGE_PATCH('{"a":1} {"zzz":9}', '{"b":2}') AS second_doc; +second_doc +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +# --------------------------------------------------------------- +# JSON_MERGE_PATCH taking a document over, everything before NULL +# --------------------------------------------------------------- +# taken over and then merged over, so the answer never holds it +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish', '{"a":1}') AS adopt_merged; +adopt_merged +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +SELECT JSON_MERGE_PATCH(NULL, '1 rubbish', '{"a":1}') AS adopt_scalar; +adopt_scalar +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 3 +SELECT JSON_MERGE_PATCH(NULL, '"s" rubbish', '{"a":1}') AS adopt_string; +adopt_string +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 5 +SELECT JSON_MERGE_PATCH(NULL, '[1,2] [9]', '{"a":1}') AS adopt_second_doc; +adopt_second_doc +{"a": 1} +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +# taken over and left as the answer, which is refused as it always +# was - the warning that refuses it is not replaced by a note +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish') AS adopt_final; +adopt_final +NULL +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 7 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 7 +# an object is the one shape that is not taken over: merged onto +# SQL NULL it contributes nothing and is dropped. Being dropped is +# no reason to say less about it than about the shapes that are +# kept, the characters standing after it being the same characters +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '[3,4]') AS drop_obj; +drop_obj +[3, 4] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT JSON_MERGE_PATCH(NULL, '{} rubbish', '[3,4]') AS drop_empty_obj; +drop_empty_obj +[3, 4] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 4 +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} {"zzz":9}', '[3,4]') AS drop_obj_doc; +drop_obj_doc +[3, 4] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# two of them, so both are named and one saying it does not stand +# for the other +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '{"b":2} nonsense') +AS drop_two; +drop_two +NULL +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +Note 4038 Syntax error in JSON text in argument 3 to function 'json_merge_patch' at position 9 +# and where the dropped object is the last word, so the answer is +# SQL NULL and there is no document left to refuse +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish') AS drop_obj_final; +drop_obj_final +NULL +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +# --------------------------------------------------------------- +# JSON_MERGE +# --------------------------------------------------------------- +SELECT JSON_MERGE('[1,2] rubbish', '{"a":1}') AS merge_lhs; +merge_lhs +[1, 2, {"a": 1}] +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 7 +SELECT JSON_MERGE('[1,2]', '{"a":1} rubbish') AS merge_rhs; +merge_rhs +[1, 2, {"a": 1}] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2} nonsense') AS merge_both; +merge_both +{"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +SELECT JSON_MERGE('[1]', '[2] rubbish', '[3]') AS merge_middle; +merge_middle +[1, 2, 3] +Warnings: +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 5 +# --------------------------------------------------------------- +# controls - nothing here may say anything +# --------------------------------------------------------------- +# an object dropped onto SQL NULL with nothing standing after it +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}', '[3,4]') AS drop_obj_clean; +drop_obj_clean +[3, 4] +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS drop_obj_clean_final; +drop_obj_clean_final +NULL +# space after the value is not text after the value +SELECT JSON_MERGE_PATCH('{"a":1} ', '{"b":2}') AS trailing_space; +trailing_space +{"a": 1, "b": 2} +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} + ') AS trailing_ws; +trailing_ws +{"a": 1, "b": 2} +SELECT JSON_MERGE('[1] ', '[2] ') AS merge_trailing_space; +merge_trailing_space +[1, 2] +# ordinary well formed arguments +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS clean_patch; +clean_patch +{"a": 1, "b": 2} +SELECT JSON_MERGE('[1]', '[2]') AS clean_merge; +clean_merge +[1, 2] +# an argument that is not a document from its first character has +# always been refused, and still is, with a warning and not a note +SELECT JSON_MERGE_PATCH('rubbish', '{"b":2}') AS bad_lhs; +bad_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 1 +SELECT JSON_MERGE_PATCH('{"a":1}', 'rubbish') AS bad_rhs; +bad_rhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 1 +SELECT JSON_MERGE('rubbish', '[2]') AS merge_bad_lhs; +merge_bad_lhs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 1 +# NULL arguments say nothing either +SELECT JSON_MERGE_PATCH(NULL, NULL) AS both_null; +both_null +NULL +SELECT JSON_MERGE('[1]', NULL) AS merge_null; +merge_null +NULL +# --------------------------------------------------------------- +# out of a table, so the arguments change from row to row and one +# row saying it does not settle what another row says +# --------------------------------------------------------------- +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES +(1, '{"a":1} rubbish', '{"b":2}'), +(2, '{"a":1}', '{"b":2} nonsense'), +(3, '{"a":1}', '{"b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +id v +1 {"a": 1, "b": 2} +2 {"a": 1, "b": 2} +3 {"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 9 +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +id v +1 {"a": 1, "b": 2} +2 {"a": 1, "b": 2} +3 {"a": 1, "b": 2} +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 9 +Note 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 9 +DROP TABLE t; +# --------------------------------------------------------------- +# where the trouble is said to be, which has to be where every +# other JSON function reading the same bytes says it is +# --------------------------------------------------------------- +# broken inside the value rather than after it, so there is no +# whole value to compose from and the merging refuses it - what is +# being compared here is where each of them says the trouble is +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS extract_says; +extract_says +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS merge_says; +merge_says +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 8 +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_says; +patch_says +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_patch' at position 8 +# deeper in, so the position is further along +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS extract_deep; +extract_deep +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS merge_deep; +merge_deep +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_merge_preserve' at position 13 +# a document that simply stops, which has no position to give +SELECT JSON_EXTRACT('{"a":1,', '$') AS extract_stops; +extract_stops +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_MERGE('{"a":1,', '{}') AS merge_stops_lhs; +merge_stops_lhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_merge_preserve' +SELECT JSON_MERGE('{}', '{"a":1,') AS merge_stops_rhs; +merge_stops_rhs +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_merge_preserve' +# --------------------------------------------------------------- +# a note does not become an error where a warning would +# --------------------------------------------------------------- +CREATE TABLE t (v JSON); +SET @sm= @@sql_mode; +SET sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t VALUES (JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}')); +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_merge_patch' at position 9 +SET sql_mode=@sm; +SELECT * FROM t; +v +{"a": 1, "b": 2} +DROP TABLE t; diff --git a/mysql-test/main/func_json_merge_trailing.test b/mysql-test/main/func_json_merge_trailing.test new file mode 100644 index 0000000000000..0d9d5de5d4225 --- /dev/null +++ b/mysql-test/main/func_json_merge_trailing.test @@ -0,0 +1,157 @@ +# +# JSON_MERGE and JSON_MERGE_PATCH read a document argument only as far as +# its first value. What they compose comes out of that value and nothing +# else, so text standing after it never reaches the answer and never +# stopped one being given. +# +# It is still not a document, and every other JSON function says so about +# the very same text. These two say so as well, with a note. A note and +# not a warning, because a warning becomes an error under a strict mode +# and would take away an answer that has always been given. +# +# So a case whose value the merging read to its end has to keep the value +# it has always had, and the note is the only thing that is new about it. +# +# A document that breaks before its first value is complete is a +# different thing, having no whole value for any of the above to be true +# of, and it is refused rather than noted. func_json_merge_broken +# carries those; the section below reaches far enough into them to show +# that both say where the trouble is in the same place. +# + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH, ordinary merge - nothing is taken over here +--echo # --------------------------------------------------------------- + +--echo # text after the value of argument 1 +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}') AS obj_lhs; +SELECT JSON_MERGE_PATCH('[1,2] rubbish', '{"a":1}') AS arr_lhs; + +--echo # text after the value of argument 2 +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish') AS obj_rhs; +SELECT JSON_MERGE_PATCH('{"a":1}', '[1,2] rubbish') AS arr_rhs; + +--echo # both arguments, so both are named +SELECT JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2} nonsense') AS both_args; + +--echo # three arguments, the middle one carrying it +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} rubbish', '{"c":3}') AS middle; + +--echo # what stands after the value is a whole document of its own, which +--echo # goes nowhere +SELECT JSON_MERGE_PATCH('{"a":1} {"zzz":9}', '{"b":2}') AS second_doc; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE_PATCH taking a document over, everything before NULL +--echo # --------------------------------------------------------------- + +--echo # taken over and then merged over, so the answer never holds it +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish', '{"a":1}') AS adopt_merged; +SELECT JSON_MERGE_PATCH(NULL, '1 rubbish', '{"a":1}') AS adopt_scalar; +SELECT JSON_MERGE_PATCH(NULL, '"s" rubbish', '{"a":1}') AS adopt_string; +SELECT JSON_MERGE_PATCH(NULL, '[1,2] [9]', '{"a":1}') AS adopt_second_doc; + +--echo # taken over and left as the answer, which is refused as it always +--echo # was - the warning that refuses it is not replaced by a note +SELECT JSON_MERGE_PATCH(NULL, '[1,2] rubbish') AS adopt_final; + +--echo # an object is the one shape that is not taken over: merged onto +--echo # SQL NULL it contributes nothing and is dropped. Being dropped is +--echo # no reason to say less about it than about the shapes that are +--echo # kept, the characters standing after it being the same characters +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '[3,4]') AS drop_obj; +SELECT JSON_MERGE_PATCH(NULL, '{} rubbish', '[3,4]') AS drop_empty_obj; +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} {"zzz":9}', '[3,4]') AS drop_obj_doc; + +--echo # two of them, so both are named and one saying it does not stand +--echo # for the other +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish', '{"b":2} nonsense') + AS drop_two; + +--echo # and where the dropped object is the last word, so the answer is +--echo # SQL NULL and there is no document left to refuse +SELECT JSON_MERGE_PATCH(NULL, '{"a":1} rubbish') AS drop_obj_final; + +--echo # --------------------------------------------------------------- +--echo # JSON_MERGE +--echo # --------------------------------------------------------------- + +SELECT JSON_MERGE('[1,2] rubbish', '{"a":1}') AS merge_lhs; +SELECT JSON_MERGE('[1,2]', '{"a":1} rubbish') AS merge_rhs; +SELECT JSON_MERGE('{"a":1} rubbish', '{"b":2} nonsense') AS merge_both; +SELECT JSON_MERGE('[1]', '[2] rubbish', '[3]') AS merge_middle; + +--echo # --------------------------------------------------------------- +--echo # controls - nothing here may say anything +--echo # --------------------------------------------------------------- + +--echo # an object dropped onto SQL NULL with nothing standing after it +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}', '[3,4]') AS drop_obj_clean; +SELECT JSON_MERGE_PATCH(NULL, '{"a":1}') AS drop_obj_clean_final; + +--echo # space after the value is not text after the value +SELECT JSON_MERGE_PATCH('{"a":1} ', '{"b":2}') AS trailing_space; +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2} + ') AS trailing_ws; +SELECT JSON_MERGE('[1] ', '[2] ') AS merge_trailing_space; + +--echo # ordinary well formed arguments +SELECT JSON_MERGE_PATCH('{"a":1}', '{"b":2}') AS clean_patch; +SELECT JSON_MERGE('[1]', '[2]') AS clean_merge; + +--echo # an argument that is not a document from its first character has +--echo # always been refused, and still is, with a warning and not a note +SELECT JSON_MERGE_PATCH('rubbish', '{"b":2}') AS bad_lhs; +SELECT JSON_MERGE_PATCH('{"a":1}', 'rubbish') AS bad_rhs; +SELECT JSON_MERGE('rubbish', '[2]') AS merge_bad_lhs; + +--echo # NULL arguments say nothing either +SELECT JSON_MERGE_PATCH(NULL, NULL) AS both_null; +SELECT JSON_MERGE('[1]', NULL) AS merge_null; + +--echo # --------------------------------------------------------------- +--echo # out of a table, so the arguments change from row to row and one +--echo # row saying it does not settle what another row says +--echo # --------------------------------------------------------------- + +CREATE TABLE t (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t VALUES + (1, '{"a":1} rubbish', '{"b":2}'), + (2, '{"a":1}', '{"b":2} nonsense'), + (3, '{"a":1}', '{"b":2}'); +SELECT id, JSON_MERGE_PATCH(a, b) AS v FROM t ORDER BY id; +SELECT id, JSON_MERGE(a, b) AS v FROM t ORDER BY id; +DROP TABLE t; + +--echo # --------------------------------------------------------------- +--echo # where the trouble is said to be, which has to be where every +--echo # other JSON function reading the same bytes says it is +--echo # --------------------------------------------------------------- + +--echo # broken inside the value rather than after it, so there is no +--echo # whole value to compose from and the merging refuses it - what is +--echo # being compared here is where each of them says the trouble is +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS extract_says; +SELECT JSON_MERGE('{}', '{"a":1 "b":2}') AS merge_says; +SELECT JSON_MERGE_PATCH('{}', '{"a":1 "b":2}') AS patch_says; + +--echo # deeper in, so the position is further along +SELECT JSON_EXTRACT('{"a":{"x":1 "y":2}}', '$') AS extract_deep; +SELECT JSON_MERGE('{}', '{"a":{"x":1 "y":2}}') AS merge_deep; + +--echo # a document that simply stops, which has no position to give +SELECT JSON_EXTRACT('{"a":1,', '$') AS extract_stops; +SELECT JSON_MERGE('{"a":1,', '{}') AS merge_stops_lhs; +SELECT JSON_MERGE('{}', '{"a":1,') AS merge_stops_rhs; + +--echo # --------------------------------------------------------------- +--echo # a note does not become an error where a warning would +--echo # --------------------------------------------------------------- + +CREATE TABLE t (v JSON); +SET @sm= @@sql_mode; +SET sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t VALUES (JSON_MERGE_PATCH('{"a":1} rubbish', '{"b":2}')); +SET sql_mode=@sm; +SELECT * FROM t; +DROP TABLE t; diff --git a/mysql-test/main/func_json_mislabel.result b/mysql-test/main/func_json_mislabel.result new file mode 100644 index 0000000000000..838e18e5de54e --- /dev/null +++ b/mysql-test/main/func_json_mislabel.result @@ -0,0 +1,86 @@ +# +# Behavioral baseline: the same stored bytes read under a different +# character set label. +# +# Going through a binary column and back out again relabels a column +# without touching its contents, so the identical bytes are handed to +# the JSON parser as a different character set. This is the only way +# to put a document in front of the parser that is not valid in the +# character set it is claimed to be in - converting it would repair or +# reject it first. What each label makes of the same bytes is +# recorded here. +# +SET NAMES utf8mb4; +# +# 1. A gbk character whose second byte is the one JSON uses to escape. +# Read as gbk it is one letter inside a string; read as anything +# that treats bytes singly it starts an escape sequence and runs +# off the end of the document. +# +CREATE TABLE t1 (j VARCHAR(60) CHARACTER SET gbk); +INSERT INTO t1 VALUES (X'7B2261223A22815C227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_gbk FROM t1; +stored v_gbk +7B2261223A22815C227D 1 +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs +FROM t1; +ext cs +22815C22 gbk +ALTER TABLE t1 MODIFY j VARBINARY(60); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t1; +stored v_binary +7B2261223A22815C227D 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +ALTER TABLE t1 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_latin1 FROM t1; +stored v_latin1 +7B2261223A22815C227D 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t1; +ext +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +# +# 2. Bytes that are one character in utf8mb4 and two in latin1. The +# document stays parseable either way, but the value it yields is +# not the same. +# +CREATE TABLE t2 (j VARCHAR(60) CHARACTER SET utf8mb4); +INSERT INTO t2 VALUES (X'7B2261223A22C3A9227D'); +SELECT HEX(j) AS stored, HEX(JSON_EXTRACT(j, '$.a')) AS ext_utf8mb4, +CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_utf8mb4 FROM t2; +stored ext_utf8mb4 chars_utf8mb4 +7B2261223A22C3A9227D 22C3A922 1 +ALTER TABLE t2 MODIFY j VARBINARY(60); +ALTER TABLE t2 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored_unchanged, JSON_VALID(j) AS v FROM t2; +stored_unchanged v +7B2261223A22C3A9227D 1 +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs, +CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_latin1 FROM t2; +ext cs chars_latin1 +22C3A922 latin1 2 +# +# 3. Relabelling in the other direction, to a character set that +# cannot hold the bytes. This one does not go through quietly. +# +CREATE TABLE t3 (j VARBINARY(60)); +INSERT INTO t3 VALUES (X'7B2261223A22FF227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t3; +stored v_binary +7B2261223A22FF227D 1 +ALTER TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +ERROR 22007: Incorrect string value: '\xFF"}' for column `test`.`t3`.`j` at row 1 +ALTER IGNORE TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +Warnings: +Warning 1366 Incorrect string value: '\xFF"}' for column `test`.`t3`.`j` at row 1 +SELECT HEX(j) AS repaired, JSON_VALID(j) AS v FROM t3; +repaired v +7B2261223A223F227D 1 +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t3; +ext +223F22 +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/func_json_mislabel.test b/mysql-test/main/func_json_mislabel.test new file mode 100644 index 0000000000000..3913ee2dfea98 --- /dev/null +++ b/mysql-test/main/func_json_mislabel.test @@ -0,0 +1,67 @@ +--source include/have_gbk.inc +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: the same stored bytes read under a different +--echo # character set label. +--echo # +--echo # Going through a binary column and back out again relabels a column +--echo # without touching its contents, so the identical bytes are handed to +--echo # the JSON parser as a different character set. This is the only way +--echo # to put a document in front of the parser that is not valid in the +--echo # character set it is claimed to be in - converting it would repair or +--echo # reject it first. What each label makes of the same bytes is +--echo # recorded here. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A gbk character whose second byte is the one JSON uses to escape. +--echo # Read as gbk it is one letter inside a string; read as anything +--echo # that treats bytes singly it starts an escape sequence and runs +--echo # off the end of the document. +--echo # + +CREATE TABLE t1 (j VARCHAR(60) CHARACTER SET gbk); +INSERT INTO t1 VALUES (X'7B2261223A22815C227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_gbk FROM t1; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs + FROM t1; +ALTER TABLE t1 MODIFY j VARBINARY(60); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t1; +ALTER TABLE t1 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_latin1 FROM t1; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t1; + +--echo # +--echo # 2. Bytes that are one character in utf8mb4 and two in latin1. The +--echo # document stays parseable either way, but the value it yields is +--echo # not the same. +--echo # + +CREATE TABLE t2 (j VARCHAR(60) CHARACTER SET utf8mb4); +INSERT INTO t2 VALUES (X'7B2261223A22C3A9227D'); +SELECT HEX(j) AS stored, HEX(JSON_EXTRACT(j, '$.a')) AS ext_utf8mb4, + CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_utf8mb4 FROM t2; +ALTER TABLE t2 MODIFY j VARBINARY(60); +ALTER TABLE t2 MODIFY j VARCHAR(60) CHARACTER SET latin1; +SELECT HEX(j) AS stored_unchanged, JSON_VALID(j) AS v FROM t2; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext, CHARSET(JSON_EXTRACT(j, '$.a')) AS cs, + CHAR_LENGTH(JSON_VALUE(j, '$.a')) AS chars_latin1 FROM t2; + +--echo # +--echo # 3. Relabelling in the other direction, to a character set that +--echo # cannot hold the bytes. This one does not go through quietly. +--echo # + +CREATE TABLE t3 (j VARBINARY(60)); +INSERT INTO t3 VALUES (X'7B2261223A22FF227D'); +SELECT HEX(j) AS stored, JSON_VALID(j) AS v_binary FROM t3; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +ALTER TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +ALTER IGNORE TABLE t3 MODIFY j VARCHAR(60) CHARACTER SET utf8mb4; +SELECT HEX(j) AS repaired, JSON_VALID(j) AS v FROM t3; +SELECT HEX(JSON_EXTRACT(j, '$.a')) AS ext FROM t3; + +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/func_json_nice_compose.result b/mysql-test/main/func_json_nice_compose.result new file mode 100644 index 0000000000000..380eb863e2f8f --- /dev/null +++ b/mysql-test/main/func_json_nice_compose.result @@ -0,0 +1,287 @@ +CREATE TABLE docs (id INT, js VARCHAR(500)); +INSERT INTO docs VALUES +(1, '{"a":1,"b":2}'), +(2, '{ "a" : 1 , "b" : 2 }'), +(3, '{"a": 1, "b": 2}'), +(4, '{"a":[1,2,{"c":3}],"b":{"d":[4,5]}}'), +(5, '[1,2,3]'), +(6, '[]'), +(7, '{}'), +(8, '{"a":[],"b":{}}'), +(9, '{"a":1.50,"b":1e2,"c":-0.0}'), +(10, '{"a":"x\\ty","b":"\\u0062","c":"quote\\"inside"}'), +(11, '{"a":null,"b":true,"c":false}'), +(12, '{"a":1,"a":2}'), +(13, '{"a":{"b":{"c":{"d":{"e":[1,{"f":2}]}}}}}'), +(14, '[{"a":1},{"a":2},[3,[4,5]]]'), +(15, '{"a":[1],"z":[2]}'); +CREATE TABLE steps (id INT, p1 VARCHAR(50), v1 VARCHAR(50), +p2 VARCHAR(50), v2 VARCHAR(50)); +INSERT INTO steps VALUES +(1, '$.a', 'one', '$.b', 'two'), +(2, '$.zz', 'new', '$.yy', 'other'), +(3, '$.a', 'one', '$.zz', 'new'), +(4, '$.zz', 'new', '$.a', 'one'), +(5, '$[0]', 'one', '$[1]', 'two'), +(6, '$[0]', 'one', '$[9]', 'two'), +(7, '$.a[0]', 'one', '$.a[1]', 'two'), +(8, '$.a[9]', 'one', '$.b', 'two'), +(9, '$.nomatch', 'none', '$.a', 'one'), +(10, '$.a', 'one', '$.nomatch', 'none'), +(11, '$.nomatch', 'none', '$.other', 'none'), +(12, '$.a.b.c', 'deep', '$.a.b', 'less'), +(13, '$.b.d[1]', 'one', '$.a[2].c', 'two'), +(14, '$[2][1][0]','one', '$[0].a', 'two'), +(15, '$.a[0]', 'one', '$.z[0]', 'two'); +# +# JSON_INSERT +# +SELECT d.id AS doc, s.id AS step, +JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) +AS BINARY)); +doc step once_at_end each_pass +# +# JSON_SET +# +SELECT d.id AS doc, s.id AS step, +JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_REPLACE +# +SELECT d.id AS doc, s.id AS step, +JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) +AS BINARY)); +doc step once_at_end each_pass +# +# JSON_REMOVE +# +SELECT d.id AS doc, s.id AS step, +JSON_REMOVE(d.js, s.p1, s.p2) AS once_at_end, +JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> +CAST(JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_ARRAY_APPEND +# +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), s.p2, s.v2) +AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), +s.p2, s.v2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_ARRAY_INSERT +# +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, +JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) +AS each_pass +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), +s.p2, s.v2) AS BINARY)); +doc step once_at_end each_pass +# +# JSON_MERGE and JSON_MERGE_PATCH, which fold their documents from +# the left in the same way +# +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, +JSON_MERGE(a.js, b.js, c.js) AS once_at_end, +JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS each_pass +FROM docs a, docs b, docs c +WHERE NOT (CAST(JSON_MERGE(a.js, b.js, c.js) AS BINARY) <=> +CAST(JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS BINARY)); +doc1 doc2 doc3 once_at_end each_pass +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, +JSON_MERGE_PATCH(a.js, b.js, c.js) AS once_at_end, +JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) AS each_pass +FROM docs a, docs b, docs c +WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js, c.js) AS BINARY) <=> +CAST(JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) +AS BINARY)); +doc1 doc2 doc3 once_at_end each_pass +# +# A value that is itself a document, which is written out rather +# than copied, and the wrapping of a value that is not an array +# +SELECT d.id AS doc, +JSON_INSERT(d.js, '$.zz', JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) +AS once_at_end, +JSON_INSERT(JSON_INSERT(d.js, '$.zz', +JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) AS each_pass +FROM docs d +WHERE NOT (CAST(JSON_INSERT(d.js, '$.zz', +JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) +AS BINARY) <=> +CAST(JSON_INSERT(JSON_INSERT(d.js, '$.zz', +JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), +'$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) +AS BINARY)); +doc once_at_end each_pass +SELECT d.id AS doc, +JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS once_at_end, +JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), '$.b', 'y') +AS each_pass +FROM docs d +WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS BINARY) <=> +CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), +'$.b', 'y') AS BINARY)); +doc once_at_end each_pass +# +# Three passes rather than two, so that a document written out in +# the loose form is read, edited and written out again +# +SELECT d.id AS doc, +JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS once_at_end, +JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), '$.q', 2), +'$.r', 3) AS each_pass +FROM docs d +WHERE NOT (CAST(JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS BINARY) <=> +CAST(JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), +'$.q', 2), '$.r', 3) AS BINARY)); +doc once_at_end each_pass +# +# Writing a document out twice gives what writing it out once gave +# +SELECT id FROM docs +WHERE NOT (CAST(JSON_LOOSE(js) AS BINARY) <=> +CAST(JSON_LOOSE(JSON_LOOSE(js)) AS BINARY)); +id +# +# Editing a document that was written out in the loose form before +# the first pass rather than after the last one gives the same +# answer. This is what lets a piece of a document be copied across +# as it stands: a piece of one written the loose way is written the +# loose way too. +# +SELECT d.id AS doc, s.id AS step, +JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_REMOVE(d.js, s.p1, s.p2) AS from_input, +JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> +CAST(JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT d.id AS doc, s.id AS step, +JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, +JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS from_loose +FROM docs d, steps s +WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> +CAST(JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) +AS BINARY)); +doc step from_input from_loose +SELECT a.id AS doc1, b.id AS doc2, +JSON_MERGE(a.js, b.js) AS from_input, +JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose +FROM docs a, docs b +WHERE NOT (CAST(JSON_MERGE(a.js, b.js) AS BINARY) <=> +CAST(JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS BINARY)); +doc1 doc2 from_input from_loose +SELECT a.id AS doc1, b.id AS doc2, +JSON_MERGE_PATCH(a.js, b.js) AS from_input, +JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose +FROM docs a, docs b +WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js) AS BINARY) <=> +CAST(JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) +AS BINARY)); +doc1 doc2 from_input from_loose +# +# What was actually produced while all of the above stayed quiet +# +# Every statement in this file passes by returning nothing, and +# a document that is nothing matches another that is nothing. +# These count the answers, so that a run in which the editing +# returned NULL throughout can no longer look like a run in +# which it agreed with itself throughout. +# +SELECT COUNT(*) AS pairs, +COUNT(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2)) AS answered +FROM docs d, steps s; +pairs answered +225 225 +SELECT COUNT(*) AS pairs, +COUNT(JSON_REMOVE(d.js, s.p1, s.p2)) AS answered +FROM docs d, steps s; +pairs answered +225 225 +SELECT COUNT(*) AS pairs, +COUNT(JSON_MERGE(a.js, b.js)) AS answered +FROM docs a, docs b; +pairs answered +225 225 +SELECT COUNT(*) AS docs, +COUNT(JSON_LOOSE(js)) AS answered +FROM docs; +docs answered +15 15 +# +# And what two of them actually say, so that a change to both +# sides at once still has somewhere to show itself. +# +SELECT id, JSON_INSERT(js, '$.p', 1, '$.q', 2) AS inserted +FROM docs WHERE id IN (1, 3, 5) ORDER BY id; +id inserted +1 {"a": 1, "b": 2, "p": 1, "q": 2} +3 {"a": 1, "b": 2, "p": 1, "q": 2} +5 [1, 2, 3] +SELECT id, JSON_LOOSE(js) AS loosened +FROM docs WHERE id IN (1, 2, 4) ORDER BY id; +id loosened +1 {"a": 1, "b": 2} +2 {"a": 1, "b": 2} +4 {"a": [1, 2, {"c": 3}], "b": {"d": [4, 5]}} +DROP TABLE docs, steps; diff --git a/mysql-test/main/func_json_nice_compose.test b/mysql-test/main/func_json_nice_compose.test new file mode 100644 index 0000000000000..29f88baf9447c --- /dev/null +++ b/mysql-test/main/func_json_nice_compose.test @@ -0,0 +1,304 @@ +# +# Editing a document that has already been written out in the loose form +# gives the same bytes as editing the original and writing the result out +# once at the end. +# +# The functions that edit a document with more than one path work through +# the paths one at a time, each pass reading what the pass before it +# wrote. Whether those intermediate documents are left as they came out +# of the input, or written out in the loose form as they go, must not +# change the answer. +# +# Every statement below asks the same question twice: once with all the +# paths given to one call, so that the document is written out once at +# the end, and once with the calls nested, so that each pass reads a +# document already written out in the loose form. A row comes back only +# when the two disagree, so a passing run says nothing at all. +# +# The bytes are compared, not the characters, so that a difference in +# case or in trailing space cannot pass for equality. +# +# Saying nothing at all is not enough on its own. A document compares +# equal to itself even when both of them are nothing: NULL is neither +# more nor less than NULL, so a fault that emptied BOTH sides would +# leave every one of these statements as quiet as a healthy server does. +# The section at the end therefore counts what came back and writes down +# what one of them says, so that the quiet above only means something +# while an answer is still being produced. +# + +CREATE TABLE docs (id INT, js VARCHAR(500)); +INSERT INTO docs VALUES + (1, '{"a":1,"b":2}'), + (2, '{ "a" : 1 , "b" : 2 }'), + (3, '{"a": 1, "b": 2}'), + (4, '{"a":[1,2,{"c":3}],"b":{"d":[4,5]}}'), + (5, '[1,2,3]'), + (6, '[]'), + (7, '{}'), + (8, '{"a":[],"b":{}}'), + (9, '{"a":1.50,"b":1e2,"c":-0.0}'), + (10, '{"a":"x\\ty","b":"\\u0062","c":"quote\\"inside"}'), + (11, '{"a":null,"b":true,"c":false}'), + (12, '{"a":1,"a":2}'), + (13, '{"a":{"b":{"c":{"d":{"e":[1,{"f":2}]}}}}}'), + (14, '[{"a":1},{"a":2},[3,[4,5]]]'), + (15, '{"a":[1],"z":[2]}'); + +CREATE TABLE steps (id INT, p1 VARCHAR(50), v1 VARCHAR(50), + p2 VARCHAR(50), v2 VARCHAR(50)); +INSERT INTO steps VALUES + (1, '$.a', 'one', '$.b', 'two'), + (2, '$.zz', 'new', '$.yy', 'other'), + (3, '$.a', 'one', '$.zz', 'new'), + (4, '$.zz', 'new', '$.a', 'one'), + (5, '$[0]', 'one', '$[1]', 'two'), + (6, '$[0]', 'one', '$[9]', 'two'), + (7, '$.a[0]', 'one', '$.a[1]', 'two'), + (8, '$.a[9]', 'one', '$.b', 'two'), + (9, '$.nomatch', 'none', '$.a', 'one'), + (10, '$.a', 'one', '$.nomatch', 'none'), + (11, '$.nomatch', 'none', '$.other', 'none'), + (12, '$.a.b.c', 'deep', '$.a.b', 'less'), + (13, '$.b.d[1]', 'one', '$.a[2].c', 'two'), + (14, '$[2][1][0]','one', '$[0].a', 'two'), + (15, '$.a[0]', 'one', '$.z[0]', 'two'); + +--echo # +--echo # JSON_INSERT +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_INSERT(JSON_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) + AS BINARY)); + +--echo # +--echo # JSON_SET +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_SET(JSON_SET(d.js, s.p1, s.v1), s.p2, s.v2) AS BINARY)); + +--echo # +--echo # JSON_REPLACE +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_REPLACE(JSON_REPLACE(d.js, s.p1, s.v1), s.p2, s.v2) + AS BINARY)); + +--echo # +--echo # JSON_REMOVE +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_REMOVE(d.js, s.p1, s.p2) AS once_at_end, + JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> + CAST(JSON_REMOVE(JSON_REMOVE(d.js, s.p1), s.p2) AS BINARY)); + +--echo # +--echo # JSON_ARRAY_APPEND +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), s.p2, s.v2) + AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, s.p1, s.v1), + s.p2, s.v2) AS BINARY)); + +--echo # +--echo # JSON_ARRAY_INSERT +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS once_at_end, + JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), s.p2, s.v2) + AS each_pass + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_INSERT(JSON_ARRAY_INSERT(d.js, s.p1, s.v1), + s.p2, s.v2) AS BINARY)); + +--echo # +--echo # JSON_MERGE and JSON_MERGE_PATCH, which fold their documents from +--echo # the left in the same way +--echo # +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, + JSON_MERGE(a.js, b.js, c.js) AS once_at_end, + JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS each_pass + FROM docs a, docs b, docs c + WHERE NOT (CAST(JSON_MERGE(a.js, b.js, c.js) AS BINARY) <=> + CAST(JSON_MERGE(JSON_MERGE(a.js, b.js), c.js) AS BINARY)); + +SELECT a.id AS doc1, b.id AS doc2, c.id AS doc3, + JSON_MERGE_PATCH(a.js, b.js, c.js) AS once_at_end, + JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) AS each_pass + FROM docs a, docs b, docs c + WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js, c.js) AS BINARY) <=> + CAST(JSON_MERGE_PATCH(JSON_MERGE_PATCH(a.js, b.js), c.js) + AS BINARY)); + +--echo # +--echo # A value that is itself a document, which is written out rather +--echo # than copied, and the wrapping of a value that is not an array +--echo # +SELECT d.id AS doc, + JSON_INSERT(d.js, '$.zz', JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) + AS once_at_end, + JSON_INSERT(JSON_INSERT(d.js, '$.zz', + JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) AS each_pass + FROM docs d + WHERE NOT (CAST(JSON_INSERT(d.js, '$.zz', + JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q'), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) + AS BINARY) <=> + CAST(JSON_INSERT(JSON_INSERT(d.js, '$.zz', + JSON_EXTRACT('{"q":[1,{"r":2}]}','$.q')), + '$.yy', JSON_EXTRACT('{"q":{"r":3}}','$.q')) + AS BINARY)); + +SELECT d.id AS doc, + JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS once_at_end, + JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), '$.b', 'y') + AS each_pass + FROM docs d + WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, '$.a', 'x', '$.b', 'y') AS BINARY) <=> + CAST(JSON_ARRAY_APPEND(JSON_ARRAY_APPEND(d.js, '$.a', 'x'), + '$.b', 'y') AS BINARY)); + +--echo # +--echo # Three passes rather than two, so that a document written out in +--echo # the loose form is read, edited and written out again +--echo # +SELECT d.id AS doc, + JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS once_at_end, + JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), '$.q', 2), + '$.r', 3) AS each_pass + FROM docs d + WHERE NOT (CAST(JSON_INSERT(d.js, '$.p', 1, '$.q', 2, '$.r', 3) AS BINARY) <=> + CAST(JSON_INSERT(JSON_INSERT(JSON_INSERT(d.js, '$.p', 1), + '$.q', 2), '$.r', 3) AS BINARY)); + +--echo # +--echo # Writing a document out twice gives what writing it out once gave +--echo # +SELECT id FROM docs + WHERE NOT (CAST(JSON_LOOSE(js) AS BINARY) <=> + CAST(JSON_LOOSE(JSON_LOOSE(js)) AS BINARY)); + +--echo # +--echo # Editing a document that was written out in the loose form before +--echo # the first pass rather than after the last one gives the same +--echo # answer. This is what lets a piece of a document be copied across +--echo # as it stands: a piece of one written the loose way is written the +--echo # loose way too. +--echo # +SELECT d.id AS doc, s.id AS step, + JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_SET(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_SET(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_REPLACE(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_REPLACE(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_REMOVE(d.js, s.p1, s.p2) AS from_input, + JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_REMOVE(d.js, s.p1, s.p2) AS BINARY) <=> + CAST(JSON_REMOVE(JSON_LOOSE(d.js), s.p1, s.p2) AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_APPEND(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_APPEND(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT d.id AS doc, s.id AS step, + JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS from_input, + JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS from_loose + FROM docs d, steps s + WHERE NOT (CAST(JSON_ARRAY_INSERT(d.js, s.p1, s.v1, s.p2, s.v2) AS BINARY) <=> + CAST(JSON_ARRAY_INSERT(JSON_LOOSE(d.js), s.p1, s.v1, s.p2, s.v2) + AS BINARY)); + +SELECT a.id AS doc1, b.id AS doc2, + JSON_MERGE(a.js, b.js) AS from_input, + JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose + FROM docs a, docs b + WHERE NOT (CAST(JSON_MERGE(a.js, b.js) AS BINARY) <=> + CAST(JSON_MERGE(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS BINARY)); + +SELECT a.id AS doc1, b.id AS doc2, + JSON_MERGE_PATCH(a.js, b.js) AS from_input, + JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) AS from_loose + FROM docs a, docs b + WHERE NOT (CAST(JSON_MERGE_PATCH(a.js, b.js) AS BINARY) <=> + CAST(JSON_MERGE_PATCH(JSON_LOOSE(a.js), JSON_LOOSE(b.js)) + AS BINARY)); + +--echo # +--echo # What was actually produced while all of the above stayed quiet +--echo # +--echo # Every statement in this file passes by returning nothing, and +--echo # a document that is nothing matches another that is nothing. +--echo # These count the answers, so that a run in which the editing +--echo # returned NULL throughout can no longer look like a run in +--echo # which it agreed with itself throughout. +--echo # +SELECT COUNT(*) AS pairs, + COUNT(JSON_INSERT(d.js, s.p1, s.v1, s.p2, s.v2)) AS answered + FROM docs d, steps s; +SELECT COUNT(*) AS pairs, + COUNT(JSON_REMOVE(d.js, s.p1, s.p2)) AS answered + FROM docs d, steps s; +SELECT COUNT(*) AS pairs, + COUNT(JSON_MERGE(a.js, b.js)) AS answered + FROM docs a, docs b; +SELECT COUNT(*) AS docs, + COUNT(JSON_LOOSE(js)) AS answered + FROM docs; + +--echo # +--echo # And what two of them actually say, so that a change to both +--echo # sides at once still has somewhere to show itself. +--echo # +SELECT id, JSON_INSERT(js, '$.p', 1, '$.q', 2) AS inserted + FROM docs WHERE id IN (1, 3, 5) ORDER BY id; +SELECT id, JSON_LOOSE(js) AS loosened + FROM docs WHERE id IN (1, 2, 4) ORDER BY id; + +DROP TABLE docs, steps; diff --git a/mysql-test/main/func_json_oom.result b/mysql-test/main/func_json_oom.result new file mode 100644 index 0000000000000..6fd7573d3ae67 --- /dev/null +++ b/mysql-test/main/func_json_oom.result @@ -0,0 +1,309 @@ +# +# Reformatting a document when there is no room to grow the buffer. +# +# The reformatting pass asks for the length of its input plus a +# little, then writes the loose form into it. The loose form spends +# two characters where the compact form spends one, so an input +# carrying enough separators needs more room than was asked for, and +# the buffer has to grow part way through being written. +# +# These tests make that growing fail. What is being pinned is that +# every one of them ends either in an error or in a value that +# parses. A document with a piece missing out of the middle is not +# an answer, and no caller checks for one. +# +SET NAMES utf8mb4; +# +# 1. How much bigger the loose form is, which is what makes the +# room asked for too little. +# +SET @compact = CONCAT('[', REPEAT('1,', 49), '1]'); +SET @pairs = CONCAT('{', REPEAT('"a":1,', 24), '"a":1}'); +SET @nested = CONCAT('{"k":[', REPEAT('1,', 49), '1]}'); +SET @small = '[1,2,3]'; +SELECT LENGTH(@compact) AS compact_len, +LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; +compact_len grew_by +101 49 +SELECT LENGTH(@pairs) AS compact_len, +LENGTH(JSON_LOOSE(@pairs)) - LENGTH(@pairs) AS grew_by; +compact_len grew_by +151 49 +# a document short enough that the room asked for is enough +SELECT LENGTH(@small) AS compact_len, +LENGTH(JSON_LOOSE(@small)) - LENGTH(@small) AS grew_by; +compact_len grew_by +7 2 +# +# 2. Which uses of a formatter actually reformat anything. +# +# A formatter handed to something that wants JSON gives it the +# argument untouched, formatting being no business of a reader. +# Only a use that wants text runs the reformatting pass at all. +# +SELECT JSON_LOOSE(@pairs) AS as_text; +as_text +{"a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1} +SELECT JSON_TYPE(JSON_LOOSE(@pairs)) AS asked_as_json; +asked_as_json +OBJECT +SELECT CONCAT(JSON_LOOSE(@pairs), '') AS asked_as_text; +asked_as_text +{"a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1, "a": 1} +SELECT JSON_VALID(JSON_LOOSE(@pairs)) AS reader_sees_the_argument, +JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS reader_sees_the_output; +reader_sees_the_argument reader_sees_the_output +1 1 +# +# 3. The formatters, with no room to grow. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; +SELECT LENGTH(JSON_LOOSE(@compact)) AS len; +SET SESSION debug_dbug = DEFAULT; +# the same statements once the room is there again +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; +parses +1 +# +# 4. A document short enough not to need the room, which must be +# unaffected. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@small), '')) AS parses; +parses +1 +SELECT JSON_LOOSE(@small) AS v; +v +[1, 2, 3] +SET SESSION debug_dbug = DEFAULT; +# +# 5. The editing functions, which reformat what they have built +# before returning it. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; +# and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; +parses +1 +# +# 6. Reading a value out, which reformats what it read. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; +parses +1 +# +# 7. Where the value would go if it were returned at all. +# +# The allocator reports the failure itself, and reports it as +# fatal, so the statement ends before anything can be assigned and +# a handler cannot let execution carry on past it. This is what +# keeps a half written document from being seen. +# +SET @out = 'untouched'; +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SET @out = JSON_LOOSE(@compact); +SET SESSION debug_dbug = DEFAULT; +SELECT @out AS assigned_value; +assigned_value +untouched +SET @reached = 0; +SET @out2 = 'untouched'; +CREATE PROCEDURE p() +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET @out2 = JSON_LOOSE(@compact); +SET @reached = 1; +END $$ +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +CALL p(); +SET SESSION debug_dbug = DEFAULT; +SELECT @reached AS handler_carried_on, @out2 AS assigned_value; +handler_carried_on assigned_value +0 untouched +DROP PROCEDURE p; +# +# 8. Several rows, so the failure is met more than once and the +# statement does not turn on it happening first. +# +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2,3]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses +FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses +FROM t1 ORDER BY id; +id parses +1 1 +2 1 +3 1 +DROP TABLE t1; +# +# The sections above fail a buffer that is being grown, which +# reaches every write the reformatting pass makes. The ones below +# fail a single write instead, and they exist because growing +# cannot reach those writes at all: the room a document takes is +# asked for before it is written, so they never run short, and the +# arms that cope with their failing are arms nobody has run. Each +# is a pair - the statement without the failure and with it - so +# that what comes back is read against what the statement answers +# when nothing is wrong. +# +# +# 9. The quotes around a value. One at each end of the escaping, +# and failing the first never reaches the second, so they are +# asked about one at a time. +# +SELECT JSON_ARRAY('abc', 'def') AS whole, +JSON_OBJECT('k', 'abc') AS whole_value; +whole whole_value +["abc", "def"] {"k": "abc"} +SET SESSION debug_dbug = '+d,json_value_open_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_open_quote; +no_open_quote +NULL +SELECT JSON_OBJECT('k', 'abc') AS no_open_quote_value; +no_open_quote_value +NULL +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_value_close_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_close_quote; +no_close_quote +NULL +SELECT JSON_OBJECT('k', 'abc') AS no_close_quote_value; +no_close_quote_value +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 10. The room the escaping asks for, which is asked for once and +# up front at the widest the value could come to. +# +SET SESSION debug_dbug = '+d,json_escape_reserve_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_room_to_escape; +no_room_to_escape +NULL +SELECT JSON_QUOTE('abc') AS no_room_to_quote; +no_room_to_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 11. The quote that opens a key. A key is a string and nothing +# else, so it is written the way a value that is a string is. +# +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole; +whole +{"k": 1, "l": 2} +SET SESSION debug_dbug = '+d,json_keyname_quote_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_key_quote; +no_key_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 12. The quotes JSON_QUOTE puts round its argument, which are the +# same two writes as section 9 made by the function whose +# whole answer they are. +# +SELECT JSON_QUOTE('abc') AS whole, JSON_QUOTE('a"b') AS whole_escaped; +whole whole_escaped +"abc" "a\"b" +SET SESSION debug_dbug = '+d,json_quote_open_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_open_quote; +no_open_quote +NULL +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_quote_close_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_close_quote; +no_close_quote +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 13. JSON_UNQUOTE returning an argument that is not a quoted +# string. It comes back as it stands, which takes a +# conversion where it carries no character set and a copy +# where it carries another one. Both roads are here, failing +# as they do in different places. +# +SELECT JSON_UNQUOTE(_binary'abc') AS from_binary, +JSON_UNQUOTE(_latin1'abc') AS from_latin1; +from_binary from_latin1 +abc abc +SET SESSION debug_dbug = '+d,json_unquote_as_is_out_of_memory'; +SELECT JSON_UNQUOTE(_binary'abc') AS no_room_binary; +no_room_binary +NULL +SELECT JSON_UNQUOTE(_latin1'abc') AS no_room_latin1; +no_room_latin1 +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 14. A value that is a document written in another character set +# than the one it is joining, which is converted on the way in +# so that what is spliced is written the way the rest of the +# document is. Both directions, a narrow value into a wide +# document and a wide one into a narrow document. +# +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS whole_into_ucs2; +whole_into_ucs2 +1 +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) +AS whole_from_latin1; +whole_from_latin1 +1 +SET SESSION debug_dbug = '+d,json_splice_convert_out_of_memory'; +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', +JSON_COMPACT('{"x":2}'))) AS no_room_into_ucs2; +no_room_into_ucs2 +NULL +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', +JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) +AS no_room_from_latin1; +no_room_from_latin1 +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 15. A path whose key is written in another character set, which +# is converted before it can be spliced into the document - a +# write of its own, and one that reaches nothing else. +# +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole, +JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS whole_set; +whole whole_set +{"a": 1, "b": 2} {"a": 1, "c": 3} +SET SESSION debug_dbug = '+d,json_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS no_room_for_key; +no_room_for_key +NULL +SELECT JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS no_room_for_key_set; +no_room_for_key_set +NULL +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_oom.test b/mysql-test/main/func_json_oom.test new file mode 100644 index 0000000000000..2c13fc5ccc99d --- /dev/null +++ b/mysql-test/main/func_json_oom.test @@ -0,0 +1,275 @@ +--source include/have_debug.inc + +--echo # +--echo # Reformatting a document when there is no room to grow the buffer. +--echo # +--echo # The reformatting pass asks for the length of its input plus a +--echo # little, then writes the loose form into it. The loose form spends +--echo # two characters where the compact form spends one, so an input +--echo # carrying enough separators needs more room than was asked for, and +--echo # the buffer has to grow part way through being written. +--echo # +--echo # These tests make that growing fail. What is being pinned is that +--echo # every one of them ends either in an error or in a value that +--echo # parses. A document with a piece missing out of the middle is not +--echo # an answer, and no caller checks for one. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. How much bigger the loose form is, which is what makes the +--echo # room asked for too little. +--echo # + +SET @compact = CONCAT('[', REPEAT('1,', 49), '1]'); +SET @pairs = CONCAT('{', REPEAT('"a":1,', 24), '"a":1}'); +SET @nested = CONCAT('{"k":[', REPEAT('1,', 49), '1]}'); +SET @small = '[1,2,3]'; +SELECT LENGTH(@compact) AS compact_len, + LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; +SELECT LENGTH(@pairs) AS compact_len, + LENGTH(JSON_LOOSE(@pairs)) - LENGTH(@pairs) AS grew_by; +--echo # a document short enough that the room asked for is enough +SELECT LENGTH(@small) AS compact_len, + LENGTH(JSON_LOOSE(@small)) - LENGTH(@small) AS grew_by; + +--echo # +--echo # 2. Which uses of a formatter actually reformat anything. +--echo # +--echo # A formatter handed to something that wants JSON gives it the +--echo # argument untouched, formatting being no business of a reader. +--echo # Only a use that wants text runs the reformatting pass at all. +--echo # + +SELECT JSON_LOOSE(@pairs) AS as_text; +SELECT JSON_TYPE(JSON_LOOSE(@pairs)) AS asked_as_json; +SELECT CONCAT(JSON_LOOSE(@pairs), '') AS asked_as_text; +SELECT JSON_VALID(JSON_LOOSE(@pairs)) AS reader_sees_the_argument, + JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS reader_sees_the_output; + +--echo # +--echo # 3. The formatters, with no room to grow. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; +--error 0,5 +SELECT LENGTH(JSON_LOOSE(@compact)) AS len; +SET SESSION debug_dbug = DEFAULT; + +--echo # the same statements once the room is there again +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@compact), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@pairs), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_DETAILED(@compact), '')) AS parses; + +--echo # +--echo # 4. A document short enough not to need the room, which must be +--echo # unaffected. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_VALID(CONCAT(JSON_LOOSE(@small), '')) AS parses; +SELECT JSON_LOOSE(@small) AS v; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 5. The editing functions, which reformat what they have built +--echo # before returning it. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; + +--echo # and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(@compact, '$[0]', 9), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_REMOVE(@compact, '$[0]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_MERGE(@compact, '[7]'), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(@compact, '$', 7), '')) AS parses; + +--echo # +--echo # 6. Reading a value out, which reformats what it read. +--echo # + +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; +SET SESSION debug_dbug = DEFAULT; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@nested, '$.k'), '')) AS parses; + +--echo # +--echo # 7. Where the value would go if it were returned at all. +--echo # +--echo # The allocator reports the failure itself, and reports it as +--echo # fatal, so the statement ends before anything can be assigned and +--echo # a handler cannot let execution carry on past it. This is what +--echo # keeps a half written document from being seen. +--echo # + +SET @out = 'untouched'; +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SET @out = JSON_LOOSE(@compact); +SET SESSION debug_dbug = DEFAULT; +SELECT @out AS assigned_value; + +SET @reached = 0; +SET @out2 = 'untouched'; +--delimiter $$ +CREATE PROCEDURE p() +BEGIN + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET @out2 = JSON_LOOSE(@compact); + SET @reached = 1; +END $$ +--delimiter ; +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +CALL p(); +SET SESSION debug_dbug = DEFAULT; +SELECT @reached AS handler_carried_on, @out2 AS assigned_value; +DROP PROCEDURE p; + +--echo # +--echo # 8. Several rows, so the failure is met more than once and the +--echo # statement does not turn on it happening first. +--echo # + +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2,3]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses + FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_LOOSE(j), '')) AS parses + FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # The sections above fail a buffer that is being grown, which +--echo # reaches every write the reformatting pass makes. The ones below +--echo # fail a single write instead, and they exist because growing +--echo # cannot reach those writes at all: the room a document takes is +--echo # asked for before it is written, so they never run short, and the +--echo # arms that cope with their failing are arms nobody has run. Each +--echo # is a pair - the statement without the failure and with it - so +--echo # that what comes back is read against what the statement answers +--echo # when nothing is wrong. +--echo # + +--echo # +--echo # 9. The quotes around a value. One at each end of the escaping, +--echo # and failing the first never reaches the second, so they are +--echo # asked about one at a time. +--echo # +SELECT JSON_ARRAY('abc', 'def') AS whole, + JSON_OBJECT('k', 'abc') AS whole_value; + +SET SESSION debug_dbug = '+d,json_value_open_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_open_quote; +SELECT JSON_OBJECT('k', 'abc') AS no_open_quote_value; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_value_close_quote_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_close_quote; +SELECT JSON_OBJECT('k', 'abc') AS no_close_quote_value; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 10. The room the escaping asks for, which is asked for once and +--echo # up front at the widest the value could come to. +--echo # +SET SESSION debug_dbug = '+d,json_escape_reserve_out_of_memory'; +SELECT JSON_ARRAY('abc', 'def') AS no_room_to_escape; +SELECT JSON_QUOTE('abc') AS no_room_to_quote; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 11. The quote that opens a key. A key is a string and nothing +--echo # else, so it is written the way a value that is a string is. +--echo # +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole; + +SET SESSION debug_dbug = '+d,json_keyname_quote_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_key_quote; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 12. The quotes JSON_QUOTE puts round its argument, which are the +--echo # same two writes as section 9 made by the function whose +--echo # whole answer they are. +--echo # +SELECT JSON_QUOTE('abc') AS whole, JSON_QUOTE('a"b') AS whole_escaped; + +SET SESSION debug_dbug = '+d,json_quote_open_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_open_quote; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_quote_close_out_of_memory'; +SELECT JSON_QUOTE('abc') AS no_close_quote; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 13. JSON_UNQUOTE returning an argument that is not a quoted +--echo # string. It comes back as it stands, which takes a +--echo # conversion where it carries no character set and a copy +--echo # where it carries another one. Both roads are here, failing +--echo # as they do in different places. +--echo # +SELECT JSON_UNQUOTE(_binary'abc') AS from_binary, + JSON_UNQUOTE(_latin1'abc') AS from_latin1; + +SET SESSION debug_dbug = '+d,json_unquote_as_is_out_of_memory'; +SELECT JSON_UNQUOTE(_binary'abc') AS no_room_binary; +SELECT JSON_UNQUOTE(_latin1'abc') AS no_room_latin1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 14. A value that is a document written in another character set +--echo # than the one it is joining, which is converted on the way in +--echo # so that what is spliced is written the way the rest of the +--echo # document is. Both directions, a narrow value into a wide +--echo # document and a wide one into a narrow document. +--echo # +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS whole_into_ucs2; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) + AS whole_from_latin1; + +SET SESSION debug_dbug = '+d,json_splice_convert_out_of_memory'; +SELECT JSON_VALID(JSON_SET(CONVERT('{"a":1}' USING ucs2), '$.b', + JSON_COMPACT('{"x":2}'))) AS no_room_into_ucs2; +SELECT JSON_VALID(JSON_SET('{"a":1}', '$.b', + JSON_COMPACT(CONVERT('{"x":2}' USING latin1)))) + AS no_room_from_latin1; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 15. A path whose key is written in another character set, which +--echo # is converted before it can be spliced into the document - a +--echo # write of its own, and one that reaches nothing else. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole, + JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS whole_set; + +SET SESSION debug_dbug = '+d,json_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS no_room_for_key; +SELECT JSON_SET(JSON_OBJECT('a', 1), _latin1'$.c', 3) AS no_room_for_key_set; +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_overlaps_depth.result b/mysql-test/main/func_json_overlaps_depth.result new file mode 100644 index 0000000000000..fae7164bbb195 --- /dev/null +++ b/mysql-test/main/func_json_overlaps_depth.result @@ -0,0 +1,198 @@ +# +# A document nested deeper than the parser will go, handed to the +# functions that compare one document against another. +# +# The parser stops at a fixed depth and records that it has. What it +# does with its nesting counter on the way out decides what a caller +# that keeps scanning afterwards will read, the counter being the +# index into the stack of states the scanner works from. +# +SET NAMES utf8mb4; +# +# 1. One container past the limit, in either argument. +# +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'{"a":1}') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_OVERLAPS('{"a":1}', +CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +# the other argument's kind does not matter +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'"s"') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'[[1]]') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +'[1,2]') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +# both arguments past the limit +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), +CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +# +# 2. The same depth reached through objects rather than arrays. +# +SELECT JSON_OVERLAPS(CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32)), +'{"a":1}') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 156 +SELECT JSON_OVERLAPS('{"a":1}', +CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32))) AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 156 +# and mixed, an array holding an object chain. It is one container +# past the limit exactly as the two above are, and the complaint it +# draws has to say so: the document is complete and balanced, and +# too deep is the only thing wrong with it. +SELECT JSON_OVERLAPS(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']'), +'{"a":1}') AS v; +v +0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 152 +SELECT JSON_DEPTH(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']')) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 152 +# +# 3. One container inside the limit, which is a document and compares. +# +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), +'{"a":1}') AS v; +v +0 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), +CONCAT(REPEAT('[',31),'1',REPEAT(']',31))) AS v; +v +1 +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), +'[1,2]') AS v; +v +0 +# +# 4. What the neighbouring functions answer on the same input. +# +SELECT JSON_CONTAINS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_contains' at position 32 +SELECT JSON_EQUALS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_equals' at position 32 +SELECT JSON_DEPTH(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +SELECT JSON_VALID(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +0 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_valid' at position 32 +SELECT JSON_TYPE(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_type' at position 32 +# +# 5. Rows of them, so the engine is reused across evaluations. +# +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES +(1, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))), +(2, '[1,2]'), +(3, CONCAT(REPEAT('[',31),'1',REPEAT(']',31))), +(4, '{"a":1}'); +SELECT id, JSON_OVERLAPS(j, '[1,2]') AS v FROM t1 ORDER BY id; +id v +1 0 +2 1 +3 0 +4 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT id, JSON_OVERLAPS('[1,2]', j) AS v FROM t1 ORDER BY id; +id v +1 0 +2 1 +3 0 +4 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +SELECT id, JSON_OVERLAPS(j, j) AS v FROM t1 ORDER BY id; +id v +1 0 +2 1 +3 1 +4 1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +DROP TABLE t1; +# +# 6. Where the complaint says the document went too deep. +# +# The position is a property of the document alone. The second +# argument decides how much comparing is attempted before the depth +# is reached, and must not decide where the scanner is said to have +# stopped: all four below are the same first argument, so all four +# positions have to be the same one. +# +# It is also the same document the other functions are given in +# section 4, so it has to be the position they report for it. +# +SET @d = CONCAT(REPEAT('[',32),'1',REPEAT(']',32)); +SELECT JSON_OVERLAPS(@d,'{"a":1}') AS obj, JSON_OVERLAPS(@d,'"s"') AS scalar, +JSON_OVERLAPS(@d,'[[1]]') AS nested, JSON_OVERLAPS(@d,'[1,2]') AS flat; +obj scalar nested flat +0 0 0 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_overlaps' at position 32 +SELECT JSON_DEPTH(@d) AS v; +v +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_depth' at position 32 +# and the other way round, where it is argument 2 that is too deep +SELECT JSON_OVERLAPS('{"a":1}',@d) AS obj, JSON_OVERLAPS('"s"',@d) AS scalar, +JSON_OVERLAPS('[[1]]',@d) AS nested, JSON_OVERLAPS('[1,2]',@d) AS flat; +obj scalar nested flat +0 0 0 0 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_overlaps' at position 32 +SET @d = NULL; diff --git a/mysql-test/main/func_json_overlaps_depth.test b/mysql-test/main/func_json_overlaps_depth.test new file mode 100644 index 0000000000000..ea050230613dc --- /dev/null +++ b/mysql-test/main/func_json_overlaps_depth.test @@ -0,0 +1,106 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # A document nested deeper than the parser will go, handed to the +--echo # functions that compare one document against another. +--echo # +--echo # The parser stops at a fixed depth and records that it has. What it +--echo # does with its nesting counter on the way out decides what a caller +--echo # that keeps scanning afterwards will read, the counter being the +--echo # index into the stack of states the scanner works from. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. One container past the limit, in either argument. +--echo # + +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '{"a":1}') AS v; +SELECT JSON_OVERLAPS('{"a":1}', + CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +--echo # the other argument's kind does not matter +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '"s"') AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '[[1]]') AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + '[1,2]') AS v; +--echo # both arguments past the limit +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), + CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; + +--echo # +--echo # 2. The same depth reached through objects rather than arrays. +--echo # + +SELECT JSON_OVERLAPS(CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32)), + '{"a":1}') AS v; +SELECT JSON_OVERLAPS('{"a":1}', + CONCAT(REPEAT('{"a":',32),'1',REPEAT('}',32))) AS v; +--echo # and mixed, an array holding an object chain. It is one container +--echo # past the limit exactly as the two above are, and the complaint it +--echo # draws has to say so: the document is complete and balanced, and +--echo # too deep is the only thing wrong with it. +SELECT JSON_OVERLAPS(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']'), + '{"a":1}') AS v; +SELECT JSON_DEPTH(CONCAT('[',REPEAT('{"a":',31),'1',REPEAT('}',31),']')) AS v; + +--echo # +--echo # 3. One container inside the limit, which is a document and compares. +--echo # + +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), + '{"a":1}') AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), + CONCAT(REPEAT('[',31),'1',REPEAT(']',31))) AS v; +SELECT JSON_OVERLAPS(CONCAT(REPEAT('[',31),'1',REPEAT(']',31)), + '[1,2]') AS v; + +--echo # +--echo # 4. What the neighbouring functions answer on the same input. +--echo # + +SELECT JSON_CONTAINS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +SELECT JSON_EQUALS(CONCAT(REPEAT('[',32),'1',REPEAT(']',32)), '1') AS v; +SELECT JSON_DEPTH(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +SELECT JSON_VALID(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; +SELECT JSON_TYPE(CONCAT(REPEAT('[',32),'1',REPEAT(']',32))) AS v; + +--echo # +--echo # 5. Rows of them, so the engine is reused across evaluations. +--echo # + +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES + (1, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))), + (2, '[1,2]'), + (3, CONCAT(REPEAT('[',31),'1',REPEAT(']',31))), + (4, '{"a":1}'); +SELECT id, JSON_OVERLAPS(j, '[1,2]') AS v FROM t1 ORDER BY id; +SELECT id, JSON_OVERLAPS('[1,2]', j) AS v FROM t1 ORDER BY id; +SELECT id, JSON_OVERLAPS(j, j) AS v FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 6. Where the complaint says the document went too deep. +--echo # +--echo # The position is a property of the document alone. The second +--echo # argument decides how much comparing is attempted before the depth +--echo # is reached, and must not decide where the scanner is said to have +--echo # stopped: all four below are the same first argument, so all four +--echo # positions have to be the same one. +--echo # +--echo # It is also the same document the other functions are given in +--echo # section 4, so it has to be the position they report for it. +--echo # + +SET @d = CONCAT(REPEAT('[',32),'1',REPEAT(']',32)); +SELECT JSON_OVERLAPS(@d,'{"a":1}') AS obj, JSON_OVERLAPS(@d,'"s"') AS scalar, + JSON_OVERLAPS(@d,'[[1]]') AS nested, JSON_OVERLAPS(@d,'[1,2]') AS flat; +SELECT JSON_DEPTH(@d) AS v; +--echo # and the other way round, where it is argument 2 that is too deep +SELECT JSON_OVERLAPS('{"a":1}',@d) AS obj, JSON_OVERLAPS('"s"',@d) AS scalar, + JSON_OVERLAPS('[[1]]',@d) AS nested, JSON_OVERLAPS('[1,2]',@d) AS flat; +SET @d = NULL; diff --git a/mysql-test/main/func_json_overlaps_diag.result b/mysql-test/main/func_json_overlaps_diag.result new file mode 100644 index 0000000000000..b3b56fb721216 --- /dev/null +++ b/mysql-test/main/func_json_overlaps_diag.result @@ -0,0 +1,155 @@ +# +# Where JSON_OVERLAPS says a document broke. +# +# The comparison walks both arguments together and puts an engine +# back where it was to try the next candidate. A refusal met +# part-way through has to survive being put back, and the place it +# is reported from has to be where the document broke, not +# wherever the walk happened to stop afterwards. +# +# There is one right attest to each of these documents and it can +# be had elsewhere: JSON_VALID and JSON_EXTRACT read the same bytes +# with no comparison around them. Every case below is recorded +# beside those two, so the position is pinned to something other +# than itself. +# +# +# 1. What the same bytes are worth to a reader with nothing else +# going on. These are the positions every case below must give. +# +SELECT JSON_VALID('{"a":1 "b":2}') AS ok; +ok +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 8 +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +SELECT JSON_EXTRACT('{"a":1, "b":2 "c":3}', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 15 +SELECT JSON_EXTRACT('{"a":{"p":1 "q":2}}', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 13 +SELECT JSON_EXTRACT('[{"z":9},{"b":2 "c":3}]', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 17 +SELECT JSON_EXTRACT('[1,2,3 4]', '$') AS v; +v +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 8 +# +# 2. The break is in argument 2, met after the walk has turned at +# least once. +# +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1 "b":2}') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +SELECT JSON_OVERLAPS('{"z":0}', '{"a":1, "b":2 "c":3}') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 15 +# +# 3. The break is below the top level, so the walk is further in +# when it meets it. +# +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1 "q":2}}') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 13 +# +# 4. The break is in argument 1, and inside an element of it. +# +SELECT JSON_OVERLAPS('[{"z":9},{"b":2 "c":3}]', '[{"a":1}]') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 17 +# +# 5. A refusal that is reached only after every candidate has been +# tried and put back. Nothing said so before: the walk ended on +# the refusal exactly as it ends on running out of candidates, +# and the answer came back with no diagnostic at all. +# +SELECT JSON_OVERLAPS('[9]', '[1,2,3 4]') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +SELECT JSON_OVERLAPS('[{"a":1}]', '[{"z":9},{"b":2 "c":3}]') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 17 +# +# 6. The same documents out of a table, so no one statement +# settles another, and both argument orders together. +# +CREATE TABLE t1 (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t1 VALUES +(1, '{"b":9}', '{"a":1 "b":2}'), +(2, '{"a":{"p":1}}', '{"a":{"p":1 "q":2}}'), +(3, '[9]', '[1,2,3 4]'), +(4, '{"b":9}', '{"a":1, "b":2}'); +SELECT id, JSON_OVERLAPS(a, b) AS v FROM t1 ORDER BY id; +id v +1 0 +2 0 +3 0 +4 0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 13 +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 8 +SELECT id, JSON_OVERLAPS(b, a) AS v FROM t1 ORDER BY id; +id v +1 0 +2 0 +3 0 +4 0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 8 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 13 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_overlaps' at position 8 +DROP TABLE t1; +# +# 7. Whole documents beside them, which have nothing to report. +# +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1, "b":9}') AS v; +v +1 +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS v; +v +1 +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1, "q":2}}') AS v; +v +0 +SELECT JSON_OVERLAPS('[9]', '[1,2,3,4]') AS v; +v +0 +# +# 8. A document that is refused before the comparison begins is +# not this case, and is reported from position 1 as it always +# was. +# +SELECT JSON_OVERLAPS('{', '[1]') AS v; +v +0 +SELECT JSON_OVERLAPS('[1]', 'x') AS v; +v +0 +Warnings: +Warning 4038 Syntax error in JSON text in argument 2 to function 'json_overlaps' at position 1 diff --git a/mysql-test/main/func_json_overlaps_diag.test b/mysql-test/main/func_json_overlaps_diag.test new file mode 100644 index 0000000000000..e7523c467ec81 --- /dev/null +++ b/mysql-test/main/func_json_overlaps_diag.test @@ -0,0 +1,92 @@ +--echo # +--echo # Where JSON_OVERLAPS says a document broke. +--echo # +--echo # The comparison walks both arguments together and puts an engine +--echo # back where it was to try the next candidate. A refusal met +--echo # part-way through has to survive being put back, and the place it +--echo # is reported from has to be where the document broke, not +--echo # wherever the walk happened to stop afterwards. +--echo # +--echo # There is one right attest to each of these documents and it can +--echo # be had elsewhere: JSON_VALID and JSON_EXTRACT read the same bytes +--echo # with no comparison around them. Every case below is recorded +--echo # beside those two, so the position is pinned to something other +--echo # than itself. +--echo # + +--echo # +--echo # 1. What the same bytes are worth to a reader with nothing else +--echo # going on. These are the positions every case below must give. +--echo # + +SELECT JSON_VALID('{"a":1 "b":2}') AS ok; +--error 0 +SELECT JSON_EXTRACT('{"a":1 "b":2}', '$') AS v; +SELECT JSON_EXTRACT('{"a":1, "b":2 "c":3}', '$') AS v; +SELECT JSON_EXTRACT('{"a":{"p":1 "q":2}}', '$') AS v; +SELECT JSON_EXTRACT('[{"z":9},{"b":2 "c":3}]', '$') AS v; +SELECT JSON_EXTRACT('[1,2,3 4]', '$') AS v; + +--echo # +--echo # 2. The break is in argument 2, met after the walk has turned at +--echo # least once. +--echo # + +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1 "b":2}') AS v; +SELECT JSON_OVERLAPS('{"z":0}', '{"a":1, "b":2 "c":3}') AS v; + +--echo # +--echo # 3. The break is below the top level, so the walk is further in +--echo # when it meets it. +--echo # + +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1 "q":2}}') AS v; + +--echo # +--echo # 4. The break is in argument 1, and inside an element of it. +--echo # + +SELECT JSON_OVERLAPS('[{"z":9},{"b":2 "c":3}]', '[{"a":1}]') AS v; + +--echo # +--echo # 5. A refusal that is reached only after every candidate has been +--echo # tried and put back. Nothing said so before: the walk ended on +--echo # the refusal exactly as it ends on running out of candidates, +--echo # and the answer came back with no diagnostic at all. +--echo # + +SELECT JSON_OVERLAPS('[9]', '[1,2,3 4]') AS v; +SELECT JSON_OVERLAPS('[{"a":1}]', '[{"z":9},{"b":2 "c":3}]') AS v; + +--echo # +--echo # 6. The same documents out of a table, so no one statement +--echo # settles another, and both argument orders together. +--echo # + +CREATE TABLE t1 (id INT, a VARCHAR(64), b VARCHAR(64)); +INSERT INTO t1 VALUES + (1, '{"b":9}', '{"a":1 "b":2}'), + (2, '{"a":{"p":1}}', '{"a":{"p":1 "q":2}}'), + (3, '[9]', '[1,2,3 4]'), + (4, '{"b":9}', '{"a":1, "b":2}'); +SELECT id, JSON_OVERLAPS(a, b) AS v FROM t1 ORDER BY id; +SELECT id, JSON_OVERLAPS(b, a) AS v FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 7. Whole documents beside them, which have nothing to report. +--echo # + +SELECT JSON_OVERLAPS('{"b":9}', '{"a":1, "b":9}') AS v; +SELECT JSON_OVERLAPS('[1,2]', '[2,3]') AS v; +SELECT JSON_OVERLAPS('{"a":{"p":1}}', '{"a":{"p":1, "q":2}}') AS v; +SELECT JSON_OVERLAPS('[9]', '[1,2,3,4]') AS v; + +--echo # +--echo # 8. A document that is refused before the comparison begins is +--echo # not this case, and is reported from position 1 as it always +--echo # was. +--echo # + +SELECT JSON_OVERLAPS('{', '[1]') AS v; +SELECT JSON_OVERLAPS('[1]', 'x') AS v; diff --git a/mysql-test/main/func_json_pass_through_scan_count.result b/mysql-test/main/func_json_pass_through_scan_count.result new file mode 100644 index 0000000000000..0e8d54406e67a --- /dev/null +++ b/mysql-test/main/func_json_pass_through_scan_count.result @@ -0,0 +1,307 @@ +# +# How many times a document is read to find out that it is one when +# it reaches a JSON function through an expression that returns a +# value one of its arguments made: CASE and its abbreviations, and a +# scalar subquery. +# +# No query can see the difference - the reading could only have found +# out what was already known - so nothing but Json_scans says whether +# it happened. The counts below are the record of it. A count that +# goes up is a reading that came back. +# +# A debug build reads values back to check what was claimed about +# them, and none of those readings are counted: they are the debug +# build's work and not the server's. A count of 0 below therefore +# means nothing read, not that a reading went uncounted. +# +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +# +# 1. TWO rows through a function that attests to what it returns, +# spliced with nothing in between. JSON_EXTRACT reads its input +# once a row; the splice reads nothing. Every count below is held +# against this one. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 2. The same value returned by a searched CASE. What the CASE +# returns is what the THEN argument returned, so what that argument +# said about it is said about it here. +# +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN JSON_EXTRACT(j, '$') +ELSE JSON_ARRAY() END) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 3. A simple CASE, which picks its argument by comparing rather +# than by testing, and the ELSE rather than a THEN. +# +FLUSH STATUS; +SELECT JSON_ARRAY(CASE id WHEN 0 THEN JSON_ARRAY() +ELSE JSON_EXTRACT(j, '$') END) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 4. COALESCE and IFNULL, which find their argument by reading it. +# +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 5. IFNULL again, this time returning its second argument. +# +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(NULL, '$'), JSON_EXTRACT(j, '$'))) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 6. IF and NVL2, which are told which argument to return by a +# switch. The switch is not what either returns and is not asked. +# +FLUSH STATUS; +SELECT JSON_ARRAY(IF(id > 0, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(NVL2(id, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 7. NULLIF, which compares its first argument and then returns it. +# The function that makes the document therefore runs more than once +# a row and each run reads its own input, which is why this count is +# the highest here. Those readings belong to the argument; the one +# the splice was making is the one that goes. +# +FLUSH STATUS; +SELECT JSON_ARRAY(NULLIF(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r +FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 8. A scalar subquery, whose value is kept in a cache and read out +# of it. What the cache keeps is a copy of what the item under it +# returned, taken while the two were about the same characters. +# +# The subquery reads a table, so that it is still a subquery by the +# time it is run: one over no table at all is replaced by the +# expression inside it and there is nothing left to pass anything on. +# A correlated one is also run behind an expression cache, which is +# another value passed on and has to say the same thing. +# +# First with nothing spliced, which is what the subquery costs on +# its own, and then with the splice on top of it. +# +FLUSH STATUS; +SELECT (SELECT JSON_EXTRACT(x.j, '$') FROM t1 x +WHERE x.id = t1.id) AS r FROM t1 ORDER BY id; +r +{"a": 1} +{"b": 2} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_EXTRACT(x.j, '$') FROM t1 x +WHERE x.id = t1.id)) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# And with a producer that reads nothing of its own, so that the +# only reading left to count would be the splice's. +# +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_OBJECT('a', y.id) FROM t1 y +WHERE y.id = t1.id)) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 9. Nested one inside another, so that the value is passed on twice +# before it is spliced. +# +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(IF(id > 0, JSON_EXTRACT(j, '$'), NULL), +JSON_ARRAY())) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 10. A value nothing attests to, passed through the same way. The +# column says nothing about itself, so the splice reads it - and +# passing it through changes neither the answer nor the count. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +r +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN j ELSE JSON_ARRAY() END) AS r +FROM t1 ORDER BY id; +r +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 11. Through a temporary table, which is written before any row +# exists and so cannot be told what a row turned out to be. What +# can be said in advance is what every argument that can be returned +# says, and both of these do. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) +ELSE JSON_ARRAY() END AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# The same for a subquery, whose one argument is the select list of +# the query inside it. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, (SELECT JSON_OBJECT('a', y.id) FROM t1 y +WHERE y.id = t1.id) AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 12. The same with one argument that says nothing. One is enough +# to leave the column with nothing said about it, and every row is +# read on the way out - including the rows the other argument made. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) +ELSE j END AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 13. A GROUP BY, which writes a temporary table of its own, with +# the same expression in the select list. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, COALESCE(JSON_OBJECT('a', id), JSON_ARRAY()) AS x +FROM t1 GROUP BY id) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 14. The WHEN expressions are not among the arguments that can be +# returned, so what they are made of settles nothing. A CASE whose +# conditions read a column still attests for its THEN and ELSE. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, CASE WHEN j LIKE '%a%' THEN JSON_OBJECT('a', id) +ELSE JSON_OBJECT('b', id) END AS x FROM t1) d +ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +DROP TABLE t1; +SET optimizer_switch=DEFAULT; +# +# 15. DECODE, which picks its argument the way a simple CASE does. +# +SET sql_mode=ORACLE; +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +FLUSH STATUS; +SELECT JSON_ARRAY(DECODE(id, 0, JSON_ARRAY(), JSON_EXTRACT(j, '$'))) AS r +FROM t2 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t2; +SET sql_mode=DEFAULT; diff --git a/mysql-test/main/func_json_pass_through_scan_count.test b/mysql-test/main/func_json_pass_through_scan_count.test new file mode 100644 index 0000000000000..b1534beb325e9 --- /dev/null +++ b/mysql-test/main/func_json_pass_through_scan_count.test @@ -0,0 +1,238 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a document is read to find out that it is one when +--echo # it reaches a JSON function through an expression that returns a +--echo # value one of its arguments made: CASE and its abbreviations, and a +--echo # scalar subquery. +--echo # +--echo # No query can see the difference - the reading could only have found +--echo # out what was already known - so nothing but Json_scans says whether +--echo # it happened. The counts below are the record of it. A count that +--echo # goes up is a reading that came back. +--echo # +--echo # A debug build reads values back to check what was claimed about +--echo # them, and none of those readings are counted: they are the debug +--echo # build's work and not the server's. A count of 0 below therefore +--echo # means nothing read, not that a reading went uncounted. +--echo # + +SET NAMES utf8mb4; + +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol +SET optimizer_switch='derived_merge=off'; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +--echo # +--echo # 1. TWO rows through a function that attests to what it returns, +--echo # spliced with nothing in between. JSON_EXTRACT reads its input +--echo # once a row; the splice reads nothing. Every count below is held +--echo # against this one. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 2. The same value returned by a searched CASE. What the CASE +--echo # returns is what the THEN argument returned, so what that argument +--echo # said about it is said about it here. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN JSON_EXTRACT(j, '$') + ELSE JSON_ARRAY() END) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 3. A simple CASE, which picks its argument by comparing rather +--echo # than by testing, and the ELSE rather than a THEN. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(CASE id WHEN 0 THEN JSON_ARRAY() + ELSE JSON_EXTRACT(j, '$') END) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 4. COALESCE and IFNULL, which find their argument by reading it. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 5. IFNULL again, this time returning its second argument. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(IFNULL(JSON_EXTRACT(NULL, '$'), JSON_EXTRACT(j, '$'))) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 6. IF and NVL2, which are told which argument to return by a +--echo # switch. The switch is not what either returns and is not asked. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(IF(id > 0, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY(NVL2(id, JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 7. NULLIF, which compares its first argument and then returns it. +--echo # The function that makes the document therefore runs more than once +--echo # a row and each run reads its own input, which is why this count is +--echo # the highest here. Those readings belong to the argument; the one +--echo # the splice was making is the one that goes. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(NULLIF(JSON_EXTRACT(j, '$'), JSON_ARRAY())) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 8. A scalar subquery, whose value is kept in a cache and read out +--echo # of it. What the cache keeps is a copy of what the item under it +--echo # returned, taken while the two were about the same characters. +--echo # +--echo # The subquery reads a table, so that it is still a subquery by the +--echo # time it is run: one over no table at all is replaced by the +--echo # expression inside it and there is nothing left to pass anything on. +--echo # A correlated one is also run behind an expression cache, which is +--echo # another value passed on and has to say the same thing. +--echo # +--echo # First with nothing spliced, which is what the subquery costs on +--echo # its own, and then with the splice on top of it. +--echo # +FLUSH STATUS; +SELECT (SELECT JSON_EXTRACT(x.j, '$') FROM t1 x + WHERE x.id = t1.id) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_EXTRACT(x.j, '$') FROM t1 x + WHERE x.id = t1.id)) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And with a producer that reads nothing of its own, so that the +--echo # only reading left to count would be the splice's. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY((SELECT JSON_OBJECT('a', y.id) FROM t1 y + WHERE y.id = t1.id)) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 9. Nested one inside another, so that the value is passed on twice +--echo # before it is spliced. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(COALESCE(IF(id > 0, JSON_EXTRACT(j, '$'), NULL), + JSON_ARRAY())) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 10. A value nothing attests to, passed through the same way. The +--echo # column says nothing about itself, so the splice reads it - and +--echo # passing it through changes neither the answer nor the count. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +SELECT JSON_ARRAY(CASE WHEN id > 0 THEN j ELSE JSON_ARRAY() END) AS r + FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 11. Through a temporary table, which is written before any row +--echo # exists and so cannot be told what a row turned out to be. What +--echo # can be said in advance is what every argument that can be returned +--echo # says, and both of these do. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) + ELSE JSON_ARRAY() END AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same for a subquery, whose one argument is the select list of +--echo # the query inside it. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, (SELECT JSON_OBJECT('a', y.id) FROM t1 y + WHERE y.id = t1.id) AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 12. The same with one argument that says nothing. One is enough +--echo # to leave the column with nothing said about it, and every row is +--echo # read on the way out - including the rows the other argument made. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, CASE WHEN id > 0 THEN JSON_OBJECT('a', id) + ELSE j END AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 13. A GROUP BY, which writes a temporary table of its own, with +--echo # the same expression in the select list. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, COALESCE(JSON_OBJECT('a', id), JSON_ARRAY()) AS x + FROM t1 GROUP BY id) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 14. The WHEN expressions are not among the arguments that can be +--echo # returned, so what they are made of settles nothing. A CASE whose +--echo # conditions read a column still attests for its THEN and ELSE. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, CASE WHEN j LIKE '%a%' THEN JSON_OBJECT('a', id) + ELSE JSON_OBJECT('b', id) END AS x FROM t1) d + ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t1; +SET optimizer_switch=DEFAULT; +--enable_ps2_protocol + +--echo # +--echo # 15. DECODE, which picks its argument the way a simple CASE does. +--echo # +--disable_ps2_protocol +SET sql_mode=ORACLE; +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +FLUSH STATUS; +SELECT JSON_ARRAY(DECODE(id, 0, JSON_ARRAY(), JSON_EXTRACT(j, '$'))) AS r + FROM t2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t2; +SET sql_mode=DEFAULT; +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_pathkey.result b/mysql-test/main/func_json_pathkey.result new file mode 100644 index 0000000000000..c707bca01ce05 --- /dev/null +++ b/mysql-test/main/func_json_pathkey.result @@ -0,0 +1,202 @@ +# +# The last step of a path, when the path and the document are written +# in different character sets. +# +# A path is parsed in its own character set. Every step of it is +# therefore a sequence of characters, and comparing a step against a +# key of the document is a comparison of characters, not of bytes. +# These tests cover the last step of the path, which is the one the +# mutating functions handle themselves instead of leaving it to the +# path walker. The interior steps are covered alongside as the +# contrast case. +# +# The keys below are non ASCII characters, written as the bytes that +# write them so that this file needs no character set of its own: +# C3 A9 is U+00E9, one byte in latin1 (E9) +# E2 82 AC is U+20AC, one byte in this server's latin1 (80) +# E6 BC A2 is U+6F22, which latin1 has no room for +# +SET NAMES utf8mb4; +# +# 1. Replacing a key that is already in the document. The key is the +# same character on both sides, encoded in two bytes in the path +# and with one in the document. +# +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"e":1}' USING latin1), +_utf8mb4'$."e"', 2)) AS ascii_key; +ascii_key +7B2265223A20327D +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS wide_key; +wide_key +7B22E9223A20327D +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS set_existing; +set_existing +7B22E9223A20327D +SELECT JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS still_valid; +still_valid +1 +# +# 2. The same document and path in one character set, as the control. +# +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, +_utf8mb4 0x242E22C3A922, 2)) AS same_cs; +same_cs +7B22C3A9223A20327D +SELECT CHARSET(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, +_utf8mb4 0x242E22C3A922, 2)) AS same_cs_charset; +same_cs_charset +utf8mb4 +# +# 3. An interior step, which the path walker handles. This one has +# always compared characters. +# +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A7B2262223A317D7D USING latin1), +_utf8mb4 0x242E22C3A9222E62, 2)) AS interior_step; +interior_step +7B22E9223A207B2262223A20327D7D +# +# 4. Adding a key that is not in the document. The bytes of the key +# have to arrive in the document's character set. +# +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS insert_wide_key; +insert_wide_key +7B2261223A20312C2022E9223A20327D +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22C3A922, 2)) AS set_new_key; +set_new_key +7B2261223A20312C2022E9223A20327D +# read the key back with a path in the document's own character set +SELECT JSON_EXTRACT(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22C3A922, 2), +CONVERT(_utf8mb4 0x242E22C3A922 USING latin1)) AS read_back; +read_back +2 +# +# 5. The other direction: a document in the wider character set and a +# path in the narrower one. +# +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, +CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) +AS narrow_path; +narrow_path +7B22C3A9223A20327D +SELECT HEX(JSON_INSERT(_utf8mb4'{"a":1}', +CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) +AS narrow_insert; +narrow_insert +7B2261223A20312C2022C3A9223A20327D +# +# 6. Removing a key. JSON_REMOVE only has to match, never to write. +# +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B22C3A9223A312C2261223A327D USING latin1), +_utf8mb4 0x242E22C3A922)) AS remove_wide; +remove_wide +7B2261223A20327D +SELECT HEX(JSON_REMOVE(_utf8mb4 0x7B22C3A9223A312C2261223A327D, +_utf8mb4 0x242E22C3A922)) AS remove_same_cs; +remove_same_cs +7B2261223A20327D +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B2261223A7B22C3A9223A317D2C2262223A327D USING latin1), +_utf8mb4 0x242E612E22C3A922)) AS remove_interior; +remove_interior +7B2261223A207B7D2C202262223A20327D +# +# 7. A key the document's character set has no room for. U+20AC is +# in this server's latin1, U+6F22 is not. +# +# A key that will not convert is spliced in as it arrived, which is +# what was done with every key before any of them was converted, and +# a note says so. Whether the document that results can be read +# back is then decided by the document's character set, on the same +# terms as before. latin1 has a character at every byte, so the +# document always comes back. ucs2 reads its bytes two at a time, +# so what comes back depends on how many there are: an even number +# reads as characters and the document is kept, an odd one leaves a +# byte over and the document is refused, which is where a key that +# would not convert was refused before. +# +# U+6F22 is in ucs2 and so is converted like any other key. +# U+1F600 is not in ucs2, and takes four bytes; 'a' before it makes +# five. +# +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E282AC22, 2)) AS euro_into_latin1; +euro_into_latin1 +7B2261223A20312C202280223A20327D +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable; +unrepresentable +7B2261223A20312C2022E6BCA2223A20327D +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_set; +unrepresentable_set +7B2261223A20312C2022E6BCA2223A20327D +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_set' at position 0 +# a ucs2 document, which has room for this key and so takes it +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), +_utf8mb4 0x242E22E6BCA222, 2)) AS representable_ucs2; +representable_ucs2 +007B002200610022003A00200031002C002000226F220022003A00200032007D +# a key ucs2 has no room for, whose bytes go in as they are. Four +# of them read back as two characters, so the document is kept +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), +_utf8mb4 0x242E22F09F988022, 2)) AS unrepresentable_ucs2; +unrepresentable_ucs2 +007B002200610022003A00200031002C00200022F09F98800022003A00200032007D +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +# and five of them do not, so it is refused on the way back out +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), +_utf8mb4 0x242E2261F09F988022, 2)) AS odd_length_ucs2; +odd_length_ucs2 +NULL +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +Warning 4035 Broken JSON string in argument 1 to function 'json_insert' at position 30 +# and inside a statement that stops on a warning +CREATE TABLE t1 (id INT, doc LONGTEXT CHARACTER SET latin1); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t1 VALUES +(1, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)), +(2, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4'$.b', 2)); +Warnings: +Note 4035 Broken JSON string in argument 2 to function 'json_insert' at position 0 +SELECT id, HEX(doc) AS stored FROM t1 ORDER BY id; +id stored +1 7B2261223A20312C2022E6BCA2223A20327D +2 7B2261223A20312C202262223A20327D +SET SESSION sql_mode=DEFAULT; +DROP TABLE t1; +# matching an absent key needs no conversion, so it still just misses +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_replace; +unrepresentable_replace +7B2261223A20317D +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4'{"a":1}' USING latin1), +_utf8mb4 0x242E22E6BCA222)) AS unrepresentable_remove; +unrepresentable_remove +7B2261223A20317D +# +# 8. A key holding the character that ends a key in the document. +# This is not valid to splice, and stays a failure. +# +SELECT JSON_INSERT('{}', '$.a"b', 1) AS quote_in_key; +quote_in_key +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 +SELECT JSON_INSERT(CONVERT(_utf8mb4'{}' USING latin1), +_utf8mb4'$.a"b', 1) AS quote_in_key_cross_cs; +quote_in_key_cross_cs +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_insert' at position 4 diff --git a/mysql-test/main/func_json_pathkey.test b/mysql-test/main/func_json_pathkey.test new file mode 100644 index 0000000000000..7f3a99196d235 --- /dev/null +++ b/mysql-test/main/func_json_pathkey.test @@ -0,0 +1,153 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # The last step of a path, when the path and the document are written +--echo # in different character sets. +--echo # +--echo # A path is parsed in its own character set. Every step of it is +--echo # therefore a sequence of characters, and comparing a step against a +--echo # key of the document is a comparison of characters, not of bytes. +--echo # These tests cover the last step of the path, which is the one the +--echo # mutating functions handle themselves instead of leaving it to the +--echo # path walker. The interior steps are covered alongside as the +--echo # contrast case. +--echo # +--echo # The keys below are non ASCII characters, written as the bytes that +--echo # write them so that this file needs no character set of its own: +--echo # C3 A9 is U+00E9, one byte in latin1 (E9) +--echo # E2 82 AC is U+20AC, one byte in this server's latin1 (80) +--echo # E6 BC A2 is U+6F22, which latin1 has no room for +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. Replacing a key that is already in the document. The key is the +--echo # same character on both sides, encoded in two bytes in the path +--echo # and with one in the document. +--echo # + +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"e":1}' USING latin1), + _utf8mb4'$."e"', 2)) AS ascii_key; +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS wide_key; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS set_existing; +SELECT JSON_VALID(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A317D USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS still_valid; + +--echo # +--echo # 2. The same document and path in one character set, as the control. +--echo # + +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, + _utf8mb4 0x242E22C3A922, 2)) AS same_cs; +SELECT CHARSET(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, + _utf8mb4 0x242E22C3A922, 2)) AS same_cs_charset; + +--echo # +--echo # 3. An interior step, which the path walker handles. This one has +--echo # always compared characters. +--echo # + +SELECT HEX(JSON_SET(CONVERT(_utf8mb4 0x7B22C3A9223A7B2262223A317D7D USING latin1), + _utf8mb4 0x242E22C3A9222E62, 2)) AS interior_step; + +--echo # +--echo # 4. Adding a key that is not in the document. The bytes of the key +--echo # have to arrive in the document's character set. +--echo # + +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS insert_wide_key; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22C3A922, 2)) AS set_new_key; +--echo # read the key back with a path in the document's own character set +SELECT JSON_EXTRACT(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22C3A922, 2), + CONVERT(_utf8mb4 0x242E22C3A922 USING latin1)) AS read_back; + +--echo # +--echo # 5. The other direction: a document in the wider character set and a +--echo # path in the narrower one. +--echo # + +SELECT HEX(JSON_REPLACE(_utf8mb4 0x7B22C3A9223A317D, + CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) + AS narrow_path; +SELECT HEX(JSON_INSERT(_utf8mb4'{"a":1}', + CONVERT(_utf8mb4 0x242E22C3A922 USING latin1), 2)) + AS narrow_insert; + +--echo # +--echo # 6. Removing a key. JSON_REMOVE only has to match, never to write. +--echo # + +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B22C3A9223A312C2261223A327D USING latin1), + _utf8mb4 0x242E22C3A922)) AS remove_wide; +SELECT HEX(JSON_REMOVE(_utf8mb4 0x7B22C3A9223A312C2261223A327D, + _utf8mb4 0x242E22C3A922)) AS remove_same_cs; +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4 0x7B2261223A7B22C3A9223A317D2C2262223A327D USING latin1), + _utf8mb4 0x242E612E22C3A922)) AS remove_interior; + +--echo # +--echo # 7. A key the document's character set has no room for. U+20AC is +--echo # in this server's latin1, U+6F22 is not. +--echo # +--echo # A key that will not convert is spliced in as it arrived, which is +--echo # what was done with every key before any of them was converted, and +--echo # a note says so. Whether the document that results can be read +--echo # back is then decided by the document's character set, on the same +--echo # terms as before. latin1 has a character at every byte, so the +--echo # document always comes back. ucs2 reads its bytes two at a time, +--echo # so what comes back depends on how many there are: an even number +--echo # reads as characters and the document is kept, an odd one leaves a +--echo # byte over and the document is refused, which is where a key that +--echo # would not convert was refused before. +--echo # +--echo # U+6F22 is in ucs2 and so is converted like any other key. +--echo # U+1F600 is not in ucs2, and takes four bytes; 'a' before it makes +--echo # five. +--echo # + +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E282AC22, 2)) AS euro_into_latin1; +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable; +SELECT HEX(JSON_SET(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_set; +--echo # a ucs2 document, which has room for this key and so takes it +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), + _utf8mb4 0x242E22E6BCA222, 2)) AS representable_ucs2; +--echo # a key ucs2 has no room for, whose bytes go in as they are. Four +--echo # of them read back as two characters, so the document is kept +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), + _utf8mb4 0x242E22F09F988022, 2)) AS unrepresentable_ucs2; +--echo # and five of them do not, so it is refused on the way back out +SELECT HEX(JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING ucs2), + _utf8mb4 0x242E2261F09F988022, 2)) AS odd_length_ucs2; +--echo # and inside a statement that stops on a warning +CREATE TABLE t1 (id INT, doc LONGTEXT CHARACTER SET latin1); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t1 VALUES + (1, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)), + (2, JSON_INSERT(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4'$.b', 2)); +SELECT id, HEX(doc) AS stored FROM t1 ORDER BY id; +SET SESSION sql_mode=DEFAULT; +DROP TABLE t1; +--echo # matching an absent key needs no conversion, so it still just misses +SELECT HEX(JSON_REPLACE(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222, 2)) AS unrepresentable_replace; +SELECT HEX(JSON_REMOVE(CONVERT(_utf8mb4'{"a":1}' USING latin1), + _utf8mb4 0x242E22E6BCA222)) AS unrepresentable_remove; + +--echo # +--echo # 8. A key holding the character that ends a key in the document. +--echo # This is not valid to splice, and stays a failure. +--echo # + +SELECT JSON_INSERT('{}', '$.a"b', 1) AS quote_in_key; +SELECT JSON_INSERT(CONVERT(_utf8mb4'{}' USING latin1), + _utf8mb4'$.a"b', 1) AS quote_in_key_cross_cs; diff --git a/mysql-test/main/func_json_punct_charset.result b/mysql-test/main/func_json_punct_charset.result new file mode 100644 index 0000000000000..e02a89c4bd8b5 --- /dev/null +++ b/mysql-test/main/func_json_punct_charset.result @@ -0,0 +1,98 @@ +# A scalar converts into swe7 and a container does not. +SELECT JSON_VALID(CONVERT('1' USING swe7)) AS scalar_reads; +scalar_reads +1 +SELECT HEX(CONVERT('[1,2]' USING swe7)) AS container_bytes; +container_bytes +3F312C323F +Warnings: +Warning 1977 Cannot convert 'latin1' character 0x5B to 'swe7' +# +# 1. Picking values out, which writes brackets when several of +# them can match and nothing at all when only one can +# +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$', '$')) AS two_paths; +two_paths +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 1 +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$[*]')) AS wildcard; +wildcard +NULL +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$')) AS one_path; +one_path +31 +# +# 2. A document in that set attested is_valid, so the +# functions below are asked their question with it and answer +# yes to every part of it but the formatting +# +SELECT HEX(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS is_valid; +is_valid +31 +SELECT JSON_VALID(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS still_doc; +still_doc +1 +# +# 3. The six that edit a document, each given that is_valid +# one. The two that have to write a container to answer +# return NULL; the other four return a scalar - the one they +# were given, except JSON_MERGE_PATCH, which returns the patch. +# +SELECT HEX(JSON_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$.b', 3)) AS ins; +ins +31 +SELECT HEX(JSON_REMOVE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$.b')) AS rem; +rem +31 +SELECT HEX(JSON_ARRAY_APPEND(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$', 3)) AS app; +app +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array_append' at position 1 +SELECT HEX(JSON_ARRAY_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +'$[0]', 3)) AS ari; +ari +31 +SELECT HEX(JSON_MERGE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) AS mrg; +mrg +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_merge_preserve' at position 1 +SELECT HEX(JSON_MERGE_PATCH(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), +JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) +AS mpt; +mpt +32 +# +# 4. The ones that have always asked before writing punctuation, +# the ones that write none, and the two that write their answer +# in utf8mb4 whatever they were given +# +SELECT HEX(JSON_SEARCH(CONVERT('1' USING swe7), 'all', CONVERT('1' USING swe7))) +AS srch; +srch +222422 +SELECT HEX(JSON_ARRAY(CONVERT('1' USING swe7), CONVERT('2' USING swe7))) AS arr; +arr +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +SELECT HEX(JSON_OBJECT(CONVERT('a' USING swe7), CONVERT('1' USING swe7))) AS obj; +obj +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_object' at position 1 +SELECT HEX(JSON_QUERY(CONVERT('1' USING swe7), '$')) AS qry; +qry +NULL +SELECT HEX(JSON_NORMALIZE(CONVERT('1' USING swe7))) AS nrm; +nrm +312E304530 +SELECT HEX(JSON_COMPACT(CONVERT('1' USING swe7))) AS cmp; +cmp +31 diff --git a/mysql-test/main/func_json_punct_charset.test b/mysql-test/main/func_json_punct_charset.test new file mode 100644 index 0000000000000..5fdae3c24aca2 --- /dev/null +++ b/mysql-test/main/func_json_punct_charset.test @@ -0,0 +1,79 @@ +# +# What the JSON functions do in a character set that cannot encode the +# punctuation a document is written with. +# +# swe7 puts national letters at the code points the brackets and braces +# live at, so a container written there is a word rather than a +# container. A document can still be written in such a set, but only a +# scalar one - writing a container would take the brackets it does not +# have. So a function handed a document in such a set was handed a +# scalar, and a function that has to write a container to give its +# answer cannot give one there at all. +# +# The answers below are the ones the server has always given. They are +# here because the functions that used to find this out by reading the +# whole of their result back no longer always do, and what they hand +# back must not change for that. +# + +# A view column and a cursor's temporary table are both as wide as the +# expression says it is, and one of the answers below is wider than that, +# so materialising it would cut it. +--disable_cursor_protocol +--disable_view_protocol + +--echo # A scalar converts into swe7 and a container does not. +SELECT JSON_VALID(CONVERT('1' USING swe7)) AS scalar_reads; +SELECT HEX(CONVERT('[1,2]' USING swe7)) AS container_bytes; + +--echo # +--echo # 1. Picking values out, which writes brackets when several of +--echo # them can match and nothing at all when only one can +--echo # +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$', '$')) AS two_paths; +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$[*]')) AS wildcard; +SELECT HEX(JSON_EXTRACT(CONVERT('1' USING swe7), '$')) AS one_path; + +--echo # +--echo # 2. A document in that set attested is_valid, so the +--echo # functions below are asked their question with it and answer +--echo # yes to every part of it but the formatting +--echo # +SELECT HEX(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS is_valid; +SELECT JSON_VALID(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2)) AS still_doc; + +--echo # +--echo # 3. The six that edit a document, each given that is_valid +--echo # one. The two that have to write a container to answer +--echo # return NULL; the other four return a scalar - the one they +--echo # were given, except JSON_MERGE_PATCH, which returns the patch. +--echo # +SELECT HEX(JSON_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$.b', 3)) AS ins; +SELECT HEX(JSON_REMOVE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$.b')) AS rem; +SELECT HEX(JSON_ARRAY_APPEND(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$', 3)) AS app; +SELECT HEX(JSON_ARRAY_INSERT(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + '$[0]', 3)) AS ari; +SELECT HEX(JSON_MERGE(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) AS mrg; +SELECT HEX(JSON_MERGE_PATCH(JSON_INSERT(CONVERT('1' USING swe7), '$.a', 2), + JSON_INSERT(CONVERT('2' USING swe7), '$.a', 2))) + AS mpt; + +--echo # +--echo # 4. The ones that have always asked before writing punctuation, +--echo # the ones that write none, and the two that write their answer +--echo # in utf8mb4 whatever they were given +--echo # +SELECT HEX(JSON_SEARCH(CONVERT('1' USING swe7), 'all', CONVERT('1' USING swe7))) + AS srch; +SELECT HEX(JSON_ARRAY(CONVERT('1' USING swe7), CONVERT('2' USING swe7))) AS arr; +SELECT HEX(JSON_OBJECT(CONVERT('a' USING swe7), CONVERT('1' USING swe7))) AS obj; +SELECT HEX(JSON_QUERY(CONVERT('1' USING swe7), '$')) AS qry; +SELECT HEX(JSON_NORMALIZE(CONVERT('1' USING swe7))) AS nrm; +SELECT HEX(JSON_COMPACT(CONVERT('1' USING swe7))) AS cmp; + +--enable_view_protocol +--enable_cursor_protocol diff --git a/mysql-test/main/func_json_result_side.result b/mysql-test/main/func_json_result_side.result new file mode 100644 index 0000000000000..4f63109d243de --- /dev/null +++ b/mysql-test/main/func_json_result_side.result @@ -0,0 +1,171 @@ +# +# What a reference passes when a document is asked of it. +# +# A name written in HAVING or in ORDER BY that repeats a select +# list entry does not hold a value of its own. It is a reference, +# and it reads the entry's RESULT side - which, once a temporary +# table has been built for the grouping or the sorting, is a +# column of that table rather than the thing that made the value. +# +# The two sides can disagree, which is the whole reason they are +# asked separately: a store into the column can shorten a value or +# write it in another character set, so what the column holds may +# be a document where the producer's answer was about something +# else, or may be nothing of the sort where the producer's was. +# Every question a JSON function asks of an argument - is it a +# document, is it written the loose way, how deep does it go - is +# therefore put to the same side the value itself came from. +# +# A user variable assigned in the select list is the case where +# the answer must be no. Reading it back performs the assignment, +# so the value cannot be asked about without being made again, and +# nothing keeps marks for a user variable in any event. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'), (2, '{"a":2,"b":[3,4]}'); +CREATE VIEW v1 AS SELECT id AS vid, j AS vj FROM t1; +# +# 1. A column read through a reference +# +# One argument is passed as a view of what is already +# there and several are each given a copy, so both formats +# of the read are put through a reference here. +# +SELECT j AS a FROM t1 ORDER BY JSON_COMPACT(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 ORDER BY JSON_LOOSE(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a ORDER BY JSON_EXTRACT(a, '$.a'); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_EXTRACT(a, '$.a') > 1; +a +{"a":2,"b":[3,4]} +# +# 2. The same value put inside another document +# +# Splicing asks how the value is formatted as well as whether it +# is a document, and it is the reference that is asked. +# +SELECT j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_INSERT('{}','$.x',a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a +ORDER BY JSON_ARRAY_APPEND('{"c":[1]}', '$.c', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +# +# 3. A view's column read through a reference +# +# A view column is itself a reference, and it hands the +# question on rather than answering it. +# +SELECT vj AS a FROM v1 ORDER BY JSON_COMPACT(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT vj AS a FROM v1 ORDER BY JSON_INSERT('{}', '$.x', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT vj AS a FROM v1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT JSON_TYPE(vj) AS ty FROM v1 GROUP BY vj; +ty +OBJECT +OBJECT +# +# 4. A user variable read through a reference +# +SELECT @v := j AS a FROM t1 ORDER BY JSON_COMPACT(a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT @w := j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT @u := j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +# +# 5. Which of them is attested, shown by the answer itself +# +# A value attested is put inside the array it is joining; +# one that is not is written out as a string and quoted. No +# query shape prints what a reference passed, so the two +# outcomes are told apart by whether the row is there. +# +# The column is attested, so its rows come back under +# OBJECT and none comes back under STRING. +# +SELECT j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; +a +# The user variable is not, so the rows change sides. +SELECT @u := j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +a +SELECT @u := j AS a FROM t1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +# A view column is attested, the view passing the question +# on to the column underneath it. +SELECT vj AS a FROM v1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +a +{"a":1,"b":[1,2]} +{"a":2,"b":[3,4]} +SELECT vj AS a FROM v1 GROUP BY a +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; +a +# +# 6. The reading that is not done +# +# Json_scans is the only place the saved reading shows. The +# query below reads a document back out of a temporary table +# and puts it inside a new one; with the column's answer taken +# away the same query reads that document again. +# +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; +DROP VIEW v1; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_result_side.test b/mysql-test/main/func_json_result_side.test new file mode 100644 index 0000000000000..c03e02b60fe78 --- /dev/null +++ b/mysql-test/main/func_json_result_side.test @@ -0,0 +1,135 @@ +--source include/have_debug.inc + +--echo # +--echo # What a reference passes when a document is asked of it. +--echo # +--echo # A name written in HAVING or in ORDER BY that repeats a select +--echo # list entry does not hold a value of its own. It is a reference, +--echo # and it reads the entry's RESULT side - which, once a temporary +--echo # table has been built for the grouping or the sorting, is a +--echo # column of that table rather than the thing that made the value. +--echo # +--echo # The two sides can disagree, which is the whole reason they are +--echo # asked separately: a store into the column can shorten a value or +--echo # write it in another character set, so what the column holds may +--echo # be a document where the producer's answer was about something +--echo # else, or may be nothing of the sort where the producer's was. +--echo # Every question a JSON function asks of an argument - is it a +--echo # document, is it written the loose way, how deep does it go - is +--echo # therefore put to the same side the value itself came from. +--echo # +--echo # A user variable assigned in the select list is the case where +--echo # the answer must be no. Reading it back performs the assignment, +--echo # so the value cannot be asked about without being made again, and +--echo # nothing keeps marks for a user variable in any event. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1,"b":[1,2]}'), (2, '{"a":2,"b":[3,4]}'); + +CREATE VIEW v1 AS SELECT id AS vid, j AS vj FROM t1; + +--echo # +--echo # 1. A column read through a reference +--echo # +--echo # One argument is passed as a view of what is already +--echo # there and several are each given a copy, so both formats +--echo # of the read are put through a reference here. +--echo # +SELECT j AS a FROM t1 ORDER BY JSON_COMPACT(a); +SELECT j AS a FROM t1 ORDER BY JSON_LOOSE(a); +SELECT j AS a FROM t1 GROUP BY a ORDER BY JSON_EXTRACT(a, '$.a'); +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_EXTRACT(a, '$.a') > 1; + +--echo # +--echo # 2. The same value put inside another document +--echo # +--echo # Splicing asks how the value is formatted as well as whether it +--echo # is a document, and it is the reference that is asked. +--echo # +SELECT j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +SELECT j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_INSERT('{}','$.x',a)); +SELECT j AS a FROM t1 GROUP BY a + ORDER BY JSON_ARRAY_APPEND('{"c":[1]}', '$.c', a); + +--echo # +--echo # 3. A view's column read through a reference +--echo # +--echo # A view column is itself a reference, and it hands the +--echo # question on rather than answering it. +--echo # +SELECT vj AS a FROM v1 ORDER BY JSON_COMPACT(a); +SELECT vj AS a FROM v1 ORDER BY JSON_INSERT('{}', '$.x', a); +SELECT vj AS a FROM v1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); +SELECT JSON_TYPE(vj) AS ty FROM v1 GROUP BY vj; + +--echo # +--echo # 4. A user variable read through a reference +--echo # +SELECT @v := j AS a FROM t1 ORDER BY JSON_COMPACT(a); +SELECT @w := j AS a FROM t1 ORDER BY JSON_INSERT('{}', '$.x', a); +SELECT @u := j AS a FROM t1 GROUP BY a HAVING JSON_VALID(JSON_COMPACT(a)); + +--echo # +--echo # 5. Which of them is attested, shown by the answer itself +--echo # +--echo # A value attested is put inside the array it is joining; +--echo # one that is not is written out as a string and quoted. No +--echo # query shape prints what a reference passed, so the two +--echo # outcomes are told apart by whether the row is there. +--echo # +--echo # The column is attested, so its rows come back under +--echo # OBJECT and none comes back under STRING. +--echo # +SELECT j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +SELECT j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; + +--echo # The user variable is not, so the rows change sides. +SELECT @u := j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +SELECT @u := j AS a FROM t1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; + +--echo # A view column is attested, the view passing the question +--echo # on to the column underneath it. +SELECT vj AS a FROM v1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'OBJECT'; +SELECT vj AS a FROM v1 GROUP BY a + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(a), '$[0]')) = 'STRING'; + +--echo # +--echo # 6. The reading that is not done +--echo # +--echo # Json_scans is the only place the saved reading shows. The +--echo # query below reads a document back out of a temporary table +--echo # and puts it inside a new one; with the column's answer taken +--echo # away the same query reads that document again. +--echo # +# What is counted below is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(a) FROM (SELECT JSON_SET(j,'$.c',id) AS a FROM t1) d GROUP BY a; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; +--enable_ps2_protocol + +DROP VIEW v1; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_scan_count.result b/mysql-test/main/func_json_scan_count.result new file mode 100644 index 0000000000000..8d1b1a8bd8cc8 --- /dev/null +++ b/mysql-test/main/func_json_scan_count.result @@ -0,0 +1,586 @@ +CREATE TABLE t1 (id INT, js VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); +CREATE TABLE t3 (id INT, js VARCHAR(200)); +INSERT INTO t3 VALUES (1, CONCAT('{"a":"', REPEAT('x', 80), '"}')), +(2, CONCAT('{"a":"', REPEAT('y', 80), '"}')); +# +# A function that edits a document reads it once to find the place +# to edit, and then reads its own result back to find out how to +# write it. Two rows, two readings each. +# +FLUSH STATUS; +SELECT JSON_INSERT(js, '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_SET(js, '$.a', 9) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_REPLACE(js, '$.a', 9) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_REMOVE(js, '$.a') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_ARRAY_APPEND(js, '$.a', 7) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_ARRAY_INSERT(js, '$.a[0]', 7) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# Merging reads both documents, and then reads the result. +# +FLUSH STATUS; +SELECT JSON_MERGE(js, '{"c":3}') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_MERGE_PATCH(js, '{"c":3}') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# A function that only reads pays for ONE reading, not two. The +# walk to the path and the writing out of what was found are the +# same reading carried on, so nothing is read twice; that is what +# this work did to reading, and the count below is where it says +# so. +# +FLUSH STATUS; +SELECT JSON_EXTRACT(js, '$.a') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A path that matches nothing costs the same one reading, there +# being nothing to write out and nothing to read back either way. +# It is here to say that finding something is what has stopped +# costing extra, not that finding nothing ever did. +# +FLUSH STATUS; +SELECT JSON_EXTRACT(js, '$.zz') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# Chained edits pay only for the innermost link. That one is +# given a document out of a column that attests is_valid false, +# so it reads its result back before returning it. Every link +# after it is handed a document answering is_valid, and +# attests to its own result the same way. +# +FLUSH STATUS; +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_REMOVE(JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4), '$.a') +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# The same for each of the others, each handed the is_valid +# result of the one before it. +# +FLUSH STATUS; +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b[0]', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# The two above put their value at a key the document already +# has, so nothing is put there and what they append to is the +# scalar that was there all along, which has to be wrapped in an +# array first. At a key the document does NOT have, the value +# goes in and the array they append to is that value. +# +FLUSH STATUS; +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c[0]', 3) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +FLUSH STATUS; +SELECT JSON_MERGE_PATCH(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# A merge whose other argument attests is_valid false reads the +# answer back, every document that goes into it having a say in +# how it is written. A written-out literal is such a document: +# nothing has read it, so nothing can say how it is written. +# +FLUSH STATUS; +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +FLUSH STATUS; +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), '{"d":4}') FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +# +# A document answering is_valid but NOT is_nice is read back like +# one answering is_valid false. +# Picking a piece out of a document returns the piece as it was +# formatted where it came from, so that is such a document; picking +# values out and putting them together writes them out afresh, so +# that one is not. The same edit is put to each below, and the +# difference between the two counts is the reading back. +# +FLUSH STATUS; +SELECT JSON_INSERT(JSON_QUERY(js, '$'), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# Asking for a document in a particular formatting reads nothing at +# all when what asked for it wanted a document rather than text. +# Both of these therefore cost what the edit alone costs, and +# neither the formatting asked for nor the reading it would have +# taken happens. +# +FLUSH STATUS; +SELECT JSON_INSERT(JSON_COMPACT(js), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_INSERT(JSON_LOOSE(js), '$.c', 3) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# A value spliced into a document being built is read too, to find +# out whether it is a document at all. A column that is not typed +# as JSON is written out as a string instead and is not read. +# +FLUSH STATUS; +SELECT JSON_ARRAY(js, js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# A column that IS typed as JSON is read, the type being a promise +# nothing on the way here has checked. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j, j) FROM t2; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# A value that something has already read and attested to is not +# read again. Two values, and neither of them costs a reading, +# so what is left is what the two of them cost to make. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$.a'), JSON_EXTRACT(js, '$.b')) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_OBJECT('x', JSON_EXTRACT(js, '$.a'), 'y', JSON_EXTRACT(js, '$.b')) +FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_INSERT(js, '$.c', 3), JSON_INSERT(js, '$.d', 4)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# The same where the value goes into a document being EDITED +# rather than built. +# +FLUSH STATUS; +SELECT JSON_INSERT(js, '$.c', JSON_EXTRACT(js, '$.a')) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +FLUSH STATUS; +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', +JSON_INSERT(js, '$.e', 5)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +# +# What is skipped there is a reading that would have measured how +# deep the value goes, and how deep it ends up decides whether the +# result can be read at all. Two things answer that without a +# reading, and the smaller of them is taken. How long the value +# is answers it for a short one, which cannot nest deeper than half +# its length, and says nothing about a long one. What the function +# that wrote the value counted as it wrote answers it whatever the +# length - which is what the documents below are, at eighty-odd +# characters and one structure deep. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$')) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_OBJECT('k', JSON_EXTRACT(js, '$')) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# And it carries, which is what the length could never do: every +# step out used to read the whole of the long value again. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3)) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# A name that only asks for a formatting hands the value straight +# on, and passes on what was counted about it with it. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_COMPACT(JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A value nobody counted is read, as it always was. The column +# here is a caller's own text: it is cut into by JSON_QUERY, which +# can say the piece is a document but not how deep the document it +# came out of went. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_QUERY(js, '$')) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# A group writes its pairs as they arrive and is as deep as the +# deepest of them, so the object it makes is spliced without being +# read. Its sister gathers its elements through a table, and the +# column of that table keeps the deepest of the rows put into it, +# so they are not read on the way back out either. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_OBJECTAGG(id, JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# The elements a group gathers through a table are not read on the +# way back out either, whatever composed them. A constructor +# attests to every value it will ever make, so the column it +# fills says so once for all its rows, and the reading that used +# to happen per element does not happen at all - which is the one +# count here that a released server never paid, it having read +# nothing anywhere and returned whatever the bytes were. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js)) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_ARRAY(id, js)) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# The same through the tree, which gathers the elements first and +# writes them out when the group is asked for. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js) ORDER BY id) FROM t3; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# And a group gathering a column of a caller's own, which nothing +# has attested to, reads every element as it always did. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(j) FROM t2; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A name given to a value in the select list and used again later +# in the same statement reaches it through a reference, and a +# reference reads what the item's RESULT side passes. For +# nearly every item that is the value the item made, so what was +# said about the value is said about this - and the splice below +# costs the reading it costs when the expression is written out +# where it is used. +# +FLUSH STATUS; +SELECT JSON_OBJECT('a', js) AS o FROM t1 HAVING JSON_ARRAY(o) IS NOT NULL; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# A grouped query moves the aggregate out of the expression around +# it and puts a reference in its place, so the array below is +# built over a reference rather than over the aggregate. All +# three answers travel together through it, the depth among them: +# the array these gather is long enough that its length says more +# levels than a document is allowed, so a depth that went missing +# here would show up as the whole thing being read again. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3 GROUP BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A search returns one path as a string and several as an array +# of strings, so what it makes nests one level at the most and it +# knows which as it writes. The paths here are long, so the +# length says nothing useful and only that count keeps the answer +# from being read again where it is spliced. +# +CREATE TABLE t4 (id INT, js TEXT); +INSERT INTO t4 VALUES +(1, '{"customer_record":{"order_lines":[{"sku_code":"xa"},{"sku_code":"xb"}]}}'), +(2, '{"customer_record":{"order_lines":[{"sku_code":"xc"},{"sku_code":"xd"}]}}'); +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_SEARCH(js, 'all', 'x%')) FROM t4; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# One path found, which is a string and nests nothing at all +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_SEARCH(js, 'one', 'x%')) FROM t4; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t4; +# +# Keys are returned as an array of the names, written a name at +# a time out of the document they were read from. What that makes, +# how it is formatted and how deep it goes are all settled by the +# writing rather than measured afterwards, so a splice of the +# answer reads nothing. The names here are long, so the length +# says more levels than a document is allowed and the depth is the +# only thing keeping the answer from being read again. +# +# Four rows, the last two being the answers that are not arrays of +# names: an object with no keys at all, and a document that is not +# an object and returns nothing. +# +CREATE TABLE t5 (id INT, js TEXT, jn TEXT); +INSERT INTO t5 VALUES +(1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}'), +CONCAT('{"o":{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}}')), +(2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}'), +CONCAT('{"o":{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}}')), +(3, '{}', '{"o":{}}'), +(4, '[1,2]', '[3,4]'); +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(js)) FROM t5; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# The two argument form, which finds the object further in and then +# writes the same answer in the same way. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_KEYS(jn, '$.o')) FROM t5; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +DROP TABLE t5; +# +# Asking whether a value is a document reads it once. There is +# nothing here left to take away. +# +FLUSH STATUS; +SELECT JSON_VALID(js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# Nothing read, nothing counted. +# +FLUSH STATUS; +SELECT id FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# A function that only reports on a document parses it through to +# check that it is one, and that parse is the whole of what it +# costs on a large document. Out of a column nothing has attested to +# anything, so it is made: once to check the document and once to +# read the value being reported on. +# +FLUSH STATUS; +SELECT JSON_TYPE(js) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# The same over a document an item has attested is_valid. The +# editing costs what it always did, and the reporting costs one +# reading rather than two: there is nothing left to check. +# +FLUSH STATUS; +SELECT JSON_TYPE(JSON_INSERT(js, '$.c', 3)) FROM t1; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# The answers are unchanged either way. JSON_LENGTH, +# JSON_EXTRACT and JSON_CONTAINS_PATH stop parsing early over an +# attested value too, which no count here can show - they carry +# the parse on inside one reading rather than starting another. +# +SELECT JSON_TYPE(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_type, +JSON_TYPE('{"a":1,"b":2}') AS plain_type; +attested_type plain_type +OBJECT OBJECT +SELECT JSON_LENGTH(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_length, +JSON_LENGTH('{"a":1,"b":2}') AS plain_length; +attested_length plain_length +2 2 +SELECT JSON_EXTRACT(JSON_INSERT('{"a":1}', '$.b', 2), '$.a') AS attested_x, +JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS plain_x; +attested_x plain_x +1 1 +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), +'one', '$.a') AS attested_cp, +JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a') AS plain_cp; +attested_cp plain_cp +1 1 +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), +'all', '$.a', '$.b') AS attested_cp_all, +JSON_CONTAINS_PATH('{"a":1,"b":2}', 'all', '$.a', '$.b') AS plain_cp_all; +attested_cp_all plain_cp_all +1 1 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +# +# A document attested in one character set, spliced into a +# result being built in another. It is converted first, because the +# result is read in the character set it is built in - and a +# conversion that lost nothing kept the characters it was given, so +# what was answered about the value still holds of what came out of +# it. One reading for the value that was built, none for putting it +# in. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', +CONVERT(JSON_OBJECT('a', 1) USING latin1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# Two of them, one reading each and none for either splice. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', +CONVERT(JSON_OBJECT('a', 1) USING latin1), +'$.q', +CONVERT(JSON_OBJECT('b', 2) USING latin1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +# +# The same value with no conversion in the way, which is what the +# two above now cost. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', JSON_OBJECT('a', 1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# A conversion that could not encode a character put a question mark +# in its place, so the value that comes out is not the value that +# was attested. The bytes that arrived go in unchanged, as they +# always have, and they are read. +# +FLUSH STATUS; +SELECT JSON_SET('{"x":1}', '$.p', +CONVERT(JSON_OBJECT('a', _utf8mb4 0xE4BDA0) USING latin1)); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +# +# A conversion FROM the binary character set is not a conversion at +# all: there is nothing to convert from, so the bytes go across +# untouched and are read as whatever the other set makes of them. +# That is a different string of characters and, for a document, +# usually not one - so nothing carries and the value that comes out +# is read wherever it is used. +# +SELECT HEX(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary; +from_binary +5B226162225D +SELECT JSON_VALID(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary_valid; +from_binary_valid +0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 2 +# +# And the same going the other way, which was always refused +# +SELECT HEX(CONVERT(JSON_ARRAY('ab') USING binary)) AS to_binary; +to_binary +5B226162225D diff --git a/mysql-test/main/func_json_scan_count.test b/mysql-test/main/func_json_scan_count.test new file mode 100644 index 0000000000000..9d15cba6c5476 --- /dev/null +++ b/mysql-test/main/func_json_scan_count.test @@ -0,0 +1,662 @@ +--source include/have_debug.inc + +# +# How many times the JSON functions read a value. +# +# These functions return the same answer however many times they read +# what they are working on, so no result anywhere says how much reading +# was done. Json_scans says it, and nothing else does: a reading that +# stops happening shows up here or nowhere. +# +# The counts below are therefore not a statement about anything a caller +# can see. They are a record of work, kept so that work taken away +# later has to be taken away on purpose and in the open, and so that +# work quietly added back shows up as a test that fails. +# +# A debug build also reads values back to check what the functions claim +# about them. That reading is not counted - it is the build's work and +# not the server's, and counting it would move these numbers for reasons +# no query is responsible for. +# + +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol + +CREATE TABLE t1 (id INT, js VARCHAR(64)); +INSERT INTO t1 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); + +CREATE TABLE t2 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))); +INSERT INTO t2 VALUES (1, '{"a":1,"b":2}'), (2, '{"a":3,"b":4}'); + +CREATE TABLE t3 (id INT, js VARCHAR(200)); +INSERT INTO t3 VALUES (1, CONCAT('{"a":"', REPEAT('x', 80), '"}')), + (2, CONCAT('{"a":"', REPEAT('y', 80), '"}')); + +--echo # +--echo # A function that edits a document reads it once to find the place +--echo # to edit, and then reads its own result back to find out how to +--echo # write it. Two rows, two readings each. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(js, '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET(js, '$.a', 9) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_REPLACE(js, '$.a', 9) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_REMOVE(js, '$.a') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_APPEND(js, '$.a', 7) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_INSERT(js, '$.a[0]', 7) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Merging reads both documents, and then reads the result. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(js, '{"c":3}') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE_PATCH(js, '{"c":3}') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A function that only reads pays for ONE reading, not two. The +--echo # walk to the path and the writing out of what was found are the +--echo # same reading carried on, so nothing is read twice; that is what +--echo # this work did to reading, and the count below is where it says +--echo # so. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_EXTRACT(js, '$.a') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A path that matches nothing costs the same one reading, there +--echo # being nothing to write out and nothing to read back either way. +--echo # It is here to say that finding something is what has stopped +--echo # costing extra, not that finding nothing ever did. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_EXTRACT(js, '$.zz') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Chained edits pay only for the innermost link. That one is +--echo # given a document out of a column that attests is_valid false, +--echo # so it reads its result back before returning it. Every link +--echo # after it is handed a document answering is_valid, and +--echo # attests to its own result the same way. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_REMOVE(JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', 4), '$.a') + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same for each of the others, each handed the is_valid +--echo # result of the one before it. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.b', JSON_ARRAY(1, 2)), '$.b[0]', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The two above put their value at a key the document already +--echo # has, so nothing is put there and what they append to is the +--echo # scalar that was there all along, which has to be wrapped in an +--echo # array first. At a key the document does NOT have, the value +--echo # goes in and the array they append to is that value. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_APPEND(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY_INSERT(JSON_INSERT(js, '$.c', JSON_ARRAY(1, 2)), '$.c[0]', 3) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE_PATCH(JSON_INSERT(js, '$.c', 3), JSON_OBJECT('d', 4)) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A merge whose other argument attests is_valid false reads the +--echo # answer back, every document that goes into it having a say in +--echo # how it is written. A written-out literal is such a document: +--echo # nothing has read it, so nothing can say how it is written. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_MERGE(JSON_INSERT(js, '$.c', 3), '{"d":4}') FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A document answering is_valid but NOT is_nice is read back like +--echo # one answering is_valid false. +--echo # Picking a piece out of a document returns the piece as it was +--echo # formatted where it came from, so that is such a document; picking +--echo # values out and putting them together writes them out afresh, so +--echo # that one is not. The same edit is put to each below, and the +--echo # difference between the two counts is the reading back. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_QUERY(js, '$'), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Asking for a document in a particular formatting reads nothing at +--echo # all when what asked for it wanted a document rather than text. +--echo # Both of these therefore cost what the edit alone costs, and +--echo # neither the formatting asked for nor the reading it would have +--echo # taken happens. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_COMPACT(js), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_LOOSE(js), '$.c', 3) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A value spliced into a document being built is read too, to find +--echo # out whether it is a document at all. A column that is not typed +--echo # as JSON is written out as a string instead and is not read. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(js, js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A column that IS typed as JSON is read, the type being a promise +--echo # nothing on the way here has checked. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(j, j) FROM t2; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A value that something has already read and attested to is not +--echo # read again. Two values, and neither of them costs a reading, +--echo # so what is left is what the two of them cost to make. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$.a'), JSON_EXTRACT(js, '$.b')) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_OBJECT('x', JSON_EXTRACT(js, '$.a'), 'y', JSON_EXTRACT(js, '$.b')) + FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_INSERT(js, '$.c', 3), JSON_INSERT(js, '$.d', 4)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same where the value goes into a document being EDITED +--echo # rather than built. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(js, '$.c', JSON_EXTRACT(js, '$.a')) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_INSERT(JSON_INSERT(js, '$.c', 3), '$.d', + JSON_INSERT(js, '$.e', 5)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # What is skipped there is a reading that would have measured how +--echo # deep the value goes, and how deep it ends up decides whether the +--echo # result can be read at all. Two things answer that without a +--echo # reading, and the smaller of them is taken. How long the value +--echo # is answers it for a short one, which cannot nest deeper than half +--echo # its length, and says nothing about a long one. What the function +--echo # that wrote the value counted as it wrote answers it whatever the +--echo # length - which is what the documents below are, at eighty-odd +--echo # characters and one structure deep. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(js, '$')) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_OBJECT('k', JSON_EXTRACT(js, '$')) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And it carries, which is what the length could never do: every +--echo # step out used to read the whole of the long value again. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_INSERT(JSON_EXTRACT(js, '$'), '$.c', 3)) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A name that only asks for a formatting hands the value straight +--echo # on, and passes on what was counted about it with it. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_COMPACT(JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A value nobody counted is read, as it always was. The column +--echo # here is a caller's own text: it is cut into by JSON_QUERY, which +--echo # can say the piece is a document but not how deep the document it +--echo # came out of went. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_QUERY(js, '$')) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A group writes its pairs as they arrive and is as deep as the +--echo # deepest of them, so the object it makes is spliced without being +--echo # read. Its sister gathers its elements through a table, and the +--echo # column of that table keeps the deepest of the rows put into it, +--echo # so they are not read on the way back out either. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_OBJECTAGG(id, JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The elements a group gathers through a table are not read on the +--echo # way back out either, whatever composed them. A constructor +--echo # attests to every value it will ever make, so the column it +--echo # fills says so once for all its rows, and the reading that used +--echo # to happen per element does not happen at all - which is the one +--echo # count here that a released server never paid, it having read +--echo # nothing anywhere and returned whatever the bytes were. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js)) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(JSON_ARRAY(id, js)) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same through the tree, which gathers the elements first and +--echo # writes them out when the group is asked for. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id, 'b', js) ORDER BY id) FROM t3; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And a group gathering a column of a caller's own, which nothing +--echo # has attested to, reads every element as it always did. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAYAGG(j) FROM t2; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A name given to a value in the select list and used again later +--echo # in the same statement reaches it through a reference, and a +--echo # reference reads what the item's RESULT side passes. For +--echo # nearly every item that is the value the item made, so what was +--echo # said about the value is said about this - and the splice below +--echo # costs the reading it costs when the expression is written out +--echo # where it is used. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_OBJECT('a', js) AS o FROM t1 HAVING JSON_ARRAY(o) IS NOT NULL; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A grouped query moves the aggregate out of the expression around +--echo # it and puts a reference in its place, so the array below is +--echo # built over a reference rather than over the aggregate. All +--echo # three answers travel together through it, the depth among them: +--echo # the array these gather is long enough that its length says more +--echo # levels than a document is allowed, so a depth that went missing +--echo # here would show up as the whole thing being read again. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAYAGG(JSON_EXTRACT(js, '$'))) FROM t3 GROUP BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A search returns one path as a string and several as an array +--echo # of strings, so what it makes nests one level at the most and it +--echo # knows which as it writes. The paths here are long, so the +--echo # length says nothing useful and only that count keeps the answer +--echo # from being read again where it is spliced. +--echo # +CREATE TABLE t4 (id INT, js TEXT); +INSERT INTO t4 VALUES + (1, '{"customer_record":{"order_lines":[{"sku_code":"xa"},{"sku_code":"xb"}]}}'), + (2, '{"customer_record":{"order_lines":[{"sku_code":"xc"},{"sku_code":"xd"}]}}'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_SEARCH(js, 'all', 'x%')) FROM t4; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # One path found, which is a string and nests nothing at all +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_SEARCH(js, 'one', 'x%')) FROM t4; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t4; + +--echo # +--echo # Keys are returned as an array of the names, written a name at +--echo # a time out of the document they were read from. What that makes, +--echo # how it is formatted and how deep it goes are all settled by the +--echo # writing rather than measured afterwards, so a splice of the +--echo # answer reads nothing. The names here are long, so the length +--echo # says more levels than a document is allowed and the depth is the +--echo # only thing keeping the answer from being read again. +--echo # +--echo # Four rows, the last two being the answers that are not arrays of +--echo # names: an object with no keys at all, and a document that is not +--echo # an object and returns nothing. +--echo # +CREATE TABLE t5 (id INT, js TEXT, jn TEXT); +INSERT INTO t5 VALUES + (1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}'), + CONCAT('{"o":{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}}')), + (2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}'), + CONCAT('{"o":{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}}')), + (3, '{}', '{"o":{}}'), + (4, '[1,2]', '[3,4]'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(js)) FROM t5; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The two argument form, which finds the object further in and then +--echo # writes the same answer in the same way. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_KEYS(jn, '$.o')) FROM t5; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t5; + +--echo # +--echo # Asking whether a value is a document reads it once. There is +--echo # nothing here left to take away. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_VALID(js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Nothing read, nothing counted. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT id FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + + +--echo # +--echo # A function that only reports on a document parses it through to +--echo # check that it is one, and that parse is the whole of what it +--echo # costs on a large document. Out of a column nothing has attested to +--echo # anything, so it is made: once to check the document and once to +--echo # read the value being reported on. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_TYPE(js) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same over a document an item has attested is_valid. The +--echo # editing costs what it always did, and the reporting costs one +--echo # reading rather than two: there is nothing left to check. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_TYPE(JSON_INSERT(js, '$.c', 3)) FROM t1; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The answers are unchanged either way. JSON_LENGTH, +--echo # JSON_EXTRACT and JSON_CONTAINS_PATH stop parsing early over an +--echo # attested value too, which no count here can show - they carry +--echo # the parse on inside one reading rather than starting another. +--echo # +SELECT JSON_TYPE(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_type, + JSON_TYPE('{"a":1,"b":2}') AS plain_type; +SELECT JSON_LENGTH(JSON_INSERT('{"a":1}', '$.b', 2)) AS attested_length, + JSON_LENGTH('{"a":1,"b":2}') AS plain_length; +SELECT JSON_EXTRACT(JSON_INSERT('{"a":1}', '$.b', 2), '$.a') AS attested_x, + JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS plain_x; +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), + 'one', '$.a') AS attested_cp, + JSON_CONTAINS_PATH('{"a":1,"b":2}', 'one', '$.a') AS plain_cp; +SELECT JSON_CONTAINS_PATH(JSON_INSERT('{"a":1}', '$.b', 2), + 'all', '$.a', '$.b') AS attested_cp_all, + JSON_CONTAINS_PATH('{"a":1,"b":2}', 'all', '$.a', '$.b') AS plain_cp_all; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; + +--echo # +--echo # A document attested in one character set, spliced into a +--echo # result being built in another. It is converted first, because the +--echo # result is read in the character set it is built in - and a +--echo # conversion that lost nothing kept the characters it was given, so +--echo # what was answered about the value still holds of what came out of +--echo # it. One reading for the value that was built, none for putting it +--echo # in. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', + CONVERT(JSON_OBJECT('a', 1) USING latin1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Two of them, one reading each and none for either splice. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', + CONVERT(JSON_OBJECT('a', 1) USING latin1), + '$.q', + CONVERT(JSON_OBJECT('b', 2) USING latin1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # The same value with no conversion in the way, which is what the +--echo # two above now cost. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', JSON_OBJECT('a', 1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A conversion that could not encode a character put a question mark +--echo # in its place, so the value that comes out is not the value that +--echo # was attested. The bytes that arrived go in unchanged, as they +--echo # always have, and they are read. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_SET('{"x":1}', '$.p', + CONVERT(JSON_OBJECT('a', _utf8mb4 0xE4BDA0) USING latin1)); +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # A conversion FROM the binary character set is not a conversion at +--echo # all: there is nothing to convert from, so the bytes go across +--echo # untouched and are read as whatever the other set makes of them. +--echo # That is a different string of characters and, for a document, +--echo # usually not one - so nothing carries and the value that comes out +--echo # is read wherever it is used. +--echo # +SELECT HEX(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary; +SELECT JSON_VALID(CONVERT(JSON_ARRAY(_binary'ab') USING ucs2)) AS from_binary_valid; + +--echo # +--echo # And the same going the other way, which was always refused +--echo # +SELECT HEX(CONVERT(JSON_ARRAY('ab') USING binary)) AS to_binary; + +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_search_typed.result b/mysql-test/main/func_json_search_typed.result new file mode 100644 index 0000000000000..92a6e43ba67eb --- /dev/null +++ b/mysql-test/main/func_json_search_typed.result @@ -0,0 +1,110 @@ +# +# JSON_SEARCH always returns a document. +# +# Every path it produces is built out of pieces of a document it +# has just read through, and the punctuation round them is its +# own. Several paths go inside brackets, which needs a character +# set that can encode a bracket; a document written in one that +# cannot is a scalar, there being no way to write a container in +# it, so it holds one value and one path comes back. +# +# So the answer is a document whatever was asked, and a column +# filled from this function is a column of documents. What that +# buys is the reading that a later reader of the column would +# otherwise have to do to find that out. +# +SET NAMES utf8mb4; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":"x","b":"y"}'), (2, '{"c":"x","d":"z"}'); +# +# 1. The two shapes of answer +# +SELECT id, JSON_SEARCH(j, 'one', 'x') AS one_path, +JSON_SEARCH(j, 'all', 'x') AS all_paths FROM t1 ORDER BY id; +id one_path all_paths +1 "$.a" "$.a" +2 "$.c" "$.c" +SELECT id, JSON_TYPE(JSON_SEARCH(j, 'one', 'x')) AS one_ty, +JSON_TYPE(JSON_SEARCH(j, 'all', '%')) AS all_ty +FROM t1 ORDER BY id; +id one_ty all_ty +1 STRING ARRAY +2 STRING ARRAY +# +# 2. Branches of a union agreeing about it +# +# A union settles what its column holds from what every branch +# says it always returns, and both of these say a document. +# +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS p +UNION +SELECT JSON_SEARCH('{"c":"z"}', 'one', 'z'); +p +"$.a" +"$.c" +SELECT JSON_TYPE(p) AS ty FROM +(SELECT JSON_SEARCH('{"a":"x"}', 'all', '%') AS p +UNION ALL +SELECT JSON_SEARCH('{"b":"y","c":"y"}', 'all', '%')) u; +ty +STRING +ARRAY +# +# 3. A list of values written out in the statement +# +VALUES (JSON_SEARCH('{"a":"x"}', 'one', 'x')), +(JSON_SEARCH('{"b":"y"}', 'one', 'y')); +JSON_SEARCH('{"a":"x"}', 'one', 'x') +"$.a" +"$.b" +# +# 4. A column of a table the server built for itself +# +SELECT p FROM (SELECT JSON_SEARCH(j, 'all', '%') AS p FROM t1) d +GROUP BY p ORDER BY p; +p +["$.a", "$.b"] +["$.c", "$.d"] +CREATE TABLE t2 AS SELECT id, JSON_SEARCH(j, 'all', '%') AS p FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t2 ORDER BY id; +id p ty +1 ["$.a", "$.b"] ARRAY +2 ["$.c", "$.d"] ARRAY +DROP TABLE t2; +# +# 5. A column of a table the statement named +# +# A path written into a column that only takes documents is +# accepted, which is the same claim put the other way round. +# +CREATE TABLE t3 (id INT, p VARCHAR(64) CHECK (JSON_VALID(p))) CHARSET utf8mb4; +INSERT INTO t3 SELECT id, JSON_SEARCH(j, 'all', '%') FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t3 ORDER BY id; +id p ty +1 ["$.a", "$.b"] ARRAY +2 ["$.c", "$.d"] ARRAY +DROP TABLE t3; +# +# 6. The reading that is not done +# +# A path spliced into a new document out of a temporary table +# the server built is taken as it stands; the same query with +# the column's answer taken away reads it again. +# +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d +GROUP BY p; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d +GROUP BY p; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_search_typed.test b/mysql-test/main/func_json_search_typed.test new file mode 100644 index 0000000000000..95a2fb596f21f --- /dev/null +++ b/mysql-test/main/func_json_search_typed.test @@ -0,0 +1,105 @@ +--source include/have_debug.inc + +--echo # +--echo # JSON_SEARCH always returns a document. +--echo # +--echo # Every path it produces is built out of pieces of a document it +--echo # has just read through, and the punctuation round them is its +--echo # own. Several paths go inside brackets, which needs a character +--echo # set that can encode a bracket; a document written in one that +--echo # cannot is a scalar, there being no way to write a container in +--echo # it, so it holds one value and one path comes back. +--echo # +--echo # So the answer is a document whatever was asked, and a column +--echo # filled from this function is a column of documents. What that +--echo # buys is the reading that a later reader of the column would +--echo # otherwise have to do to find that out. +--echo # + +SET NAMES utf8mb4; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":"x","b":"y"}'), (2, '{"c":"x","d":"z"}'); + +--echo # +--echo # 1. The two shapes of answer +--echo # +SELECT id, JSON_SEARCH(j, 'one', 'x') AS one_path, + JSON_SEARCH(j, 'all', 'x') AS all_paths FROM t1 ORDER BY id; +SELECT id, JSON_TYPE(JSON_SEARCH(j, 'one', 'x')) AS one_ty, + JSON_TYPE(JSON_SEARCH(j, 'all', '%')) AS all_ty + FROM t1 ORDER BY id; + +--echo # +--echo # 2. Branches of a union agreeing about it +--echo # +--echo # A union settles what its column holds from what every branch +--echo # says it always returns, and both of these say a document. +--echo # +SELECT JSON_SEARCH('{"a":"x","b":"y"}', 'one', 'x') AS p +UNION +SELECT JSON_SEARCH('{"c":"z"}', 'one', 'z'); + +SELECT JSON_TYPE(p) AS ty FROM + (SELECT JSON_SEARCH('{"a":"x"}', 'all', '%') AS p + UNION ALL + SELECT JSON_SEARCH('{"b":"y","c":"y"}', 'all', '%')) u; + +--echo # +--echo # 3. A list of values written out in the statement +--echo # +VALUES (JSON_SEARCH('{"a":"x"}', 'one', 'x')), + (JSON_SEARCH('{"b":"y"}', 'one', 'y')); + +--echo # +--echo # 4. A column of a table the server built for itself +--echo # +SELECT p FROM (SELECT JSON_SEARCH(j, 'all', '%') AS p FROM t1) d + GROUP BY p ORDER BY p; + +CREATE TABLE t2 AS SELECT id, JSON_SEARCH(j, 'all', '%') AS p FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t2 ORDER BY id; +DROP TABLE t2; + +--echo # +--echo # 5. A column of a table the statement named +--echo # +--echo # A path written into a column that only takes documents is +--echo # accepted, which is the same claim put the other way round. +--echo # +CREATE TABLE t3 (id INT, p VARCHAR(64) CHECK (JSON_VALID(p))) CHARSET utf8mb4; +INSERT INTO t3 SELECT id, JSON_SEARCH(j, 'all', '%') FROM t1; +SELECT id, p, JSON_TYPE(p) AS ty FROM t3 ORDER BY id; +DROP TABLE t3; + +--echo # +--echo # 6. The reading that is not done +--echo # +--echo # A path spliced into a new document out of a temporary table +--echo # the server built is taken as it stands; the same query with +--echo # the column's answer taken away reads it again. +--echo # +# What is counted below is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol +SET optimizer_switch='derived_merge=off'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d + GROUP BY p; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +SET SESSION debug_dbug = '+d,json_tmp_store_kept_short'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(p) FROM (SELECT JSON_SEARCH(j,'all','%') AS p FROM t1) d + GROUP BY p; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug = DEFAULT; +SET optimizer_switch=DEFAULT; +--enable_ps2_protocol + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_search_wide_charset.result b/mysql-test/main/func_json_search_wide_charset.result new file mode 100644 index 0000000000000..0b49d0879785e --- /dev/null +++ b/mysql-test/main/func_json_search_wide_charset.result @@ -0,0 +1,116 @@ +SET NAMES utf8mb4; +# +# A match under an array index, so the path carries a number. +# The number goes in as one byte per digit whatever the set, and +# the bytes below show it standing among characters of two and of +# four. +# +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'all', +CONVERT('V' USING ucs2))) AS ucs2_all; +ucs2_all +005B00220024005B30005D005B30005D0022002C002000220024005B31005D005B30005D0022005D +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'one', +CONVERT('V' USING ucs2))) AS ucs2_one; +ucs2_one +00220024005B30005D005B30005D0022 +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf16), 'all', +CONVERT('V' USING utf16))) AS utf16_all; +utf16_all +005B00220024005B30005D005B30005D0022002C002000220024005B31005D005B30005D0022005D +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf32), 'all', +CONVERT('V' USING utf32))) AS utf32_all; +utf32_all +0000005B00000022000000240000005B300000005D0000005B300000005D000000220000002C0000002000000022000000240000005B310000005D0000005B300000005D000000220000005D +# +# Reading such a path back, and splicing it into something being +# built. Four bytes to the character is the case that shows it: +# the path does not read, and asking is what finds that out. +# +SET @u = CONVERT('[["V"],["V"]]' USING utf32); +SET @m = CONVERT('V' USING utf32); +SELECT JSON_VALID(JSON_SEARCH(@u, 'all', @m)) AS utf32_answer_reads; +utf32_answer_reads +0 +Warnings: +Note 4035 Broken JSON string in argument 1 to function 'json_valid' at position 16 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@u, 'all', @m))) AS utf32_spliced; +utf32_spliced +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 +SELECT JSON_TYPE(JSON_SEARCH(@u, 'all', @m)) AS utf32_type; +utf32_type +NULL +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_type' at position 16 +# +# The same at two bytes to the character. +# +SET @d = CONVERT('[["V"],["V"]]' USING ucs2); +SET @n = CONVERT('V' USING ucs2); +SELECT JSON_VALID(JSON_SEARCH(@d, 'all', @n)) AS ucs2_answer_reads; +ucs2_answer_reads +1 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@d, 'all', @n))) AS ucs2_spliced; +ucs2_spliced +1 +SELECT HEX(JSON_MERGE_PRESERVE(CONVERT('[0]' USING ucs2), +JSON_SEARCH(@d, 'all', @n))) AS ucs2_merged; +ucs2_merged +005B0030002C002000220024005B30005D005B30005D0022002C002000220024005B31005D005B30005D0022005D +# +# A path with no array step in it has no number in it and is +# encoded all the way through, so it reads and it splices. The +# two shapes are told apart by what was written into them. +# +SET @o = CONVERT('{"a":{"b":"V"}}' USING ucs2); +SELECT HEX(JSON_SEARCH(@o, 'one', @n)) AS keys_only; +keys_only +00220024002E0061002E00620022 +SELECT JSON_VALID(JSON_SEARCH(@o, 'one', @n)) AS keys_only_reads; +keys_only_reads +1 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@o, 'one', @n))) AS keys_only_spliced; +keys_only_spliced +1 +# +# A character set that does write a character in one byte is not +# this case at all, and none of it moves. +# +SELECT JSON_SEARCH('[["V"],["V"]]', 'all', 'V') AS utf8_all; +utf8_all +["$[0][0]", "$[1][0]"] +SELECT JSON_VALID(JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_reads; +utf8_reads +1 +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH('[["V"],["V"]]', 'all', 'V'))) AS utf8_spliced; +utf8_spliced +1 +SELECT JSON_MERGE_PRESERVE('[0]', JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_merged; +utf8_merged +[0, "$[0][0]", "$[1][0]"] +SELECT JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), 'all', +CONVERT('V' USING latin1)) AS latin1_all; +latin1_all +["$[0][0]", "$[1][0]"] +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), +'all', +CONVERT('V' USING latin1)))) AS latin1_spliced; +latin1_spliced +1 +# +# Out of a table, so no one statement settles another, and with +# both shapes of path in the same statement. +# +CREATE TABLE t (id INT, d BLOB, n BLOB); +INSERT INTO t VALUES (1, CONVERT('[["V"],["V"]]' USING utf32), CONVERT('V' USING utf32)), +(2, CONVERT('{"a":{"b":"V"}}' USING utf32), CONVERT('V' USING utf32)); +SELECT id, JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT(d USING utf32), 'all', +CONVERT(n USING utf32)))) AS spliced +FROM t ORDER BY id; +id spliced +1 NULL +2 1 +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_array' at position 16 +DROP TABLE t; diff --git a/mysql-test/main/func_json_search_wide_charset.test b/mysql-test/main/func_json_search_wide_charset.test new file mode 100644 index 0000000000000..3805cd98c2797 --- /dev/null +++ b/mysql-test/main/func_json_search_wide_charset.test @@ -0,0 +1,91 @@ +# +# The path JSON_SEARCH returns, in a character set that does not +# write a character in one byte. +# +# A path is built out of punctuation the function writes, keys it copies +# from the document, and the index of an array step. The punctuation and +# the keys arrive in the character set of the result. The index does +# not: it is written as the digits themselves, so in ucs2, utf16 or +# utf32 it is the one part of the path not formatted the way the rest of +# it is, and every character after it is read from the wrong place. +# +# What is pinned here is that nothing is claimed on that path's behalf. +# Whether such a path can be read at all is settled differently by +# different character sets, and by asking it is settled at all - what +# must not happen is the answer being taken on trust instead. +# + +SET NAMES utf8mb4; + +--echo # +--echo # A match under an array index, so the path carries a number. +--echo # The number goes in as one byte per digit whatever the set, and +--echo # the bytes below show it standing among characters of two and of +--echo # four. +--echo # +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'all', + CONVERT('V' USING ucs2))) AS ucs2_all; +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING ucs2), 'one', + CONVERT('V' USING ucs2))) AS ucs2_one; +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf16), 'all', + CONVERT('V' USING utf16))) AS utf16_all; +SELECT HEX(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING utf32), 'all', + CONVERT('V' USING utf32))) AS utf32_all; + +--echo # +--echo # Reading such a path back, and splicing it into something being +--echo # built. Four bytes to the character is the case that shows it: +--echo # the path does not read, and asking is what finds that out. +--echo # +SET @u = CONVERT('[["V"],["V"]]' USING utf32); +SET @m = CONVERT('V' USING utf32); +SELECT JSON_VALID(JSON_SEARCH(@u, 'all', @m)) AS utf32_answer_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@u, 'all', @m))) AS utf32_spliced; +SELECT JSON_TYPE(JSON_SEARCH(@u, 'all', @m)) AS utf32_type; + +--echo # +--echo # The same at two bytes to the character. +--echo # +SET @d = CONVERT('[["V"],["V"]]' USING ucs2); +SET @n = CONVERT('V' USING ucs2); +SELECT JSON_VALID(JSON_SEARCH(@d, 'all', @n)) AS ucs2_answer_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@d, 'all', @n))) AS ucs2_spliced; +SELECT HEX(JSON_MERGE_PRESERVE(CONVERT('[0]' USING ucs2), + JSON_SEARCH(@d, 'all', @n))) AS ucs2_merged; + +--echo # +--echo # A path with no array step in it has no number in it and is +--echo # encoded all the way through, so it reads and it splices. The +--echo # two shapes are told apart by what was written into them. +--echo # +SET @o = CONVERT('{"a":{"b":"V"}}' USING ucs2); +SELECT HEX(JSON_SEARCH(@o, 'one', @n)) AS keys_only; +SELECT JSON_VALID(JSON_SEARCH(@o, 'one', @n)) AS keys_only_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(@o, 'one', @n))) AS keys_only_spliced; + +--echo # +--echo # A character set that does write a character in one byte is not +--echo # this case at all, and none of it moves. +--echo # +SELECT JSON_SEARCH('[["V"],["V"]]', 'all', 'V') AS utf8_all; +SELECT JSON_VALID(JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_reads; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH('[["V"],["V"]]', 'all', 'V'))) AS utf8_spliced; +SELECT JSON_MERGE_PRESERVE('[0]', JSON_SEARCH('[["V"],["V"]]', 'all', 'V')) AS utf8_merged; + +SELECT JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), 'all', + CONVERT('V' USING latin1)) AS latin1_all; +SELECT JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT('[["V"],["V"]]' USING latin1), + 'all', + CONVERT('V' USING latin1)))) AS latin1_spliced; + +--echo # +--echo # Out of a table, so no one statement settles another, and with +--echo # both shapes of path in the same statement. +--echo # +CREATE TABLE t (id INT, d BLOB, n BLOB); +INSERT INTO t VALUES (1, CONVERT('[["V"],["V"]]' USING utf32), CONVERT('V' USING utf32)), + (2, CONVERT('{"a":{"b":"V"}}' USING utf32), CONVERT('V' USING utf32)); +SELECT id, JSON_VALID(JSON_ARRAY(JSON_SEARCH(CONVERT(d USING utf32), 'all', + CONVERT(n USING utf32)))) AS spliced + FROM t ORDER BY id; +DROP TABLE t; diff --git a/mysql-test/main/func_json_sp.result b/mysql-test/main/func_json_sp.result new file mode 100644 index 0000000000000..0b0e355c7882d --- /dev/null +++ b/mysql-test/main/func_json_sp.result @@ -0,0 +1,460 @@ +# +# Behavioral baseline: JSON values held in stored program variables. +# +# A variable declared JSON is not the same thing as a column declared +# JSON: it carries no check constraint and, as the first section +# shows, JSON functions do not treat it as a document. This test +# records that difference, what the declared type does to a value on +# assignment, and what happens when a variable is both the source and +# the target of the same statement. +# +SET NAMES utf8mb4; +# +# 1. A variable declared JSON compared with a column declared JSON. +# A JSON column is embedded into a constructor as a document; the +# variable is quoted like any other string. +# +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'); +CREATE PROCEDURE p_decl() +BEGIN +DECLARE v JSON DEFAULT '{"a":1}'; +DECLARE t LONGTEXT DEFAULT '{"a":1}'; +SELECT v AS var_json; +SELECT JSON_ARRAY(v) AS from_json_var; +SELECT JSON_ARRAY(t) AS from_text_var; +SELECT JSON_ARRAY(j) AS from_json_column FROM t1; +SELECT JSON_OBJECT('k', v) AS obj_from_json_var; +SELECT JSON_OBJECT('k', j) AS obj_from_json_column FROM t1; +SELECT JSON_VALID(JSON_ARRAY(v)) AS var_still_valid; +SELECT JSON_VALID(JSON_ARRAY(j)) AS col_still_valid FROM t1; +SELECT JSON_SET(v, '$.b', 2) AS set_over_var; +SELECT JSON_EXTRACT(v, '$.a') AS extract_from_var; +SELECT JSON_TYPE(v) AS type_of_var; +END $$ +CALL p_decl(); +var_json +{"a":1} +from_json_var +["{\"a\":1}"] +from_text_var +["{\"a\":1}"] +from_json_column +[{"a":1}] +obj_from_json_var +{"k": "{\"a\":1}"} +obj_from_json_column +{"k": {"a":1}} +var_still_valid +1 +col_still_valid +1 +set_over_var +{"a": 1, "b": 2} +extract_from_var +1 +type_of_var +OBJECT +DROP PROCEDURE p_decl; +# the declared type as the server records it +CREATE FUNCTION f_ret_json() RETURNS JSON +BEGIN +RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +SELECT f_ret_json() AS ret; +ret +{"a": 1, "b": 2} +SELECT JSON_ARRAY(f_ret_json()) AS embedded; +embedded +["{\"a\": 1, \"b\": 2}"] +SELECT JSON_VALID(JSON_ARRAY(f_ret_json())) AS still_valid; +still_valid +1 +SELECT JSON_TYPE(f_ret_json()) AS t; +t +OBJECT +DROP FUNCTION f_ret_json; +# a function declared to return LONGTEXT, for comparison +CREATE FUNCTION f_ret_text() RETURNS LONGTEXT +BEGIN +RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +SELECT f_ret_text() AS ret; +ret +{"a": 1, "b": 2} +SELECT JSON_ARRAY(f_ret_text()) AS embedded; +embedded +["{\"a\": 1, \"b\": 2}"] +DROP FUNCTION f_ret_text; +# +# 2. What the declared type does to a value on assignment. +# +# a value that fits, and the same value converted to other types +CREATE PROCEDURE p_types() +BEGIN +DECLARE longer VARCHAR(100); +DECLARE num INT; +DECLARE bin VARBINARY(100); +SET longer = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT longer AS intact, JSON_VALID(longer) AS still_valid; +SET num = JSON_EXTRACT('{"a":42}', '$.a'); +SELECT num AS as_number; +SET bin = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT HEX(bin) AS as_binary, JSON_VALID(bin) AS still_valid; +END $$ +CALL p_types(); +intact still_valid +{"a": 9, "b": 2} 1 +as_number +42 +as_binary still_valid +7B2261223A20392C202262223A20327D 1 +DROP PROCEDURE p_types; +# a value too long for the variable. A stored program runs under the +# sql_mode it was CREATED with, so setting the mode around the CALL +# decides nothing and each mode needs a procedure of its own. In +# strict mode the assignment is refused; in non-strict mode it +# truncates silently - no error and no warning - leaving a value that +# is no longer a document. +SET @@sql_mode='STRICT_ALL_TABLES'; +CREATE PROCEDURE p_cut_strict() +BEGIN +DECLARE v VARCHAR(8); +SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +SET @@sql_mode=''; +CREATE PROCEDURE p_cut_lax() +BEGIN +DECLARE v VARCHAR(8); +SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); +SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +SET @@sql_mode=DEFAULT; +# the one created in strict mode, called under each mode in turn +SET @@sql_mode=''; +CALL p_cut_strict(); +ERROR 22001: Data too long for column 'v' at row 0 +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_cut_strict(); +ERROR 22001: Data too long for column 'v' at row 0 +SET @@sql_mode=DEFAULT; +# the one created in non-strict mode, the same way +SET @@sql_mode=''; +CALL p_cut_lax(); +cut h still_valid +{"a": 9, 7B2261223A20392C 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_cut_lax(); +cut h still_valid +{"a": 9, 7B2261223A20392C 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_cut_strict; +DROP PROCEDURE p_cut_lax; +# the same value into a column of the same width, for contrast +CREATE TABLE t_narrow (v VARCHAR(8)); +SET @@sql_mode=''; +INSERT INTO t_narrow VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +Warnings: +Warning 1265 Data truncated for column 'v' at row 1 +SELECT v, HEX(v) AS h, JSON_VALID(v) AS still_valid FROM t_narrow; +v h still_valid +{"a": 9, 7B2261223A20392C 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SET @@sql_mode=DEFAULT; +DROP TABLE t_narrow; +# an assignment that loses characters rather than length: the value +# fits, but the variable's character set cannot carry it. +CREATE PROCEDURE p_lossy_latin1() +BEGIN +DECLARE v VARCHAR(100) CHARACTER SET latin1; +SET v = JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4)); +SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +CREATE PROCEDURE p_lossy_swe7() +BEGIN +DECLARE w VARCHAR(100) CHARACTER SET swe7; +SET w = JSON_SET('{"a":1}', '$.b', 2); +SELECT HEX(w) AS as_swe7, JSON_VALID(w) AS still_valid; +END $$ +CREATE PROCEDURE p_lossless_latin1() +BEGIN +DECLARE v VARCHAR(100) CHARACTER SET latin1; +SET v = JSON_SET('{"a":1}', '$.b', 2); +SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +SET @@sql_mode=''; +CALL p_lossy_latin1(); +ERROR 22007: Incorrect string value: '\xE4\xBD\xA0"}' for column ``.``.`v` at row 0 +CALL p_lossy_swe7(); +ERROR 22007: Incorrect string value: '{"a": ...' for column ``.``.`w` at row 0 +CALL p_lossless_latin1(); +as_latin1 still_valid +7B2261223A20312C202262223A20327D 1 +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_lossy_latin1(); +ERROR 22007: Incorrect string value: '\xE4\xBD\xA0"}' for column ``.``.`v` at row 0 +CALL p_lossy_swe7(); +ERROR 22007: Incorrect string value: '{"a": ...' for column ``.``.`w` at row 0 +CALL p_lossless_latin1(); +as_latin1 still_valid +7B2261223A20312C202262223A20327D 1 +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_lossy_latin1; +DROP PROCEDURE p_lossy_swe7; +DROP PROCEDURE p_lossless_latin1; +# the same lossy value into a column, which is where non-strict mode +# keeps the value instead of refusing it +CREATE TABLE t_lossy (v VARCHAR(100) CHARACTER SET latin1); +SET @@sql_mode=''; +INSERT INTO t_lossy +VALUES (JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4))); +Warnings: +Warning 1366 Incorrect string value: '\xE4\xBD\xA0"}' for column `test`.`t_lossy`.`v` at row 1 +SELECT HEX(v) AS stored, JSON_VALID(v) AS still_valid FROM t_lossy; +stored still_valid +7B2261223A20312C202262223A20223F227D 1 +SET @@sql_mode=DEFAULT; +DROP TABLE t_lossy; +# +# 3. A variable that is both the source and the target. +# +CREATE PROCEDURE p_self() +BEGIN +DECLARE v JSON DEFAULT '{"a":1}'; +SET v = JSON_SET(v, '$.b', 2); +SELECT v AS after_one; +SET v = JSON_SET(v, '$.c', 3); +SELECT v AS after_two; +SET v = JSON_INSERT(v, '$.d', 4); +SELECT v AS after_insert; +SET v = JSON_REMOVE(v, '$.a'); +SELECT v AS after_remove; +SET v = JSON_MERGE(v, '{"e":5}'); +SELECT v AS after_merge; +SET v = JSON_ARRAY(v); +SELECT v AS after_wrap; +SET v = JSON_EXTRACT(v, '$[0]'); +SELECT v AS after_extract; +SELECT JSON_VALID(v) AS still_valid; +END $$ +CALL p_self(); +after_one +{"a": 1, "b": 2} +after_two +{"a": 1, "b": 2, "c": 3} +after_insert +{"a": 1, "b": 2, "c": 3, "d": 4} +after_remove +{"b": 2, "c": 3, "d": 4} +after_merge +{"b": 2, "c": 3, "d": 4, "e": 5} +after_wrap +["{\"b\": 2, \"c\": 3, \"d\": 4, \"e\": 5}"] +after_extract +"{\"b\": 2, \"c\": 3, \"d\": 4, \"e\": 5}" +still_valid +1 +DROP PROCEDURE p_self; +# a long document repeatedly rewritten in place, so the variable's +# storage has to grow and be reused +CREATE PROCEDURE p_grow() +BEGIN +DECLARE v JSON DEFAULT '{"a":1}'; +DECLARE i INT DEFAULT 0; +WHILE i < 20 DO +SET v = JSON_SET(v, CONCAT('$.k', i), REPEAT('x', 50)); +SET i = i + 1; +END WHILE; +SELECT LENGTH(v) AS len, JSON_VALID(v) AS still_valid, JSON_LENGTH(v) AS members; +SELECT JSON_EXTRACT(v, '$.k0') AS first_added; +SELECT JSON_EXTRACT(v, '$.k19') AS last_added; +END $$ +CALL p_grow(); +len still_valid members +1218 1 21 +first_added +"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +last_added +"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +DROP PROCEDURE p_grow; +# +# 4. Variables passed in and out of procedures. +# +CREATE PROCEDURE p_out(IN a JSON, OUT b JSON, INOUT c JSON) +BEGIN +SET b = JSON_SET(a, '$.from_in', 1); +SET c = JSON_SET(c, '$.from_inout', 1); +SELECT JSON_ARRAY(a) AS in_embedded, JSON_ARRAY(c) AS inout_embedded; +END $$ +SET @in = '{"a":1}'; +SET @out = NULL; +SET @inout = '{"c":1}'; +CALL p_out(@in, @out, @inout); +in_embedded inout_embedded +["{\"a\":1}"] ["{\"c\": 1, \"from_inout\": 1}"] +SELECT @out AS out_value, @inout AS inout_value; +out_value inout_value +{"a": 1, "from_in": 1} {"c": 1, "from_inout": 1} +SELECT JSON_VALID(@out) AS out_valid, JSON_VALID(@inout) AS inout_valid; +out_valid inout_valid +1 1 +SELECT JSON_ARRAY(@out) AS user_var_embedded; +user_var_embedded +["{\"a\": 1, \"from_in\": 1}"] +SELECT JSON_VALID(JSON_ARRAY(@out)) AS still_valid; +still_valid +1 +DROP PROCEDURE p_out; +# a variable holding a document that is not valid +SET @bad = '{"a":1,'; +SELECT JSON_VALID(@bad) AS v; +v +0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT JSON_ARRAY(@bad) AS embedded; +embedded +["{\"a\":1,"] +SELECT JSON_VALID(JSON_ARRAY(@bad)) AS still_valid; +still_valid +1 +SELECT JSON_SET(@bad, '$.b', 1) AS mutated; +mutated +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +# +# 5. Variables inside a ROW, and a variable fed from a cursor. +# +CREATE PROCEDURE p_row() +BEGIN +DECLARE r ROW (a JSON, b LONGTEXT); +SET r.a = JSON_SET('{"a":1}', '$.b', 2); +SET r.b = JSON_SET('{"a":1}', '$.b', 2); +SELECT r.a AS row_json, r.b AS row_text; +SELECT JSON_ARRAY(r.a) AS from_row_json, JSON_ARRAY(r.b) AS from_row_text; +SELECT JSON_VALID(r.a) AS a_valid, JSON_VALID(r.b) AS b_valid; +SET r.a = JSON_SET(r.a, '$.c', 3); +SELECT r.a AS after_self_assign; +END $$ +CALL p_row(); +row_json row_text +{"a": 1, "b": 2} {"a": 1, "b": 2} +from_row_json from_row_text +[{"a": 1, "b": 2}] ["{\"a\": 1, \"b\": 2}"] +a_valid b_valid +1 1 +after_self_assign +{"a": 1, "b": 2, "c": 3} +DROP PROCEDURE p_row; +CREATE PROCEDURE p_cursor() +BEGIN +DECLARE done INT DEFAULT 0; +DECLARE v JSON; +DECLARE cur CURSOR FOR SELECT j FROM t1; +DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1; +OPEN cur; +FETCH cur INTO v; +CLOSE cur; +SELECT v AS fetched; +SELECT JSON_ARRAY(v) AS embedded; +SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; +SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +CALL p_cursor(); +fetched +{"a":1} +embedded +["{\"a\":1}"] +still_valid +1 +mutated +{"a": 1, "b": 2} +DROP PROCEDURE p_cursor; +# a cursor over a column that no longer satisfies its check +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +CREATE PROCEDURE p_cursor_bad() +BEGIN +DECLARE v JSON; +DECLARE cur CURSOR FOR SELECT j FROM t1 WHERE j = '{"a":1,'; +OPEN cur; +FETCH cur INTO v; +CLOSE cur; +SELECT v AS fetched, JSON_VALID(v) AS valid; +SELECT JSON_ARRAY(v) AS embedded; +SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +CALL p_cursor_bad(); +fetched valid +{"a":1, 0 +embedded +["{\"a\":1,"] +mutated +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +DROP PROCEDURE p_cursor_bad; +# +# 6. A variable read several times in one call, including through a +# stored function that reads it again while the outer call is still +# evaluating. No SQL-visible way to REASSIGN the variable from +# inside an argument has been identified; if one is found, the +# aliasing shape belongs here too. +# +CREATE FUNCTION f_side(x JSON) RETURNS INT +BEGIN +RETURN JSON_LENGTH(x); +END $$ +CREATE PROCEDURE p_multi() +BEGIN +DECLARE v JSON DEFAULT '{"a":1,"b":2}'; +SELECT JSON_SET(v, '$.c', f_side(v)) AS both_uses; +SELECT JSON_MERGE(v, v) AS twice; +SELECT JSON_ARRAY(v, v) AS twice_embedded; +SELECT JSON_SET(v, '$.x', 1, '$.y', JSON_EXTRACT(v, '$.a')) AS mixed; +END $$ +CALL p_multi(); +both_uses +{"a": 1, "b": 2, "c": 2} +twice +{"a": [1, 1], "b": [2, 2]} +twice_embedded +["{\"a\":1,\"b\":2}", "{\"a\":1,\"b\":2}"] +mixed +{"a": 1, "b": 2, "x": 1, "y": 1} +DROP PROCEDURE p_multi; +DROP FUNCTION f_side; +# +# 7. A trigger, where the row values are columns rather than variables. +# +CREATE TABLE t2 (j JSON, note LONGTEXT); +CREATE TRIGGER tr BEFORE INSERT ON t2 FOR EACH ROW +BEGIN +SET NEW.note = JSON_ARRAY(NEW.j); +END $$ +INSERT INTO t2 (j) VALUES ('{"a":1}'); +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2; +j note note_valid +{"a":1} [{"a":1}] 1 +# A row whose document does not read leaves the trigger with +# nothing to set the column to. Saying so is a warning, and an +# INSERT under strict mode makes an error of it, so the row does +# not go in at all - where a released server stored the bytes the +# constructor had spliced and let the column keep them. +SET SESSION check_constraint_checks = OFF; +INSERT INTO t2 (j) VALUES ('{"a":1,'); +ERROR HY000: Unexpected end of JSON text in argument 1 to function 'json_array' +SET SESSION check_constraint_checks = ON; +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2 ORDER BY j; +j note note_valid +{"a":1} [{"a":1}] 1 +DROP TABLE t2; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp.test b/mysql-test/main/func_json_sp.test new file mode 100644 index 0000000000000..980ce3c8a5d3c --- /dev/null +++ b/mysql-test/main/func_json_sp.test @@ -0,0 +1,377 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: JSON values held in stored program variables. +--echo # +--echo # A variable declared JSON is not the same thing as a column declared +--echo # JSON: it carries no check constraint and, as the first section +--echo # shows, JSON functions do not treat it as a document. This test +--echo # records that difference, what the declared type does to a value on +--echo # assignment, and what happens when a variable is both the source and +--echo # the target of the same statement. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A variable declared JSON compared with a column declared JSON. +--echo # A JSON column is embedded into a constructor as a document; the +--echo # variable is quoted like any other string. +--echo # + +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('{"a":1}'); + +--delimiter $$ +CREATE PROCEDURE p_decl() +BEGIN + DECLARE v JSON DEFAULT '{"a":1}'; + DECLARE t LONGTEXT DEFAULT '{"a":1}'; + SELECT v AS var_json; + SELECT JSON_ARRAY(v) AS from_json_var; + SELECT JSON_ARRAY(t) AS from_text_var; + SELECT JSON_ARRAY(j) AS from_json_column FROM t1; + SELECT JSON_OBJECT('k', v) AS obj_from_json_var; + SELECT JSON_OBJECT('k', j) AS obj_from_json_column FROM t1; + SELECT JSON_VALID(JSON_ARRAY(v)) AS var_still_valid; + SELECT JSON_VALID(JSON_ARRAY(j)) AS col_still_valid FROM t1; + SELECT JSON_SET(v, '$.b', 2) AS set_over_var; + SELECT JSON_EXTRACT(v, '$.a') AS extract_from_var; + SELECT JSON_TYPE(v) AS type_of_var; +END $$ +--delimiter ; +CALL p_decl(); +DROP PROCEDURE p_decl; + +--echo # the declared type as the server records it +--delimiter $$ +CREATE FUNCTION f_ret_json() RETURNS JSON +BEGIN + RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +--delimiter ; +SELECT f_ret_json() AS ret; +SELECT JSON_ARRAY(f_ret_json()) AS embedded; +SELECT JSON_VALID(JSON_ARRAY(f_ret_json())) AS still_valid; +SELECT JSON_TYPE(f_ret_json()) AS t; +DROP FUNCTION f_ret_json; + +--echo # a function declared to return LONGTEXT, for comparison +--delimiter $$ +CREATE FUNCTION f_ret_text() RETURNS LONGTEXT +BEGIN + RETURN JSON_SET('{"a":1}', '$.b', 2); +END $$ +--delimiter ; +SELECT f_ret_text() AS ret; +SELECT JSON_ARRAY(f_ret_text()) AS embedded; +DROP FUNCTION f_ret_text; + +--echo # +--echo # 2. What the declared type does to a value on assignment. +--echo # + +--echo # a value that fits, and the same value converted to other types +--delimiter $$ +CREATE PROCEDURE p_types() +BEGIN + DECLARE longer VARCHAR(100); + DECLARE num INT; + DECLARE bin VARBINARY(100); + SET longer = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT longer AS intact, JSON_VALID(longer) AS still_valid; + SET num = JSON_EXTRACT('{"a":42}', '$.a'); + SELECT num AS as_number; + SET bin = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT HEX(bin) AS as_binary, JSON_VALID(bin) AS still_valid; +END $$ +--delimiter ; +CALL p_types(); +DROP PROCEDURE p_types; + +--echo # a value too long for the variable. A stored program runs under the +--echo # sql_mode it was CREATED with, so setting the mode around the CALL +--echo # decides nothing and each mode needs a procedure of its own. In +--echo # strict mode the assignment is refused; in non-strict mode it +--echo # truncates silently - no error and no warning - leaving a value that +--echo # is no longer a document. +SET @@sql_mode='STRICT_ALL_TABLES'; +--delimiter $$ +CREATE PROCEDURE p_cut_strict() +BEGIN + DECLARE v VARCHAR(8); + SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +SET @@sql_mode=''; +--delimiter $$ +CREATE PROCEDURE p_cut_lax() +BEGIN + DECLARE v VARCHAR(8); + SET v = JSON_SET('{"a":1,"b":2}', '$.a', 9); + SELECT v AS cut, HEX(v) AS h, JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +SET @@sql_mode=DEFAULT; +--echo # the one created in strict mode, called under each mode in turn +SET @@sql_mode=''; +--error ER_DATA_TOO_LONG +CALL p_cut_strict(); +SET @@sql_mode='STRICT_ALL_TABLES'; +--error ER_DATA_TOO_LONG +CALL p_cut_strict(); +SET @@sql_mode=DEFAULT; +--echo # the one created in non-strict mode, the same way +SET @@sql_mode=''; +CALL p_cut_lax(); +SET @@sql_mode='STRICT_ALL_TABLES'; +CALL p_cut_lax(); +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_cut_strict; +DROP PROCEDURE p_cut_lax; +--echo # the same value into a column of the same width, for contrast +CREATE TABLE t_narrow (v VARCHAR(8)); +SET @@sql_mode=''; +INSERT INTO t_narrow VALUES (JSON_SET('{"a":1,"b":2}', '$.a', 9)); +SELECT v, HEX(v) AS h, JSON_VALID(v) AS still_valid FROM t_narrow; +SET @@sql_mode=DEFAULT; +DROP TABLE t_narrow; + +--echo # an assignment that loses characters rather than length: the value +--echo # fits, but the variable's character set cannot carry it. +--delimiter $$ +CREATE PROCEDURE p_lossy_latin1() +BEGIN + DECLARE v VARCHAR(100) CHARACTER SET latin1; + SET v = JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4)); + SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +CREATE PROCEDURE p_lossy_swe7() +BEGIN + DECLARE w VARCHAR(100) CHARACTER SET swe7; + SET w = JSON_SET('{"a":1}', '$.b', 2); + SELECT HEX(w) AS as_swe7, JSON_VALID(w) AS still_valid; +END $$ +CREATE PROCEDURE p_lossless_latin1() +BEGIN + DECLARE v VARCHAR(100) CHARACTER SET latin1; + SET v = JSON_SET('{"a":1}', '$.b', 2); + SELECT HEX(v) AS as_latin1, JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +SET @@sql_mode=''; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_latin1(); +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_swe7(); +CALL p_lossless_latin1(); +SET @@sql_mode='STRICT_ALL_TABLES'; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_latin1(); +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +CALL p_lossy_swe7(); +CALL p_lossless_latin1(); +SET @@sql_mode=DEFAULT; +DROP PROCEDURE p_lossy_latin1; +DROP PROCEDURE p_lossy_swe7; +DROP PROCEDURE p_lossless_latin1; +--echo # the same lossy value into a column, which is where non-strict mode +--echo # keeps the value instead of refusing it +CREATE TABLE t_lossy (v VARCHAR(100) CHARACTER SET latin1); +SET @@sql_mode=''; +INSERT INTO t_lossy + VALUES (JSON_SET('{"a":1}', '$.b', CONVERT(_utf8mb4 X'E4BDA0' USING utf8mb4))); +SELECT HEX(v) AS stored, JSON_VALID(v) AS still_valid FROM t_lossy; +SET @@sql_mode=DEFAULT; +DROP TABLE t_lossy; + +--echo # +--echo # 3. A variable that is both the source and the target. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_self() +BEGIN + DECLARE v JSON DEFAULT '{"a":1}'; + SET v = JSON_SET(v, '$.b', 2); + SELECT v AS after_one; + SET v = JSON_SET(v, '$.c', 3); + SELECT v AS after_two; + SET v = JSON_INSERT(v, '$.d', 4); + SELECT v AS after_insert; + SET v = JSON_REMOVE(v, '$.a'); + SELECT v AS after_remove; + SET v = JSON_MERGE(v, '{"e":5}'); + SELECT v AS after_merge; + SET v = JSON_ARRAY(v); + SELECT v AS after_wrap; + SET v = JSON_EXTRACT(v, '$[0]'); + SELECT v AS after_extract; + SELECT JSON_VALID(v) AS still_valid; +END $$ +--delimiter ; +CALL p_self(); +DROP PROCEDURE p_self; + +--echo # a long document repeatedly rewritten in place, so the variable's +--echo # storage has to grow and be reused +--delimiter $$ +CREATE PROCEDURE p_grow() +BEGIN + DECLARE v JSON DEFAULT '{"a":1}'; + DECLARE i INT DEFAULT 0; + WHILE i < 20 DO + SET v = JSON_SET(v, CONCAT('$.k', i), REPEAT('x', 50)); + SET i = i + 1; + END WHILE; + SELECT LENGTH(v) AS len, JSON_VALID(v) AS still_valid, JSON_LENGTH(v) AS members; + SELECT JSON_EXTRACT(v, '$.k0') AS first_added; + SELECT JSON_EXTRACT(v, '$.k19') AS last_added; +END $$ +--delimiter ; +CALL p_grow(); +DROP PROCEDURE p_grow; + +--echo # +--echo # 4. Variables passed in and out of procedures. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_out(IN a JSON, OUT b JSON, INOUT c JSON) +BEGIN + SET b = JSON_SET(a, '$.from_in', 1); + SET c = JSON_SET(c, '$.from_inout', 1); + SELECT JSON_ARRAY(a) AS in_embedded, JSON_ARRAY(c) AS inout_embedded; +END $$ +--delimiter ; +SET @in = '{"a":1}'; +SET @out = NULL; +SET @inout = '{"c":1}'; +CALL p_out(@in, @out, @inout); +SELECT @out AS out_value, @inout AS inout_value; +SELECT JSON_VALID(@out) AS out_valid, JSON_VALID(@inout) AS inout_valid; +SELECT JSON_ARRAY(@out) AS user_var_embedded; +SELECT JSON_VALID(JSON_ARRAY(@out)) AS still_valid; +DROP PROCEDURE p_out; + +--echo # a variable holding a document that is not valid +SET @bad = '{"a":1,'; +SELECT JSON_VALID(@bad) AS v; +SELECT JSON_ARRAY(@bad) AS embedded; +SELECT JSON_VALID(JSON_ARRAY(@bad)) AS still_valid; +SELECT JSON_SET(@bad, '$.b', 1) AS mutated; + +--echo # +--echo # 5. Variables inside a ROW, and a variable fed from a cursor. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_row() +BEGIN + DECLARE r ROW (a JSON, b LONGTEXT); + SET r.a = JSON_SET('{"a":1}', '$.b', 2); + SET r.b = JSON_SET('{"a":1}', '$.b', 2); + SELECT r.a AS row_json, r.b AS row_text; + SELECT JSON_ARRAY(r.a) AS from_row_json, JSON_ARRAY(r.b) AS from_row_text; + SELECT JSON_VALID(r.a) AS a_valid, JSON_VALID(r.b) AS b_valid; + SET r.a = JSON_SET(r.a, '$.c', 3); + SELECT r.a AS after_self_assign; +END $$ +--delimiter ; +CALL p_row(); +DROP PROCEDURE p_row; + +--delimiter $$ +CREATE PROCEDURE p_cursor() +BEGIN + DECLARE done INT DEFAULT 0; + DECLARE v JSON; + DECLARE cur CURSOR FOR SELECT j FROM t1; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1; + OPEN cur; + FETCH cur INTO v; + CLOSE cur; + SELECT v AS fetched; + SELECT JSON_ARRAY(v) AS embedded; + SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; + SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +--delimiter ; +CALL p_cursor(); +DROP PROCEDURE p_cursor; + +--echo # a cursor over a column that no longer satisfies its check +SET SESSION check_constraint_checks = OFF; +INSERT INTO t1 VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +--delimiter $$ +CREATE PROCEDURE p_cursor_bad() +BEGIN + DECLARE v JSON; + DECLARE cur CURSOR FOR SELECT j FROM t1 WHERE j = '{"a":1,'; + OPEN cur; + FETCH cur INTO v; + CLOSE cur; + SELECT v AS fetched, JSON_VALID(v) AS valid; + SELECT JSON_ARRAY(v) AS embedded; + SELECT JSON_SET(v, '$.b', 2) AS mutated; +END $$ +--delimiter ; +CALL p_cursor_bad(); +DROP PROCEDURE p_cursor_bad; + +--echo # +--echo # 6. A variable read several times in one call, including through a +--echo # stored function that reads it again while the outer call is still +--echo # evaluating. No SQL-visible way to REASSIGN the variable from +--echo # inside an argument has been identified; if one is found, the +--echo # aliasing shape belongs here too. +--echo # + +--delimiter $$ +CREATE FUNCTION f_side(x JSON) RETURNS INT +BEGIN + RETURN JSON_LENGTH(x); +END $$ +--delimiter ; +--delimiter $$ +CREATE PROCEDURE p_multi() +BEGIN + DECLARE v JSON DEFAULT '{"a":1,"b":2}'; + SELECT JSON_SET(v, '$.c', f_side(v)) AS both_uses; + SELECT JSON_MERGE(v, v) AS twice; + SELECT JSON_ARRAY(v, v) AS twice_embedded; + SELECT JSON_SET(v, '$.x', 1, '$.y', JSON_EXTRACT(v, '$.a')) AS mixed; +END $$ +--delimiter ; +CALL p_multi(); +DROP PROCEDURE p_multi; +DROP FUNCTION f_side; + +--echo # +--echo # 7. A trigger, where the row values are columns rather than variables. +--echo # + +CREATE TABLE t2 (j JSON, note LONGTEXT); +--delimiter $$ +CREATE TRIGGER tr BEFORE INSERT ON t2 FOR EACH ROW +BEGIN + SET NEW.note = JSON_ARRAY(NEW.j); +END $$ +--delimiter ; +INSERT INTO t2 (j) VALUES ('{"a":1}'); +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2; +--echo # A row whose document does not read leaves the trigger with +--echo # nothing to set the column to. Saying so is a warning, and an +--echo # INSERT under strict mode makes an error of it, so the row does +--echo # not go in at all - where a released server stored the bytes the +--echo # constructor had spliced and let the column keep them. +SET SESSION check_constraint_checks = OFF; +--error ER_JSON_EOS +INSERT INTO t2 (j) VALUES ('{"a":1,'); +SET SESSION check_constraint_checks = ON; +SELECT j, note, JSON_VALID(note) AS note_valid FROM t2 ORDER BY j; +DROP TABLE t2; + +DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp_scan_count.result b/mysql-test/main/func_json_sp_scan_count.result new file mode 100644 index 0000000000000..3a4cff5492163 --- /dev/null +++ b/mysql-test/main/func_json_sp_scan_count.result @@ -0,0 +1,274 @@ +# +# How many times a value out of a stored program's variable is read +# to find out whether it is a document. +# +# No query can see the difference - the reading could only have found +# out what was already known - so nothing but Json_scans says whether +# it happened. The counts below are the record of it. A count that +# goes up is a reading that came back. +# +# A debug build reads values back to check what was claimed about +# them, and none of those readings are counted: they are the debug +# build's work and not the server's. A count of 0 below therefore +# means nothing read, not that a reading went uncounted. +# +SET NAMES utf8mb4; +# +# 1. A variable assigned by hand, then edited twice. Nobody said +# anything about what was typed, so the first edit reads it and then +# reads back what it composed. The second edit is over what the +# first one put there, which was attested to as it went in, so it +# reads once. +# +CREATE PROCEDURE p_literal() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= '{"a":1}'; +SET v= JSON_SET(v, '$.b', 2); +SET v= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_literal(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +# +# 2. The same three statements, with the first one assigning what a +# function attested rather than what somebody typed. +# +CREATE PROCEDURE p_attested() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_SET(v, '$.b', 2); +SET v= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_attested(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 3. One edit on its own, either way round. +# +CREATE PROCEDURE p_one_literal() +BEGIN +DECLARE v LONGTEXT DEFAULT '{"a":1}'; +SET v= JSON_SET(v, '$.b', 2); +END $$ +CREATE PROCEDURE p_one_attested() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_SET(v, '$.b', 2); +END $$ +FLUSH STATUS; +CALL p_one_literal(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +CALL p_one_attested(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# +# 4. One variable assigned from another. Moving a value from one +# variable to another reads nothing, and what was said about it +# arrives with it. +# +CREATE PROCEDURE p_var_to_var() +BEGIN +DECLARE a LONGTEXT DEFAULT NULL; +DECLARE b LONGTEXT DEFAULT NULL; +SET a= JSON_OBJECT('a', 1); +SET b= a; +SET b= JSON_SET(b, '$.b', 2); +END $$ +FLUSH STATUS; +CALL p_var_to_var(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# +# 5. The other mutators over the same variable, once each. +# +CREATE PROCEDURE p_mutators() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE r LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1, 'arr', JSON_ARRAY(1, 2)); +SET r= JSON_INSERT(v, '$.b', 2); +SET r= JSON_REPLACE(v, '$.a', 9); +SET r= JSON_REMOVE(v, '$.a'); +SET r= JSON_ARRAY_APPEND(v, '$.arr', 3); +SET r= JSON_ARRAY_INSERT(v, '$.arr[0]', 0); +END $$ +FLUSH STATUS; +CALL p_mutators(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 5 +# +# 6. A member of a row variable, written one member at a time and +# then a whole row at a time. A row is written a member at a time +# either way - the whole-row form walks the members and puts each +# one through the same funnel - so both say the same thing and both +# are read the same number of times. +# +CREATE PROCEDURE p_row_member() +BEGIN +DECLARE r ROW(a LONGTEXT, b LONGTEXT); +DECLARE out1 LONGTEXT DEFAULT NULL; +SET r.a= JSON_OBJECT('x', 1); +SET r.b= JSON_ARRAY(1, 2); +SET out1= JSON_SET(r.a, '$.y', 2); +SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +CREATE PROCEDURE p_row_whole() +BEGIN +DECLARE r ROW(a LONGTEXT, b LONGTEXT); +DECLARE out1 LONGTEXT DEFAULT NULL; +SET r= ROW(JSON_OBJECT('x', 1), JSON_ARRAY(1, 2)); +SET out1= JSON_SET(r.a, '$.y', 2); +SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +FLUSH STATUS; +CALL p_row_member(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +FLUSH STATUS; +CALL p_row_whole(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 7. A value that does not fit what it is being assigned to. What +# was said about it was said about the value passed and not +# about what fitted, so the answer goes with the part that did not. +# +# Which of the two things happens depends on the mode. Under the +# strict one the assignment does not land at all and the variable is +# left holding nothing, so there is nothing to read; under the loose +# one the front of the value is put down, quietly and without so +# much as a warning, and what is there is not a document. It is the +# second that the store having to be asked what it kept is for: no +# error is raised on that path and nothing else would say. +CREATE PROCEDURE p_truncated() +BEGIN +DECLARE fits VARCHAR(64) DEFAULT NULL; +DECLARE cut VARCHAR(8) DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET fits= JSON_OBJECT('a', 1); +SET out1= JSON_SET(fits, '$.b', 2); +SET cut= JSON_OBJECT('aaaaaaaaaa', 1); +SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; +SET out1= JSON_SET(cut, '$.b', 2); +END $$ +FLUSH STATUS; +CALL p_truncated(); +what_landed reads_as_document +NULL NULL +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# the same body under the mode that lets a short store through, which +# a procedure keeps from the moment it is created and not from the +# moment it is called +SET @save_sql_mode= @@sql_mode; +SET sql_mode=''; +CREATE PROCEDURE p_truncated_loose() +BEGIN +DECLARE fits VARCHAR(64) DEFAULT NULL; +DECLARE cut VARCHAR(8) DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET fits= JSON_OBJECT('a', 1); +SET out1= JSON_SET(fits, '$.b', 2); +SET cut= JSON_OBJECT('aaaaaaaaaa', 1); +SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; +SET out1= JSON_SET(cut, '$.b', 2); +END $$ +SET sql_mode= @save_sql_mode; +FLUSH STATUS; +CALL p_truncated_loose(); +what_landed reads_as_document +{"aaaaaa 0 +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_set' +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +# +# 8. An assignment that never landed. The answer the one before it +# left is given up with it, and the variable is left holding +# nothing, so what a reader finds is a NULL and not a stale +# document with a stale answer beside it. +# +CREATE PROCEDURE p_failed() +BEGIN +DECLARE v VARCHAR(20) DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_OBJECT('bbbbbbbbbbbbbbbbbbbb', 2); +SELECT v AS after_failure; +SET out1= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_failed(); +after_failure +NULL +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 9. A value out of a table by SELECT INTO, from a column that says +# nothing and from a function that does. Two edits rather than one, +# so that the reading the function's own scan pays for does not hide +# the readings the edits do or do not do. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); +CREATE PROCEDURE p_from_column() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +SELECT j INTO v FROM t1 WHERE id= 1; +SET out1= JSON_SET(v, '$.b', 2); +SET out1= JSON_SET(v, '$.c', 3); +END $$ +CREATE PROCEDURE p_from_function() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE out1 LONGTEXT DEFAULT NULL; +SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 1; +SET out1= JSON_SET(v, '$.b', 2); +SET out1= JSON_SET(v, '$.c', 3); +END $$ +FLUSH STATUS; +CALL p_from_column(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +FLUSH STATUS; +CALL p_from_function(); +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 3 +DROP PROCEDURE p_literal; +DROP PROCEDURE p_attested; +DROP PROCEDURE p_one_literal; +DROP PROCEDURE p_one_attested; +DROP PROCEDURE p_var_to_var; +DROP PROCEDURE p_mutators; +DROP PROCEDURE p_row_member; +DROP PROCEDURE p_row_whole; +DROP PROCEDURE p_truncated; +DROP PROCEDURE p_truncated_loose; +DROP PROCEDURE p_failed; +DROP PROCEDURE p_from_column; +DROP PROCEDURE p_from_function; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp_scan_count.test b/mysql-test/main/func_json_sp_scan_count.test new file mode 100644 index 0000000000000..36e6872fd548c --- /dev/null +++ b/mysql-test/main/func_json_sp_scan_count.test @@ -0,0 +1,275 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a value out of a stored program's variable is read +--echo # to find out whether it is a document. +--echo # +--echo # No query can see the difference - the reading could only have found +--echo # out what was already known - so nothing but Json_scans says whether +--echo # it happened. The counts below are the record of it. A count that +--echo # goes up is a reading that came back. +--echo # +--echo # A debug build reads values back to check what was claimed about +--echo # them, and none of those readings are counted: they are the debug +--echo # build's work and not the server's. A count of 0 below therefore +--echo # means nothing read, not that a reading went uncounted. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A variable assigned by hand, then edited twice. Nobody said +--echo # anything about what was typed, so the first edit reads it and then +--echo # reads back what it composed. The second edit is over what the +--echo # first one put there, which was attested to as it went in, so it +--echo # reads once. +--echo # +--delimiter $$ +CREATE PROCEDURE p_literal() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= '{"a":1}'; + SET v= JSON_SET(v, '$.b', 2); + SET v= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_literal(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 2. The same three statements, with the first one assigning what a +--echo # function attested rather than what somebody typed. +--echo # +--delimiter $$ +CREATE PROCEDURE p_attested() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_SET(v, '$.b', 2); + SET v= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_attested(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 3. One edit on its own, either way round. +--echo # +--delimiter $$ +CREATE PROCEDURE p_one_literal() +BEGIN + DECLARE v LONGTEXT DEFAULT '{"a":1}'; + SET v= JSON_SET(v, '$.b', 2); +END $$ +CREATE PROCEDURE p_one_attested() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_SET(v, '$.b', 2); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_one_literal(); +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +CALL p_one_attested(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 4. One variable assigned from another. Moving a value from one +--echo # variable to another reads nothing, and what was said about it +--echo # arrives with it. +--echo # +--delimiter $$ +CREATE PROCEDURE p_var_to_var() +BEGIN + DECLARE a LONGTEXT DEFAULT NULL; + DECLARE b LONGTEXT DEFAULT NULL; + SET a= JSON_OBJECT('a', 1); + SET b= a; + SET b= JSON_SET(b, '$.b', 2); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_var_to_var(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 5. The other mutators over the same variable, once each. +--echo # +--delimiter $$ +CREATE PROCEDURE p_mutators() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE r LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1, 'arr', JSON_ARRAY(1, 2)); + SET r= JSON_INSERT(v, '$.b', 2); + SET r= JSON_REPLACE(v, '$.a', 9); + SET r= JSON_REMOVE(v, '$.a'); + SET r= JSON_ARRAY_APPEND(v, '$.arr', 3); + SET r= JSON_ARRAY_INSERT(v, '$.arr[0]', 0); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_mutators(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 6. A member of a row variable, written one member at a time and +--echo # then a whole row at a time. A row is written a member at a time +--echo # either way - the whole-row form walks the members and puts each +--echo # one through the same funnel - so both say the same thing and both +--echo # are read the same number of times. +--echo # +--delimiter $$ +CREATE PROCEDURE p_row_member() +BEGIN + DECLARE r ROW(a LONGTEXT, b LONGTEXT); + DECLARE out1 LONGTEXT DEFAULT NULL; + SET r.a= JSON_OBJECT('x', 1); + SET r.b= JSON_ARRAY(1, 2); + SET out1= JSON_SET(r.a, '$.y', 2); + SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +CREATE PROCEDURE p_row_whole() +BEGIN + DECLARE r ROW(a LONGTEXT, b LONGTEXT); + DECLARE out1 LONGTEXT DEFAULT NULL; + SET r= ROW(JSON_OBJECT('x', 1), JSON_ARRAY(1, 2)); + SET out1= JSON_SET(r.a, '$.y', 2); + SET out1= JSON_ARRAY_APPEND(r.b, '$', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_row_member(); +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +CALL p_row_whole(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 7. A value that does not fit what it is being assigned to. What +--echo # was said about it was said about the value passed and not +--echo # about what fitted, so the answer goes with the part that did not. +--echo # +--echo # Which of the two things happens depends on the mode. Under the +--echo # strict one the assignment does not land at all and the variable is +--echo # left holding nothing, so there is nothing to read; under the loose +--echo # one the front of the value is put down, quietly and without so +--echo # much as a warning, and what is there is not a document. It is the +--echo # second that the store having to be asked what it kept is for: no +--echo # error is raised on that path and nothing else would say. +--delimiter $$ +CREATE PROCEDURE p_truncated() +BEGIN + DECLARE fits VARCHAR(64) DEFAULT NULL; + DECLARE cut VARCHAR(8) DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET fits= JSON_OBJECT('a', 1); + SET out1= JSON_SET(fits, '$.b', 2); + SET cut= JSON_OBJECT('aaaaaaaaaa', 1); + SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; + SET out1= JSON_SET(cut, '$.b', 2); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_truncated(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # the same body under the mode that lets a short store through, which +--echo # a procedure keeps from the moment it is created and not from the +--echo # moment it is called +SET @save_sql_mode= @@sql_mode; +SET sql_mode=''; +--delimiter $$ +CREATE PROCEDURE p_truncated_loose() +BEGIN + DECLARE fits VARCHAR(64) DEFAULT NULL; + DECLARE cut VARCHAR(8) DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET fits= JSON_OBJECT('a', 1); + SET out1= JSON_SET(fits, '$.b', 2); + SET cut= JSON_OBJECT('aaaaaaaaaa', 1); + SELECT cut AS what_landed, JSON_VALID(cut) AS reads_as_document; + SET out1= JSON_SET(cut, '$.b', 2); +END $$ +--delimiter ; +SET sql_mode= @save_sql_mode; +FLUSH STATUS; +CALL p_truncated_loose(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 8. An assignment that never landed. The answer the one before it +--echo # left is given up with it, and the variable is left holding +--echo # nothing, so what a reader finds is a NULL and not a stale +--echo # document with a stale answer beside it. +--echo # +--delimiter $$ +CREATE PROCEDURE p_failed() +BEGIN + DECLARE v VARCHAR(20) DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_OBJECT('bbbbbbbbbbbbbbbbbbbb', 2); + SELECT v AS after_failure; + SET out1= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_failed(); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 9. A value out of a table by SELECT INTO, from a column that says +--echo # nothing and from a function that does. Two edits rather than one, +--echo # so that the reading the function's own scan pays for does not hide +--echo # the readings the edits do or do not do. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); + +--delimiter $$ +CREATE PROCEDURE p_from_column() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + SELECT j INTO v FROM t1 WHERE id= 1; + SET out1= JSON_SET(v, '$.b', 2); + SET out1= JSON_SET(v, '$.c', 3); +END $$ +CREATE PROCEDURE p_from_function() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE out1 LONGTEXT DEFAULT NULL; + SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 1; + SET out1= JSON_SET(v, '$.b', 2); + SET out1= JSON_SET(v, '$.c', 3); +END $$ +--delimiter ; +FLUSH STATUS; +CALL p_from_column(); +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +CALL p_from_function(); +SHOW STATUS LIKE 'Json_scans'; + +DROP PROCEDURE p_literal; +DROP PROCEDURE p_attested; +DROP PROCEDURE p_one_literal; +DROP PROCEDURE p_one_attested; +DROP PROCEDURE p_var_to_var; +DROP PROCEDURE p_mutators; +DROP PROCEDURE p_row_member; +DROP PROCEDURE p_row_whole; +DROP PROCEDURE p_truncated; +DROP PROCEDURE p_truncated_loose; +DROP PROCEDURE p_failed; +DROP PROCEDURE p_from_column; +DROP PROCEDURE p_from_function; +DROP TABLE t1; diff --git a/mysql-test/main/func_json_sp_trust.result b/mysql-test/main/func_json_sp_trust.result new file mode 100644 index 0000000000000..a9fc0930a4c02 --- /dev/null +++ b/mysql-test/main/func_json_sp_trust.result @@ -0,0 +1,431 @@ +# +# What a JSON function reads out of a stored program's variable. +# +# A variable is written by one funnel and by nothing else, and every +# assignment to one arrives there with the expression that made the +# value still in hand. So what was put in a variable can be said at +# the moment it is put there, and a function later handed that +# variable as the document to work on can be spared finding out +# again what is in it. +# +# A variable is not typed as a document however it was declared, so +# a value out of one is still quoted into a document rather than +# spliced into it. That is decided by the type and is left alone +# here. The answers below are the same either way - the reading +# that is left out could only have found out what was already +# known - so this file is here to say that they stay the same. +# +SET NAMES utf8mb4; +# +# 1. A variable assigned by a function that attests to its result, +# then read as a document. +# +CREATE PROCEDURE p_read() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_SET('{"a":1}', '$.b', 2); +SELECT v AS held; +SELECT JSON_SET(v, '$.c', 3) AS edited; +SELECT JSON_INSERT(v, '$.c', 3) AS inserted; +SELECT JSON_REPLACE(v, '$.a', 9) AS replaced; +SELECT JSON_REMOVE(v, '$.a') AS removed; +SELECT JSON_ARRAY_APPEND(JSON_SET(v, '$.arr', JSON_ARRAY(1)), '$.arr', 2) +AS appended; +SELECT JSON_MERGE_PATCH(v, '{"d":4}') AS merged; +SELECT JSON_EXTRACT(v, '$.a') AS extracted; +SELECT JSON_TYPE(v) AS what, JSON_DEPTH(v) AS deep, JSON_LENGTH(v) AS len; +SELECT JSON_VALID(v) AS valid, JSON_KEYS(v) AS keys_of; +END $$ +CALL p_read(); +held +{"a": 1, "b": 2} +edited +{"a": 1, "b": 2, "c": 3} +inserted +{"a": 1, "b": 2, "c": 3} +replaced +{"a": 9, "b": 2} +removed +{"b": 2} +appended +{"a": 1, "b": 2, "arr": [1, 2]} +merged +{"a": 1, "b": 2, "d": 4} +extracted +1 +what deep len +OBJECT 2 2 +valid keys_of +1 ["a", "b"] +DROP PROCEDURE p_read; +# +# 2. The same value spliced into a new document. A variable is not +# typed as a document, so it is quoted - which is what a released +# server does with it and what it goes on doing. +# +CREATE PROCEDURE p_splice() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE j JSON DEFAULT NULL; +SET v= JSON_SET('{"a":1}', '$.b', 2); +SET j= JSON_SET('{"a":1}', '$.b', 2); +SELECT JSON_ARRAY(v) AS text_var, JSON_ARRAY(j) AS json_var; +SELECT JSON_OBJECT('k', v) AS obj_text, JSON_OBJECT('k', j) AS obj_json; +SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; +END $$ +CALL p_splice(); +text_var json_var +["{\"a\": 1, \"b\": 2}"] ["{\"a\": 1, \"b\": 2}"] +obj_text obj_json +{"k": "{\"a\": 1, \"b\": 2}"} {"k": "{\"a\": 1, \"b\": 2}"} +still_valid +1 +DROP PROCEDURE p_splice; +# +# 3. One variable assigned from another. The assignment reaches the +# funnel as a read of the field behind the source, so what was said +# about the value travels with it and does not have to be said +# again. +# +CREATE PROCEDURE p_var_to_var() +BEGIN +DECLARE a LONGTEXT DEFAULT NULL; +DECLARE b LONGTEXT DEFAULT NULL; +SET a= JSON_SET('{"a":1}', '$.b', 2); +SET b= a; +SELECT JSON_SET(b, '$.c', 3) AS edited; +SET a= 'not a document at all'; +SELECT JSON_SET(b, '$.d', 4) AS still_edited; +SELECT b AS unchanged; +END $$ +CALL p_var_to_var(); +edited +{"a": 1, "b": 2, "c": 3} +still_edited +{"a": 1, "b": 2, "d": 4} +unchanged +{"a": 1, "b": 2} +DROP PROCEDURE p_var_to_var; +# +# 4. A variable that is both the source and the target. +# +CREATE PROCEDURE p_self() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v= JSON_OBJECT('a', 1); +SET v= JSON_SET(v, '$.b', 2); +SET v= JSON_SET(v, '$.c', 3); +SELECT v AS built; +SET v= v; +SELECT JSON_SET(v, '$.d', 4) AS after_self_assign; +END $$ +CALL p_self(); +built +{"a": 1, "b": 2, "c": 3} +after_self_assign +{"a": 1, "b": 2, "c": 3, "d": 4} +DROP PROCEDURE p_self; +# +# 5. Assignments that put something else there. A literal is not +# attested by anybody, and a value the store had to change is +# not the value that was attested. +# +CREATE PROCEDURE p_untrusted() +BEGIN +DECLARE lit LONGTEXT DEFAULT NULL; +DECLARE short_v VARCHAR(8) DEFAULT NULL; +DECLARE num INT DEFAULT NULL; +SET lit= '{"a":1}'; +SELECT JSON_SET(lit, '$.b', 2) AS from_literal; +SELECT JSON_TYPE(lit) AS what; +SET num= JSON_LENGTH(JSON_SET('{"a":1}', '$.b', 2)); +SELECT JSON_SET(CONCAT('[', num, ']'), '$[1]', 9) AS from_number; +BEGIN +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET short_v= JSON_SET('{"a":1}', '$.b', 2); +END; +SELECT short_v AS truncated; +SELECT JSON_VALID(short_v) AS valid_after_truncation; +END $$ +CALL p_untrusted(); +from_literal +{"a": 1, "b": 2} +what +OBJECT +from_number +[2, 9] +truncated +NULL +valid_after_truncation +NULL +DROP PROCEDURE p_untrusted; +# +# 6. An assignment that does not happen. What the one before it +# left standing goes with it, the bytes under it being whatever the +# store got as far as putting down. +# +CREATE PROCEDURE p_failed() +BEGIN +DECLARE v VARCHAR(20) DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET v= JSON_SET('{"a":1}', '$.b', 2); +SELECT JSON_SET(v, '$.c', 3) AS edited; +SET v= JSON_SET('{"aaaaaaaaaa":1}', '$.bbbbbbbbbb', 2); +SELECT v AS after_failure; +SELECT JSON_VALID(v) AS valid_after_failure; +END $$ +CALL p_failed(); +edited +{"a": 1, "b": 2, "c": 3} +after_failure +NULL +valid_after_failure +NULL +DROP PROCEDURE p_failed; +# +# 7. Row variables. A row is written a member at a time whichever +# form the assignment takes, each member going through the same +# funnel a plain variable goes through, so each of them is answered +# for on its own and by whatever made the value that landed in it. +# +CREATE PROCEDURE p_row() +BEGIN +DECLARE r ROW(a LONGTEXT, b LONGTEXT); +SET r.a= JSON_SET('{"x":1}', '$.y', 2); +SET r.b= JSON_SET('[1]', '$[1]', 2); +SELECT JSON_SET(r.a, '$.z', 3) AS member_a; +SELECT JSON_ARRAY_APPEND(r.b, '$', 3) AS member_b; +SET r= ROW(JSON_OBJECT('p', 1), JSON_ARRAY(7, 8)); +SELECT JSON_SET(r.a, '$.q', 2) AS whole_row_a; +SELECT JSON_ARRAY_APPEND(r.b, '$', 9) AS whole_row_b; +SET r= ROW('{"lit":1}', '[0]'); +SELECT JSON_SET(r.a, '$.q', 2) AS literal_row_a; +END $$ +CALL p_row(); +member_a +{"x": 1, "y": 2, "z": 3} +member_b +[1, 2, 3] +whole_row_a +{"p": 1, "q": 2} +whole_row_b +[7, 8, 9] +literal_row_a +{"lit": 1, "q": 2} +DROP PROCEDURE p_row; +# +# 8. A row assignment that fails part of the way through. The +# members it did not reach are still holding what the assignment +# before this one left, so their answers go; the ones it wrote keep +# theirs, having been attested on the way in. +# +CREATE PROCEDURE p_row_partial() +BEGIN +DECLARE r ROW(a VARCHAR(20), b VARCHAR(8)); +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET r.a= JSON_SET('{"x":1}', '$.y', 2); +SET r.b= JSON_SET('[1]', '$[1]', 2); +SELECT JSON_SET(r.a, '$.z', 3) AS before_a; +SET r= ROW(JSON_OBJECT('p', 1), JSON_OBJECT('qqqqqqqqqq', 22222)); +SELECT r.a AS after_a, r.b AS after_b; +SELECT JSON_VALID(r.a) AS valid_a, JSON_VALID(r.b) AS valid_b; +END $$ +CALL p_row_partial(); +before_a +{"x": 1, "y": 2, "z": 3} +after_a after_b +{"p": 1} NULL +valid_a valid_b +1 NULL +DROP PROCEDURE p_row_partial; +# +# 9. A value that arrives from a table rather than from a function, +# by SELECT INTO and by FETCH. +# +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); +CREATE PROCEDURE p_into() +BEGIN +DECLARE done INT DEFAULT 0; +DECLARE v LONGTEXT DEFAULT NULL; +DECLARE c CURSOR FOR SELECT j FROM t1 ORDER BY id; +DECLARE CONTINUE HANDLER FOR NOT FOUND SET done= 1; +SELECT j INTO v FROM t1 WHERE id= 1; +SELECT JSON_SET(v, '$.b', 2) AS from_select_into; +SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 2; +SELECT JSON_ARRAY_APPEND(v, '$', 3) AS from_extract_into; +OPEN c; +read_loop: LOOP +FETCH c INTO v; +IF done THEN +LEAVE read_loop; +END IF; +SELECT JSON_TYPE(v) AS fetched_type, JSON_DEPTH(v) AS fetched_depth; +END LOOP; +CLOSE c; +END $$ +CALL p_into(); +from_select_into +{"a": 1, "b": 2} +from_extract_into +[1, 2, 3] +fetched_type fetched_depth +OBJECT 2 +fetched_type fetched_depth +ARRAY 2 +DROP PROCEDURE p_into; +# +# 10. Through a stored function, whose parameter is a variable of the +# frame the call makes and whose result is not one at all. +# +CREATE FUNCTION f_edit(doc LONGTEXT) RETURNS LONGTEXT +BEGIN +RETURN JSON_SET(doc, '$.f', 1); +END $$ +SELECT f_edit(JSON_OBJECT('a', 1)) AS called_with_document; +called_with_document +{"a": 1, "f": 1} +SELECT f_edit('{"a":1}') AS called_with_literal; +called_with_literal +{"a": 1, "f": 1} +SELECT JSON_SET(f_edit(JSON_OBJECT('a', 1)), '$.g', 2) AS over_the_result; +over_the_result +{"a": 1, "f": 1, "g": 2} +SELECT JSON_ARRAY(f_edit(JSON_OBJECT('a', 1))) AS spliced_result; +spliced_result +["{\"a\": 1, \"f\": 1}"] +SELECT f_edit(j) AS over_a_column FROM t1 ORDER BY id; +over_a_column +{"a": 1, "f": 1} +[1, 2] +DROP FUNCTION f_edit; +# +# 11. Values the declared type has to change on the way in: a +# character set that can hold every character of the document, and +# one that cannot. +# +CREATE PROCEDURE p_charset() +BEGIN +DECLARE l1 VARCHAR(64) CHARACTER SET latin1 DEFAULT NULL; +DECLARE u VARCHAR(64) CHARACTER SET utf8mb4 DEFAULT NULL; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET u= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); +SELECT JSON_SET(u, '$.c', 'y') AS utf8_var; +SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); +SELECT JSON_SET(l1, '$.c', 'y') AS latin1_var; +SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xC3A4); +SELECT HEX(l1) AS latin1_converted; +SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xE4B8AD); +SELECT HEX(l1) AS latin1_lossy; +SELECT JSON_VALID(l1) AS valid_after_loss; +END $$ +CALL p_charset(); +utf8_var +{"a": "e", "b": "z", "c": "y"} +latin1_var +{"a": "e", "b": "z", "c": "y"} +latin1_converted +7B2261223A202265222C202262223A2022E4227D +latin1_lossy +NULL +valid_after_loss +NULL +DROP PROCEDURE p_charset; +# +# 12. A declared type that changes nothing on the way in and still +# leaves something else there. A store into the binary type keeps +# the bytes and calls them by another name, so a document written +# in a wide set arrives as bytes that read as no document at all. +# Once for a conversion written out in the assignment, and once for +# a variable already holding the wide encoding. +# +CREATE PROCEDURE p_binary() +BEGIN +DECLARE b BLOB DEFAULT NULL; +DECLARE w VARCHAR(64) CHARACTER SET ucs2 DEFAULT NULL; +SET b= CONVERT(JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z') +USING ucs2); +SELECT HEX(b) AS binary_converted, JSON_VALID(b) AS valid_converted; +SET w= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); +SELECT JSON_VALID(w) AS valid_wide; +SET b= w; +SELECT HEX(b) AS binary_from_wide, JSON_VALID(b) AS valid_from_wide; +END $$ +CALL p_binary(); +binary_converted valid_converted +007B002200610022003A0020002200650022002C0020002200620022003A00200022007A0022007D 0 +valid_wide +1 +binary_from_wide valid_from_wide +007B002200610022003A0020002200650022002C0020002200620022003A00200022007A0022007D 0 +Warnings: +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +DROP PROCEDURE p_binary; +DROP TABLE t1; +# +# 13. A row variable written from a list of values rather than from +# a row. Section 8 assigns a whole row at once; SELECT INTO and +# FETCH INTO write the members one at a time from a list and stop at +# the first that fails. What the variable holds afterwards is a +# mixture of three things: the members before the failure carry what +# this assignment wrote, the one it stopped at is left holding +# nothing, and the ones after it still carry what the assignment +# before left. Anything said about the last two belongs to an +# assignment that did not finish, and goes with it. +# +# A variable is never typed as a document, so a value read out of +# one is quoted into a new document rather than spliced into it, in +# both states - which is what the whole-assignment case is printed +# for. What the answers show is that reading the mixture says the +# same things about it as reading a variable nothing went wrong +# with. +# +SET @@sql_mode='STRICT_ALL_TABLES'; +CREATE PROCEDURE p_row_into() +BEGIN +DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION +SELECT r.a AS written_by_this_assignment, +r.b AS the_member_it_stopped_at, +r.c AS left_by_the_assignment_before, +JSON_VALID(r.c) AS still_a_document, +JSON_TYPE(r.c) AS ty, +JSON_ARRAY(r.c) AS quoted; +SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; +SELECT r.a AS a, r.b AS b, r.c AS c, +JSON_TYPE(r.c) AS ty, +JSON_ARRAY(r.c) AS quoted_after_a_whole_assignment; +SELECT JSON_OBJECT('second', 2), 'not a number', JSON_OBJECT('fourth', 4) +INTO r; +SELECT 'the handler ran and the call went on' AS resumed; +END $$ +CREATE PROCEDURE p_row_fetch() +BEGIN +DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); +DECLARE cur CURSOR FOR +SELECT JSON_OBJECT('fifth', 5) AS a, 'not a number' AS b, +JSON_OBJECT('sixth', 6) AS c; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION +SELECT r.a AS written_by_this_fetch, +r.b AS the_member_it_stopped_at, +r.c AS left_by_the_assignment_before, +JSON_VALID(r.c) AS still_a_document, +JSON_ARRAY(r.c) AS quoted; +SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; +OPEN cur; +FETCH cur INTO r; +CLOSE cur; +END $$ +CALL p_row_into(); +a b c ty quoted_after_a_whole_assignment +{"first": 1} 1 {"third": 3} OBJECT ["{\"third\": 3}"] +written_by_this_assignment the_member_it_stopped_at left_by_the_assignment_before still_a_document ty quoted +{"second": 2} NULL {"third": 3} 1 OBJECT ["{\"third\": 3}"] +resumed +the handler ran and the call went on +CALL p_row_fetch(); +written_by_this_fetch the_member_it_stopped_at left_by_the_assignment_before still_a_document quoted +{"fifth": 5} NULL {"third": 3} 1 ["{\"third\": 3}"] +DROP PROCEDURE p_row_into; +DROP PROCEDURE p_row_fetch; +SET @@sql_mode=DEFAULT; diff --git a/mysql-test/main/func_json_sp_trust.test b/mysql-test/main/func_json_sp_trust.test new file mode 100644 index 0000000000000..a2614b5d80845 --- /dev/null +++ b/mysql-test/main/func_json_sp_trust.test @@ -0,0 +1,366 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # What a JSON function reads out of a stored program's variable. +--echo # +--echo # A variable is written by one funnel and by nothing else, and every +--echo # assignment to one arrives there with the expression that made the +--echo # value still in hand. So what was put in a variable can be said at +--echo # the moment it is put there, and a function later handed that +--echo # variable as the document to work on can be spared finding out +--echo # again what is in it. +--echo # +--echo # A variable is not typed as a document however it was declared, so +--echo # a value out of one is still quoted into a document rather than +--echo # spliced into it. That is decided by the type and is left alone +--echo # here. The answers below are the same either way - the reading +--echo # that is left out could only have found out what was already +--echo # known - so this file is here to say that they stay the same. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A variable assigned by a function that attests to its result, +--echo # then read as a document. +--echo # +--delimiter $$ +CREATE PROCEDURE p_read() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_SET('{"a":1}', '$.b', 2); + SELECT v AS held; + SELECT JSON_SET(v, '$.c', 3) AS edited; + SELECT JSON_INSERT(v, '$.c', 3) AS inserted; + SELECT JSON_REPLACE(v, '$.a', 9) AS replaced; + SELECT JSON_REMOVE(v, '$.a') AS removed; + SELECT JSON_ARRAY_APPEND(JSON_SET(v, '$.arr', JSON_ARRAY(1)), '$.arr', 2) + AS appended; + SELECT JSON_MERGE_PATCH(v, '{"d":4}') AS merged; + SELECT JSON_EXTRACT(v, '$.a') AS extracted; + SELECT JSON_TYPE(v) AS what, JSON_DEPTH(v) AS deep, JSON_LENGTH(v) AS len; + SELECT JSON_VALID(v) AS valid, JSON_KEYS(v) AS keys_of; +END $$ +--delimiter ; +CALL p_read(); +DROP PROCEDURE p_read; + +--echo # +--echo # 2. The same value spliced into a new document. A variable is not +--echo # typed as a document, so it is quoted - which is what a released +--echo # server does with it and what it goes on doing. +--echo # +--delimiter $$ +CREATE PROCEDURE p_splice() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE j JSON DEFAULT NULL; + SET v= JSON_SET('{"a":1}', '$.b', 2); + SET j= JSON_SET('{"a":1}', '$.b', 2); + SELECT JSON_ARRAY(v) AS text_var, JSON_ARRAY(j) AS json_var; + SELECT JSON_OBJECT('k', v) AS obj_text, JSON_OBJECT('k', j) AS obj_json; + SELECT JSON_VALID(JSON_ARRAY(v)) AS still_valid; +END $$ +--delimiter ; +CALL p_splice(); +DROP PROCEDURE p_splice; + +--echo # +--echo # 3. One variable assigned from another. The assignment reaches the +--echo # funnel as a read of the field behind the source, so what was said +--echo # about the value travels with it and does not have to be said +--echo # again. +--echo # +--delimiter $$ +CREATE PROCEDURE p_var_to_var() +BEGIN + DECLARE a LONGTEXT DEFAULT NULL; + DECLARE b LONGTEXT DEFAULT NULL; + SET a= JSON_SET('{"a":1}', '$.b', 2); + SET b= a; + SELECT JSON_SET(b, '$.c', 3) AS edited; + SET a= 'not a document at all'; + SELECT JSON_SET(b, '$.d', 4) AS still_edited; + SELECT b AS unchanged; +END $$ +--delimiter ; +CALL p_var_to_var(); +DROP PROCEDURE p_var_to_var; + +--echo # +--echo # 4. A variable that is both the source and the target. +--echo # +--delimiter $$ +CREATE PROCEDURE p_self() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v= JSON_OBJECT('a', 1); + SET v= JSON_SET(v, '$.b', 2); + SET v= JSON_SET(v, '$.c', 3); + SELECT v AS built; + SET v= v; + SELECT JSON_SET(v, '$.d', 4) AS after_self_assign; +END $$ +--delimiter ; +CALL p_self(); +DROP PROCEDURE p_self; + +--echo # +--echo # 5. Assignments that put something else there. A literal is not +--echo # attested by anybody, and a value the store had to change is +--echo # not the value that was attested. +--echo # +--delimiter $$ +CREATE PROCEDURE p_untrusted() +BEGIN + DECLARE lit LONGTEXT DEFAULT NULL; + DECLARE short_v VARCHAR(8) DEFAULT NULL; + DECLARE num INT DEFAULT NULL; + SET lit= '{"a":1}'; + SELECT JSON_SET(lit, '$.b', 2) AS from_literal; + SELECT JSON_TYPE(lit) AS what; + SET num= JSON_LENGTH(JSON_SET('{"a":1}', '$.b', 2)); + SELECT JSON_SET(CONCAT('[', num, ']'), '$[1]', 9) AS from_number; + BEGIN + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET short_v= JSON_SET('{"a":1}', '$.b', 2); + END; + SELECT short_v AS truncated; + SELECT JSON_VALID(short_v) AS valid_after_truncation; +END $$ +--delimiter ; +CALL p_untrusted(); +DROP PROCEDURE p_untrusted; + +--echo # +--echo # 6. An assignment that does not happen. What the one before it +--echo # left standing goes with it, the bytes under it being whatever the +--echo # store got as far as putting down. +--echo # +--delimiter $$ +CREATE PROCEDURE p_failed() +BEGIN + DECLARE v VARCHAR(20) DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET v= JSON_SET('{"a":1}', '$.b', 2); + SELECT JSON_SET(v, '$.c', 3) AS edited; + SET v= JSON_SET('{"aaaaaaaaaa":1}', '$.bbbbbbbbbb', 2); + SELECT v AS after_failure; + SELECT JSON_VALID(v) AS valid_after_failure; +END $$ +--delimiter ; +CALL p_failed(); +DROP PROCEDURE p_failed; + +--echo # +--echo # 7. Row variables. A row is written a member at a time whichever +--echo # form the assignment takes, each member going through the same +--echo # funnel a plain variable goes through, so each of them is answered +--echo # for on its own and by whatever made the value that landed in it. +--echo # +--delimiter $$ +CREATE PROCEDURE p_row() +BEGIN + DECLARE r ROW(a LONGTEXT, b LONGTEXT); + SET r.a= JSON_SET('{"x":1}', '$.y', 2); + SET r.b= JSON_SET('[1]', '$[1]', 2); + SELECT JSON_SET(r.a, '$.z', 3) AS member_a; + SELECT JSON_ARRAY_APPEND(r.b, '$', 3) AS member_b; + SET r= ROW(JSON_OBJECT('p', 1), JSON_ARRAY(7, 8)); + SELECT JSON_SET(r.a, '$.q', 2) AS whole_row_a; + SELECT JSON_ARRAY_APPEND(r.b, '$', 9) AS whole_row_b; + SET r= ROW('{"lit":1}', '[0]'); + SELECT JSON_SET(r.a, '$.q', 2) AS literal_row_a; +END $$ +--delimiter ; +CALL p_row(); +DROP PROCEDURE p_row; + +--echo # +--echo # 8. A row assignment that fails part of the way through. The +--echo # members it did not reach are still holding what the assignment +--echo # before this one left, so their answers go; the ones it wrote keep +--echo # theirs, having been attested on the way in. +--echo # +--delimiter $$ +CREATE PROCEDURE p_row_partial() +BEGIN + DECLARE r ROW(a VARCHAR(20), b VARCHAR(8)); + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET r.a= JSON_SET('{"x":1}', '$.y', 2); + SET r.b= JSON_SET('[1]', '$[1]', 2); + SELECT JSON_SET(r.a, '$.z', 3) AS before_a; + SET r= ROW(JSON_OBJECT('p', 1), JSON_OBJECT('qqqqqqqqqq', 22222)); + SELECT r.a AS after_a, r.b AS after_b; + SELECT JSON_VALID(r.a) AS valid_a, JSON_VALID(r.b) AS valid_b; +END $$ +--delimiter ; +CALL p_row_partial(); +DROP PROCEDURE p_row_partial; + +--echo # +--echo # 9. A value that arrives from a table rather than from a function, +--echo # by SELECT INTO and by FETCH. +--echo # +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); + +--delimiter $$ +CREATE PROCEDURE p_into() +BEGIN + DECLARE done INT DEFAULT 0; + DECLARE v LONGTEXT DEFAULT NULL; + DECLARE c CURSOR FOR SELECT j FROM t1 ORDER BY id; + DECLARE CONTINUE HANDLER FOR NOT FOUND SET done= 1; + SELECT j INTO v FROM t1 WHERE id= 1; + SELECT JSON_SET(v, '$.b', 2) AS from_select_into; + SELECT JSON_EXTRACT(j, '$') INTO v FROM t1 WHERE id= 2; + SELECT JSON_ARRAY_APPEND(v, '$', 3) AS from_extract_into; + OPEN c; + read_loop: LOOP + FETCH c INTO v; + IF done THEN + LEAVE read_loop; + END IF; + SELECT JSON_TYPE(v) AS fetched_type, JSON_DEPTH(v) AS fetched_depth; + END LOOP; + CLOSE c; +END $$ +--delimiter ; +CALL p_into(); +DROP PROCEDURE p_into; + +--echo # +--echo # 10. Through a stored function, whose parameter is a variable of the +--echo # frame the call makes and whose result is not one at all. +--echo # +--delimiter $$ +CREATE FUNCTION f_edit(doc LONGTEXT) RETURNS LONGTEXT +BEGIN + RETURN JSON_SET(doc, '$.f', 1); +END $$ +--delimiter ; +SELECT f_edit(JSON_OBJECT('a', 1)) AS called_with_document; +SELECT f_edit('{"a":1}') AS called_with_literal; +SELECT JSON_SET(f_edit(JSON_OBJECT('a', 1)), '$.g', 2) AS over_the_result; +SELECT JSON_ARRAY(f_edit(JSON_OBJECT('a', 1))) AS spliced_result; +SELECT f_edit(j) AS over_a_column FROM t1 ORDER BY id; +DROP FUNCTION f_edit; + +--echo # +--echo # 11. Values the declared type has to change on the way in: a +--echo # character set that can hold every character of the document, and +--echo # one that cannot. +--echo # +--delimiter $$ +CREATE PROCEDURE p_charset() +BEGIN + DECLARE l1 VARCHAR(64) CHARACTER SET latin1 DEFAULT NULL; + DECLARE u VARCHAR(64) CHARACTER SET utf8mb4 DEFAULT NULL; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + SET u= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); + SELECT JSON_SET(u, '$.c', 'y') AS utf8_var; + SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); + SELECT JSON_SET(l1, '$.c', 'y') AS latin1_var; + SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xC3A4); + SELECT HEX(l1) AS latin1_converted; + SET l1= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4 0xE4B8AD); + SELECT HEX(l1) AS latin1_lossy; + SELECT JSON_VALID(l1) AS valid_after_loss; +END $$ +--delimiter ; +CALL p_charset(); +DROP PROCEDURE p_charset; + +--echo # +--echo # 12. A declared type that changes nothing on the way in and still +--echo # leaves something else there. A store into the binary type keeps +--echo # the bytes and calls them by another name, so a document written +--echo # in a wide set arrives as bytes that read as no document at all. +--echo # Once for a conversion written out in the assignment, and once for +--echo # a variable already holding the wide encoding. +--echo # +--delimiter $$ +CREATE PROCEDURE p_binary() +BEGIN + DECLARE b BLOB DEFAULT NULL; + DECLARE w VARCHAR(64) CHARACTER SET ucs2 DEFAULT NULL; + SET b= CONVERT(JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z') + USING ucs2); + SELECT HEX(b) AS binary_converted, JSON_VALID(b) AS valid_converted; + SET w= JSON_SET(_utf8mb4'{"a":"e"}', '$.b', _utf8mb4'z'); + SELECT JSON_VALID(w) AS valid_wide; + SET b= w; + SELECT HEX(b) AS binary_from_wide, JSON_VALID(b) AS valid_from_wide; +END $$ +--delimiter ; +CALL p_binary(); +DROP PROCEDURE p_binary; + +DROP TABLE t1; + +--echo # +--echo # 13. A row variable written from a list of values rather than from +--echo # a row. Section 8 assigns a whole row at once; SELECT INTO and +--echo # FETCH INTO write the members one at a time from a list and stop at +--echo # the first that fails. What the variable holds afterwards is a +--echo # mixture of three things: the members before the failure carry what +--echo # this assignment wrote, the one it stopped at is left holding +--echo # nothing, and the ones after it still carry what the assignment +--echo # before left. Anything said about the last two belongs to an +--echo # assignment that did not finish, and goes with it. +--echo # +--echo # A variable is never typed as a document, so a value read out of +--echo # one is quoted into a new document rather than spliced into it, in +--echo # both states - which is what the whole-assignment case is printed +--echo # for. What the answers show is that reading the mixture says the +--echo # same things about it as reading a variable nothing went wrong +--echo # with. +--echo # +SET @@sql_mode='STRICT_ALL_TABLES'; + +--delimiter $$ +CREATE PROCEDURE p_row_into() +BEGIN + DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION + SELECT r.a AS written_by_this_assignment, + r.b AS the_member_it_stopped_at, + r.c AS left_by_the_assignment_before, + JSON_VALID(r.c) AS still_a_document, + JSON_TYPE(r.c) AS ty, + JSON_ARRAY(r.c) AS quoted; + SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; + SELECT r.a AS a, r.b AS b, r.c AS c, + JSON_TYPE(r.c) AS ty, + JSON_ARRAY(r.c) AS quoted_after_a_whole_assignment; + SELECT JSON_OBJECT('second', 2), 'not a number', JSON_OBJECT('fourth', 4) + INTO r; + SELECT 'the handler ran and the call went on' AS resumed; +END $$ + +CREATE PROCEDURE p_row_fetch() +BEGIN + DECLARE r ROW(a VARCHAR(64), b INT, c VARCHAR(64)); + DECLARE cur CURSOR FOR + SELECT JSON_OBJECT('fifth', 5) AS a, 'not a number' AS b, + JSON_OBJECT('sixth', 6) AS c; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION + SELECT r.a AS written_by_this_fetch, + r.b AS the_member_it_stopped_at, + r.c AS left_by_the_assignment_before, + JSON_VALID(r.c) AS still_a_document, + JSON_ARRAY(r.c) AS quoted; + SELECT JSON_OBJECT('first', 1), 1, JSON_OBJECT('third', 3) INTO r; + OPEN cur; + FETCH cur INTO r; + CLOSE cur; +END $$ +--delimiter ; + +CALL p_row_into(); +CALL p_row_fetch(); + +DROP PROCEDURE p_row_into; +DROP PROCEDURE p_row_fetch; +SET @@sql_mode=DEFAULT; diff --git a/mysql-test/main/func_json_splice_depth.result b/mysql-test/main/func_json_splice_depth.result new file mode 100644 index 0000000000000..2ad7c15a3bb6d --- /dev/null +++ b/mysql-test/main/func_json_splice_depth.result @@ -0,0 +1,204 @@ +SET @d29= CONCAT(REPEAT('{"a":', 29), '1', REPEAT('}', 29)); +SET @d30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SET @p30= CONCAT('$', REPEAT('.a', 30)); +# +# 1. Two structures put 30 deep make 32 and do not read back; +# the same two put 29 deep make 31 and do +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, +JSON_EXTRACT('{"b":{"c":1}}', '$')) AS too_deep; +too_deep +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_replace' at position 15 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 187 +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, +JSON_EXTRACT('{"b":{"c":1}}', '$')) AS in_limit; +in_limit +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"b": {"c": 1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 2. A value of nothing but brackets, where how long it is says +# exactly how deep it goes and there is no room to be wrong by +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, +JSON_EXTRACT('[[1]]', '$')) AS br_limit; +br_limit +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": [[1]]}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, +JSON_EXTRACT('[[1]]', '$')) AS br_deep; +br_deep +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_replace' at position 5 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 182 +# +# 3. A scalar sits at whatever depth it is put and adds none of +# its own +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, +JSON_EXTRACT('[7]', '$[0]')) AS scalar_deep; +scalar_deep +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": 7}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 4. The same through a document answering is_valid false, which is +# read back in full and so is where the limit has always been +# noticed +# +SELECT JSON_REPLACE(@d30, @p30, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_deep; +raw_deep +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_replace' at position 157 +SELECT JSON_REPLACE(@d29, @p29, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_limit; +raw_limit +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"b": {"c": 1}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 5. A value written out by hand is a string rather than a +# document, so it goes in quoted and takes one level whatever +# is written in it +# +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, '{"b":{"c":1}}') AS quoted; +quoted +{"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": {"a": "{\"b\":{\"c\":1}}"}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +# +# 6. A function that BUILDS a document puts the value one level +# inside the array or object it is making +# +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d30a= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_ARRAY(JSON_EXTRACT(@d31, '$')) AS built_deep; +built_deep +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 63 +SELECT JSON_ARRAY(JSON_EXTRACT(@d30a, '$')) AS built_limit; +built_limit +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +# +# 7. A chain padded until its length says nothing about it - each +# level here is fourteen characters, so half the length is +# seven times the limit and only a count can tell one of these +# from the other +# +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), +'1', REPEAT('}', 31)); +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), +'1', REPEAT('}', 30)); +SELECT JSON_ARRAY(JSON_EXTRACT(@k31, '$')) AS padded_deep; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k31, '$'))) AS padded_deep_depth; +padded_deep_depth +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 +SELECT JSON_ARRAY(JSON_EXTRACT(@k30, '$')) AS padded_limit; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_limit_depth; +padded_limit_depth +32 +# +# 8. The same carried one step further out, where the count is all +# there is: the array of case 7 is longer again and is what the +# outer one is asked to take +# +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_two; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$')))) +AS padded_two_depth; +padded_two_depth +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 483 +# +# 9. A document that was EDITED is as deep as it was or as deep as +# what went into it, whichever is more, and says so without +# being read again +# +SET @k29= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 29), +'1', REPEAT('}', 29)); +SET @k28= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 28), +'1', REPEAT('}', 28)); +SET @kp29= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 29)); +SET @kp28= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 28)); +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, +JSON_EXTRACT('{"b":{"c":1}}', '$'))) +AS edited_limit; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, +JSON_EXTRACT('{"b":{"c":1}}', '$')))) +AS edited_limit_depth; +edited_limit_depth +32 +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, +JSON_EXTRACT('{"b":{"c":1}}', '$'))) +AS edited_deep; +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, +JSON_EXTRACT('{"b":{"c":1}}', '$')))) +AS edited_deep_depth; +edited_deep_depth +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 479 +# +# 10. Wrapping, where the new array is a structure the document +# being read never had. What was found is put inside it and +# the value goes in beside them both, so the value sits one +# level below where the path reached. +# +# A value written out as a string has nothing inside it, so +# where it sits is the whole of how deep the answer goes and +# there is nothing left to read to find that out. +# +SET @w31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @w30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @wp31= CONCAT('$', REPEAT('[0]', 31)); +SET @wp30= CONCAT('$', REPEAT('[0]', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, 2) AS wrap_deep; +wrap_deep +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +SELECT JSON_ARRAY_APPEND(@w31, @wp31, 2) AS wrap_deep_read_back; +wrap_deep_read_back +NULL +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +# +# One level shallower both answer, which is what says the two +# above are refused for their depth and not for their shape. +# +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w30, '$'), @wp30, 2) AS wrap_limit; +wrap_limit +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1, 2]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +SELECT JSON_ARRAY_APPEND(@w30, @wp30, 2) AS wrap_limit_read_back; +wrap_limit_read_back +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1, 2]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] +# +# A value that is a document of its own is read, and reaches +# the same refusal through what the reading found rather than +# through where it was put. +# +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, +JSON_EXTRACT('2', '$')) AS wrap_deep_typed; +wrap_deep_typed +NULL +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_array_append' at position 1 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 32 +# +# The functions that wrap while INSERTING name the place they +# are inserting at, so reaching this far down takes a path one +# step longer than a path is allowed to be, and the path is +# what refuses them. Appending names the value instead and +# needs no step for the array it adds, which is why it is the +# one that gets here. +# +SELECT JSON_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) +AS insert_wrap_deep; +insert_wrap_deep +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_insert' at position 96 +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) +AS ainsert_wrap_deep; +ainsert_wrap_deep +NULL +Warnings: +Warning 4043 Limit of 32 on JSON path depth is reached in argument 2 to function 'json_array_insert' at position 96 diff --git a/mysql-test/main/func_json_splice_depth.test b/mysql-test/main/func_json_splice_depth.test new file mode 100644 index 0000000000000..1c236ec19fffe --- /dev/null +++ b/mysql-test/main/func_json_splice_depth.test @@ -0,0 +1,182 @@ +# +# How deep a value put into a document is allowed to end up. +# +# A document can hold 32 structures inside one another and no more. +# A value spliced into one is written where the path reached, so how +# deep it ends up is how deep the path went plus how deep the value +# goes on its own, and it is the total the limit is about. +# +# The value's own depth is found by reading it. A value that something +# else has already read and attested to is not read again, so the +# reading that would have measured it is not done. Two things stand in +# for it, and the smaller is taken. A value only nests as deeply as it +# is long, which is exact for a value of nothing but brackets and +# useless for a long shallow one. And the function that wrote the value +# may have counted the depth as it wrote, which is exact whatever the +# length. +# +# The cases below are the ones that tell those apart, and each is set +# where being wrong by one would show: a value of nothing but brackets +# is as deep as half its length; a scalar is no deep at all however long +# the document holding it; and a long padded chain is beyond anything +# its length can say, so only a count admits it. +# +# The answers are the ones the server has always given. +# + +SET @d29= CONCAT(REPEAT('{"a":', 29), '1', REPEAT('}', 29)); +SET @d30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SET @p30= CONCAT('$', REPEAT('.a', 30)); + +--echo # +--echo # 1. Two structures put 30 deep make 32 and do not read back; +--echo # the same two put 29 deep make 31 and do +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, + JSON_EXTRACT('{"b":{"c":1}}', '$')) AS too_deep; +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, + JSON_EXTRACT('{"b":{"c":1}}', '$')) AS in_limit; + +--echo # +--echo # 2. A value of nothing but brackets, where how long it is says +--echo # exactly how deep it goes and there is no room to be wrong by +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d29, '$'), @p29, + JSON_EXTRACT('[[1]]', '$')) AS br_limit; +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, + JSON_EXTRACT('[[1]]', '$')) AS br_deep; + +--echo # +--echo # 3. A scalar sits at whatever depth it is put and adds none of +--echo # its own +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, + JSON_EXTRACT('[7]', '$[0]')) AS scalar_deep; + +--echo # +--echo # 4. The same through a document answering is_valid false, which is +--echo # read back in full and so is where the limit has always been +--echo # noticed +--echo # +SELECT JSON_REPLACE(@d30, @p30, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_deep; +SELECT JSON_REPLACE(@d29, @p29, JSON_EXTRACT('{"b":{"c":1}}', '$')) AS raw_limit; + +--echo # +--echo # 5. A value written out by hand is a string rather than a +--echo # document, so it goes in quoted and takes one level whatever +--echo # is written in it +--echo # +SELECT JSON_REPLACE(JSON_EXTRACT(@d30, '$'), @p30, '{"b":{"c":1}}') AS quoted; + +--echo # +--echo # 6. A function that BUILDS a document puts the value one level +--echo # inside the array or object it is making +--echo # +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @d30a= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_ARRAY(JSON_EXTRACT(@d31, '$')) AS built_deep; +SELECT JSON_ARRAY(JSON_EXTRACT(@d30a, '$')) AS built_limit; + +--echo # +--echo # 7. A chain padded until its length says nothing about it - each +--echo # level here is fourteen characters, so half the length is +--echo # seven times the limit and only a count can tell one of these +--echo # from the other +--echo # +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), + '1', REPEAT('}', 31)); +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), + '1', REPEAT('}', 30)); +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(@k31, '$')) AS padded_deep; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k31, '$'))) AS padded_deep_depth; +--disable_result_log +SELECT JSON_ARRAY(JSON_EXTRACT(@k30, '$')) AS padded_limit; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_limit_depth; + +--echo # +--echo # 8. The same carried one step further out, where the count is all +--echo # there is: the array of case 7 is longer again and is what the +--echo # outer one is asked to take +--echo # +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$'))) AS padded_two; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_ARRAY(JSON_EXTRACT(@k30, '$')))) + AS padded_two_depth; + +--echo # +--echo # 9. A document that was EDITED is as deep as it was or as deep as +--echo # what went into it, whichever is more, and says so without +--echo # being read again +--echo # +SET @k29= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 29), + '1', REPEAT('}', 29)); +SET @k28= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 28), + '1', REPEAT('}', 28)); +SET @kp29= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 29)); +SET @kp28= CONCAT('$', REPEAT(CONCAT('.', REPEAT('k', 10)), 28)); +--disable_result_log +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, + JSON_EXTRACT('{"b":{"c":1}}', '$'))) + AS edited_limit; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k28, '$'), @kp28, + JSON_EXTRACT('{"b":{"c":1}}', '$')))) + AS edited_limit_depth; +--disable_result_log +SELECT JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, + JSON_EXTRACT('{"b":{"c":1}}', '$'))) + AS edited_deep; +--enable_result_log +SELECT JSON_DEPTH(JSON_ARRAY(JSON_REPLACE(JSON_EXTRACT(@k29, '$'), @kp29, + JSON_EXTRACT('{"b":{"c":1}}', '$')))) + AS edited_deep_depth; + +--echo # +--echo # 10. Wrapping, where the new array is a structure the document +--echo # being read never had. What was found is put inside it and +--echo # the value goes in beside them both, so the value sits one +--echo # level below where the path reached. +--echo # +--echo # A value written out as a string has nothing inside it, so +--echo # where it sits is the whole of how deep the answer goes and +--echo # there is nothing left to read to find that out. +--echo # +SET @w31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SET @w30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SET @wp31= CONCAT('$', REPEAT('[0]', 31)); +SET @wp30= CONCAT('$', REPEAT('[0]', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, 2) AS wrap_deep; +SELECT JSON_ARRAY_APPEND(@w31, @wp31, 2) AS wrap_deep_read_back; + +--echo # +--echo # One level shallower both answer, which is what says the two +--echo # above are refused for their depth and not for their shape. +--echo # +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w30, '$'), @wp30, 2) AS wrap_limit; +SELECT JSON_ARRAY_APPEND(@w30, @wp30, 2) AS wrap_limit_read_back; + +--echo # +--echo # A value that is a document of its own is read, and reaches +--echo # the same refusal through what the reading found rather than +--echo # through where it was put. +--echo # +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@w31, '$'), @wp31, + JSON_EXTRACT('2', '$')) AS wrap_deep_typed; + +--echo # +--echo # The functions that wrap while INSERTING name the place they +--echo # are inserting at, so reaching this far down takes a path one +--echo # step longer than a path is allowed to be, and the path is +--echo # what refuses them. Appending names the value instead and +--echo # needs no step for the array it adds, which is why it is the +--echo # one that gets here. +--echo # +SELECT JSON_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) + AS insert_wrap_deep; +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT(@w31, '$'), CONCAT(@wp31, '[1]'), 2) + AS ainsert_wrap_deep; diff --git a/mysql-test/main/func_json_splice_oom.result b/mysql-test/main/func_json_splice_oom.result new file mode 100644 index 0000000000000..6fa48c9042549 --- /dev/null +++ b/mysql-test/main/func_json_splice_oom.result @@ -0,0 +1,240 @@ +# +# No room to grow the buffer while a value is being written out +# into the document it is joining. +# +# A value that is a document but is not written the loose way is +# written out again as it is spliced, whenever the document it +# joins is loose and would stop being so otherwise. That writing +# grows the buffer the same way every other writing here does, +# and can fail the same way. +# +# The failure has to be reachable from this writing and not only +# from the older ones. A writing that no injection can fail is a +# writing whose handling of failure nobody has ever run, and this +# file is what keeps that from happening quietly. +# +SET NAMES utf8mb4; +# +# 1. A value that is a document, and is not loose +# +# Read out of a document written the compact way, so it is +# formatted that way itself and has to be written out again when +# it joins one that is loose. +# +SET @compact = CONCAT('[', REPEAT('1,', 999), '1]'); +SET @small = '[1,2]'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) +AS spliced; +spliced +{"a": 1, "d": [1, 2]} +SELECT LENGTH(@compact) AS compact_len, +LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; +compact_len grew_by +2001 999 +# +# 2. The same splice with no room to grow +# +# This must not answer. If it does, the injection is not +# reaching the writing above and this file is testing nothing. +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@compact, '$')) +AS spliced; +ERROR HY000: Out of memory (Needed 1288 bytes) +SET SESSION debug_dbug = DEFAULT; +# +# 3. The same statement once the room is there again +# +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', +JSON_QUERY(@compact, '$')), '')) +AS parses; +parses +1 +# +# 4. The other functions that splice a value the same way +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) +AS st; +SELECT JSON_REPLACE(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) +AS rep; +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', JSON_QUERY(@compact, '$')) +AS app; +SELECT JSON_ARRAY_INSERT(JSON_ARRAY(1), '$[0]', JSON_QUERY(@compact, '$')) +AS ari; +SET SESSION debug_dbug = DEFAULT; +# and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(JSON_OBJECT('a', 1), '$.a', +JSON_QUERY(@compact, '$')), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', +JSON_QUERY(@compact, '$')), '')) AS parses; +parses +1 +# +# 5. A value short enough that the room asked for is enough, +# which must be unaffected +# +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) +AS spliced; +spliced +{"a": 1, "d": [1, 2]} +SET SESSION debug_dbug = DEFAULT; +# +# 6. Several rows, so the failure is met more than once +# +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', +JSON_QUERY(j, '$')), '')) AS parses +FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', +JSON_QUERY(j, '$')), '')) AS parses +FROM t1 ORDER BY id; +id parses +1 1 +2 1 +3 1 +DROP TABLE t1; +# +# 7. No room to pass a document that was composed in place +# +# Where the item behind every document a function was given attests +# is_valid and is_nice, what it composed is the answer and is +# returned rather than read back. Returning it is a copy, and +# a copy is a write that can run short of room like any other. +# +# Every argument is built by a producer that attests to what +# it passes. A string literal attests is_valid false for +# its document, so a statement built out of literals +# never composes an answer at all and would say nothing here. +# +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) AS whole; +whole +{"a": [1, 2]} +SET SESSION debug_dbug = '+d,json_return_out_of_memory'; +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) +AS appended; +appended +NULL +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) +AS inserted; +inserted +NULL +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +added +NULL +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +removed +NULL +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS merged; +merged +NULL +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS patched; +patched +NULL +SET SESSION debug_dbug = DEFAULT; +# and the same six once the room is there again, which is what +# says the refusals above are about the copy and not about the +# statements +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) +AS appended; +appended +{"a": [1, 2]} +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) +AS inserted; +inserted +{"a": [2, 1]} +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +added +{"a": 1, "b": 2} +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +removed +{"a": 1} +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS merged; +merged +{"a": 1, "b": 2} +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) +AS patched; +patched +{"a": 1, "b": 2} +# +# 8. No room while the values a path asked for are written out +# +# A reading function writes the values it found into the answer +# itself, with a comma between them where more than one was +# asked for, rather than composing a document and reading it +# back. Both writes are its own and both can run short. +# +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS whole, +JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS whole_one; +whole whole_one +[1, 2] 1 +SET SESSION debug_dbug = '+d,json_extract_comma_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_comma; +no_comma +NULL +# one value alone needs no comma, so this one is untouched +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS one_value; +one_value +1 +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_extract_scalar_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS no_value; +no_value +NULL +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_value_of_several; +no_value_of_several +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 9. No room for a key the document did not have +# +# A path naming a key that is not there has the key written +# into the document, which is a write the composing does +# rather than one a reading back would have done. +# +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS whole; +whole +{"a": 1, "b": 2} +SET SESSION debug_dbug = '+d,json_insert_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_key; +no_key +NULL +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_key_set; +no_key_set +NULL +SET SESSION debug_dbug = DEFAULT; +# +# 10. The colon that joins a key to its value, and the key a path +# names that the document did not have +# +# Both are written in the formatting this commit settled, so +# both are failed here rather than where the key itself is. +# +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole, +JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole_added; +whole whole_added +{"k": 1, "l": 2} {"a": 1, "b": 2} +SET SESSION debug_dbug = '+d,json_keyname_colon_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_colon; +no_colon +NULL +SET SESSION debug_dbug = DEFAULT; +SET SESSION debug_dbug = '+d,json_insert_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_room_added_key; +no_room_added_key +NULL +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_room_added_key_set; +no_room_added_key_set +NULL +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_splice_oom.test b/mysql-test/main/func_json_splice_oom.test new file mode 100644 index 0000000000000..a72d7d0a1e727 --- /dev/null +++ b/mysql-test/main/func_json_splice_oom.test @@ -0,0 +1,200 @@ +--source include/have_debug.inc + +--echo # +--echo # No room to grow the buffer while a value is being written out +--echo # into the document it is joining. +--echo # +--echo # A value that is a document but is not written the loose way is +--echo # written out again as it is spliced, whenever the document it +--echo # joins is loose and would stop being so otherwise. That writing +--echo # grows the buffer the same way every other writing here does, +--echo # and can fail the same way. +--echo # +--echo # The failure has to be reachable from this writing and not only +--echo # from the older ones. A writing that no injection can fail is a +--echo # writing whose handling of failure nobody has ever run, and this +--echo # file is what keeps that from happening quietly. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A value that is a document, and is not loose +--echo # +--echo # Read out of a document written the compact way, so it is +--echo # formatted that way itself and has to be written out again when +--echo # it joins one that is loose. +--echo # +SET @compact = CONCAT('[', REPEAT('1,', 999), '1]'); +SET @small = '[1,2]'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) + AS spliced; +SELECT LENGTH(@compact) AS compact_len, + LENGTH(JSON_LOOSE(@compact)) - LENGTH(@compact) AS grew_by; + +--echo # +--echo # 2. The same splice with no room to grow +--echo # +--echo # This must not answer. If it does, the injection is not +--echo # reaching the writing above and this file is testing nothing. +--echo # +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 5 +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@compact, '$')) + AS spliced; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 3. The same statement once the room is there again +--echo # +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', + JSON_QUERY(@compact, '$')), '')) + AS parses; + +--echo # +--echo # 4. The other functions that splice a value the same way +--echo # +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) + AS st; +--error 0,5 +SELECT JSON_REPLACE(JSON_OBJECT('a', 1), '$.a', JSON_QUERY(@compact, '$')) + AS rep; +--error 0,5 +SELECT JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', JSON_QUERY(@compact, '$')) + AS app; +--error 0,5 +SELECT JSON_ARRAY_INSERT(JSON_ARRAY(1), '$[0]', JSON_QUERY(@compact, '$')) + AS ari; +SET SESSION debug_dbug = DEFAULT; + +--echo # and once the room is there again +SELECT JSON_VALID(CONCAT(JSON_SET(JSON_OBJECT('a', 1), '$.a', + JSON_QUERY(@compact, '$')), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_ARRAY_APPEND(JSON_ARRAY(1), '$', + JSON_QUERY(@compact, '$')), '')) AS parses; + +--echo # +--echo # 5. A value short enough that the room asked for is enough, +--echo # which must be unaffected +--echo # +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.d', JSON_QUERY(@small, '$')) + AS spliced; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 6. Several rows, so the failure is met more than once +--echo # +CREATE TABLE t1 (id INT, j LONGTEXT); +INSERT INTO t1 VALUES (1, '[1,2]'); +INSERT INTO t1 VALUES (2, CONCAT('[', REPEAT('1,', 49), '1]')); +INSERT INTO t1 VALUES (3, CONCAT('[', REPEAT('2,', 49), '2]')); +SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; +--error 0,5 +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', + JSON_QUERY(j, '$')), '')) AS parses + FROM t1 ORDER BY id; +SET SESSION debug_dbug = DEFAULT; +SELECT id, JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('a', 1), '$.d', + JSON_QUERY(j, '$')), '')) AS parses + FROM t1 ORDER BY id; +DROP TABLE t1; + +--echo # +--echo # 7. No room to pass a document that was composed in place +--echo # +--echo # Where the item behind every document a function was given attests +--echo # is_valid and is_nice, what it composed is the answer and is +--echo # returned rather than read back. Returning it is a copy, and +--echo # a copy is a write that can run short of room like any other. +--echo # +--echo # Every argument is built by a producer that attests to what +--echo # it passes. A string literal attests is_valid false for +--echo # its document, so a statement built out of literals +--echo # never composes an answer at all and would say nothing here. +--echo # +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) AS whole; + +SET SESSION debug_dbug = '+d,json_return_out_of_memory'; +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) + AS appended; +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) + AS inserted; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS merged; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS patched; +SET SESSION debug_dbug = DEFAULT; + +--echo # and the same six once the room is there again, which is what +--echo # says the refusals above are about the copy and not about the +--echo # statements +SELECT JSON_ARRAY_APPEND(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a', 2) + AS appended; +SELECT JSON_ARRAY_INSERT(JSON_OBJECT('a', JSON_ARRAY(1)), '$.a[0]', 2) + AS inserted; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS added; +SELECT JSON_REMOVE(JSON_OBJECT('a', 1, 'b', 2), '$.b') AS removed; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS merged; +SELECT JSON_MERGE_PATCH(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) + AS patched; + +--echo # +--echo # 8. No room while the values a path asked for are written out +--echo # +--echo # A reading function writes the values it found into the answer +--echo # itself, with a comma between them where more than one was +--echo # asked for, rather than composing a document and reading it +--echo # back. Both writes are its own and both can run short. +--echo # +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS whole, + JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS whole_one; + +SET SESSION debug_dbug = '+d,json_extract_comma_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_comma; +--echo # one value alone needs no comma, so this one is untouched +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS one_value; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_extract_scalar_out_of_memory'; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a') AS no_value; +SELECT JSON_EXTRACT('{"a":1,"b":2}', '$.a', '$.b') AS no_value_of_several; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 9. No room for a key the document did not have +--echo # +--echo # A path naming a key that is not there has the key written +--echo # into the document, which is a write the composing does +--echo # rather than one a reading back would have done. +--echo # +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS whole; + +SET SESSION debug_dbug = '+d,json_insert_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_key; +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_key_set; +SET SESSION debug_dbug = DEFAULT; + +--echo # +--echo # 10. The colon that joins a key to its value, and the key a path +--echo # names that the document did not have +--echo # +--echo # Both are written in the formatting this commit settled, so +--echo # both are failed here rather than where the key itself is. +--echo # +SELECT JSON_OBJECT('k', 1, 'l', 2) AS whole, + JSON_INSERT(JSON_OBJECT('a', 1), _latin1'$.b', 2) AS whole_added; + +SET SESSION debug_dbug = '+d,json_keyname_colon_out_of_memory'; +SELECT JSON_OBJECT('k', 1, 'l', 2) AS no_colon; +SET SESSION debug_dbug = DEFAULT; + +SET SESSION debug_dbug = '+d,json_insert_path_key_out_of_memory'; +SELECT JSON_INSERT(JSON_OBJECT('a', 1), '$.b', 2) AS no_room_added_key; +SELECT JSON_SET(JSON_OBJECT('a', 1), '$.c', 3) AS no_room_added_key_set; +SET SESSION debug_dbug = DEFAULT; diff --git a/mysql-test/main/func_json_table.result b/mysql-test/main/func_json_table.result new file mode 100644 index 0000000000000..5cec1551fd48d --- /dev/null +++ b/mysql-test/main/func_json_table.result @@ -0,0 +1,87 @@ +# +# Behavioral baseline: JSON_TABLE. +# +# JSON_TABLE parses its document itself and presents the pieces as +# columns of a table that only exists for the duration of the query. +# This test records what those columns are treated as when they are +# fed back into JSON functions, what happens when the document comes +# from a column whose stored bytes are not valid, and what a DEFAULT +# clause is allowed to put into a column declared JSON. +# +SET NAMES utf8mb4; +# +# 1. A column declared JSON, consumed by JSON functions. Compare the +# embedding decision here with the one a JSON table column gets in +# func_json_columns. +# +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('[{"a": {"x": 1}}, {"a":{"y":2}}]'); +# the fragment keeps the spacing it had in the input document +SELECT jt.frag AS v, HEX(jt.frag) AS h +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +v h +{"x": 1} 7B2278223A2020317D +{"y":2} 7B2279223A327D +# fed to a constructor it is quoted, not embedded +SELECT JSON_ARRAY(jt.frag) AS embedded, JSON_VALID(JSON_ARRAY(jt.frag)) AS ok +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +embedded ok +["{\"x\": 1}"] 1 +["{\"y\":2}"] 1 +SELECT JSON_OBJECT('k', jt.frag) AS v +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +v +{"k": "{\"x\": 1}"} +{"k": "{\"y\":2}"} +# but a mutator and JSON_TYPE both take it as a document +SELECT JSON_SET(jt.frag, '$.z', 3) AS v, JSON_TYPE(jt.frag) AS t +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +v t +{"x": 1, "z": 3} OBJECT +{"y": 2, "z": 3} OBJECT +# a column declared VARCHAR over the same path +SELECT JSON_ARRAY(jt.txt) AS quoted +FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (txt VARCHAR(50) PATH '$.a' NULL ON ERROR)) jt; +quoted +[null] +[null] +DROP TABLE t1; +# +# 2. The document coming from a column whose stored bytes no longer +# satisfy its check constraint. +# +CREATE TABLE tp (id INT PRIMARY KEY, j JSON); +INSERT INTO tp VALUES (1, '[{"a":1},{"a":2}]'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES (2, '[{"a":1},'), (3, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))); +SET SESSION check_constraint_checks = ON; +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +WHERE id = 1; +id a +1 1 +1 2 +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +WHERE id = 2; +ERROR HY000: Unexpected end of JSON text in argument 1 to function 'JSON_TABLE' +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +WHERE id = 3; +ERROR HY000: Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'JSON_TABLE' at position 32 +# one bad row aborts the whole statement, it is not skipped +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt +ORDER BY id; +ERROR HY000: Unexpected end of JSON text in argument 1 to function 'JSON_TABLE' +DROP TABLE tp; +# +# 3. A DEFAULT clause supplying a value for a column declared JSON. +# The default is stored as given, without being checked. +# +SELECT jt.frag AS v, JSON_ARRAY(jt.frag) AS embedded, +JSON_VALID(JSON_ARRAY(jt.frag)) AS still_valid +FROM JSON_TABLE('[{"b":1}]', '$[*]' + COLUMNS (frag JSON PATH '$.a' DEFAULT '{"broken":' ON EMPTY)) jt; +v embedded still_valid +{"broken": ["{\"broken\":"] 1 +SELECT * FROM JSON_TABLE('[{"a":1},{"a":2}]', '$' + COLUMNS (frag JSON PATH '$[*].a' DEFAULT 'not json' ON ERROR)) jt; +frag +not json diff --git a/mysql-test/main/func_json_table.test b/mysql-test/main/func_json_table.test new file mode 100644 index 0000000000000..54ddf135ffb3f --- /dev/null +++ b/mysql-test/main/func_json_table.test @@ -0,0 +1,74 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # Behavioral baseline: JSON_TABLE. +--echo # +--echo # JSON_TABLE parses its document itself and presents the pieces as +--echo # columns of a table that only exists for the duration of the query. +--echo # This test records what those columns are treated as when they are +--echo # fed back into JSON functions, what happens when the document comes +--echo # from a column whose stored bytes are not valid, and what a DEFAULT +--echo # clause is allowed to put into a column declared JSON. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. A column declared JSON, consumed by JSON functions. Compare the +--echo # embedding decision here with the one a JSON table column gets in +--echo # func_json_columns. +--echo # + +CREATE TABLE t1 (j JSON); +INSERT INTO t1 VALUES ('[{"a": {"x": 1}}, {"a":{"y":2}}]'); +--echo # the fragment keeps the spacing it had in the input document +SELECT jt.frag AS v, HEX(jt.frag) AS h + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +--echo # fed to a constructor it is quoted, not embedded +SELECT JSON_ARRAY(jt.frag) AS embedded, JSON_VALID(JSON_ARRAY(jt.frag)) AS ok + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +SELECT JSON_OBJECT('k', jt.frag) AS v + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +--echo # but a mutator and JSON_TYPE both take it as a document +SELECT JSON_SET(jt.frag, '$.z', 3) AS v, JSON_TYPE(jt.frag) AS t + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (frag JSON PATH '$.a')) jt; +--echo # a column declared VARCHAR over the same path +SELECT JSON_ARRAY(jt.txt) AS quoted + FROM t1, JSON_TABLE(t1.j, '$[*]' COLUMNS (txt VARCHAR(50) PATH '$.a' NULL ON ERROR)) jt; +DROP TABLE t1; + +--echo # +--echo # 2. The document coming from a column whose stored bytes no longer +--echo # satisfy its check constraint. +--echo # + +CREATE TABLE tp (id INT PRIMARY KEY, j JSON); +INSERT INTO tp VALUES (1, '[{"a":1},{"a":2}]'); +SET SESSION check_constraint_checks = OFF; +INSERT INTO tp VALUES (2, '[{"a":1},'), (3, CONCAT(REPEAT('[',32),'1',REPEAT(']',32))); +SET SESSION check_constraint_checks = ON; +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + WHERE id = 1; +--error ER_JSON_EOS +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + WHERE id = 2; +--error ER_JSON_DEPTH +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + WHERE id = 3; +--echo # one bad row aborts the whole statement, it is not skipped +--error ER_JSON_EOS +SELECT id, jt.a FROM tp, JSON_TABLE(tp.j, '$[*]' COLUMNS (a INT PATH '$.a')) jt + ORDER BY id; +DROP TABLE tp; + +--echo # +--echo # 3. A DEFAULT clause supplying a value for a column declared JSON. +--echo # The default is stored as given, without being checked. +--echo # + +SELECT jt.frag AS v, JSON_ARRAY(jt.frag) AS embedded, + JSON_VALID(JSON_ARRAY(jt.frag)) AS still_valid + FROM JSON_TABLE('[{"b":1}]', '$[*]' + COLUMNS (frag JSON PATH '$.a' DEFAULT '{"broken":' ON EMPTY)) jt; +SELECT * FROM JSON_TABLE('[{"a":1},{"a":2}]', '$' + COLUMNS (frag JSON PATH '$[*].a' DEFAULT 'not json' ON ERROR)) jt; diff --git a/mysql-test/main/func_json_tmp_pushdown.result b/mysql-test/main/func_json_tmp_pushdown.result new file mode 100644 index 0000000000000..c2c567b5be2b5 --- /dev/null +++ b/mysql-test/main/func_json_tmp_pushdown.result @@ -0,0 +1,50 @@ +# +# A temporary table an engine fills carries no attestations about +# its values. +# +# A query pushed down whole borrows the server's own temporary table +# for its columns and then has the engine write the rows into it, so +# nothing a column was made from ever sees what arrives in it. The +# table drops its attestations where it is filled, and a debug +# build asserts at the other end - where the engine's rows are read +# - that something on the way in did drop them. +# +# Nothing here is a document. The engine below gathers a sum, and a +# sum is not one; what is reached is that dropping itself, which is +# what is being asked about. The results are ordinary results and +# are here to stay ordinary. +# +# Each case is explained before it is run. An engine that stopped +# taking one of these queries would leave the case running down the +# ordinary path and passing without going near what it is here for, +# and the EXPLAIN is what says it did not. +# +# +# 1. A group gathered by the engine and sent straight out. +# +EXPLAIN SELECT SUM(seq) FROM seq_1_to_10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +SELECT SUM(seq) FROM seq_1_to_10; +SUM(seq) +55 +# +# 2. The same held in a temporary table on the way, which is the +# other half of that execution and the half that writes rows. +# +EXPLAIN SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; +SUM(seq) COUNT(*) +5050 100 +# +# 3. Two gatherings ordered, so the rows are held and then read +# back out of the table they were held in. +# +EXPLAIN SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY +SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; +SUM(seq) COUNT(seq) +15 5 diff --git a/mysql-test/main/func_json_tmp_pushdown.test b/mysql-test/main/func_json_tmp_pushdown.test new file mode 100644 index 0000000000000..cbd203215b0bb --- /dev/null +++ b/mysql-test/main/func_json_tmp_pushdown.test @@ -0,0 +1,43 @@ +--source include/have_sequence.inc + +--echo # +--echo # A temporary table an engine fills carries no attestations about +--echo # its values. +--echo # +--echo # A query pushed down whole borrows the server's own temporary table +--echo # for its columns and then has the engine write the rows into it, so +--echo # nothing a column was made from ever sees what arrives in it. The +--echo # table drops its attestations where it is filled, and a debug +--echo # build asserts at the other end - where the engine's rows are read +--echo # - that something on the way in did drop them. +--echo # +--echo # Nothing here is a document. The engine below gathers a sum, and a +--echo # sum is not one; what is reached is that dropping itself, which is +--echo # what is being asked about. The results are ordinary results and +--echo # are here to stay ordinary. +--echo # +--echo # Each case is explained before it is run. An engine that stopped +--echo # taking one of these queries would leave the case running down the +--echo # ordinary path and passing without going near what it is here for, +--echo # and the EXPLAIN is what says it did not. +--echo # + +--echo # +--echo # 1. A group gathered by the engine and sent straight out. +--echo # +EXPLAIN SELECT SUM(seq) FROM seq_1_to_10; +SELECT SUM(seq) FROM seq_1_to_10; + +--echo # +--echo # 2. The same held in a temporary table on the way, which is the +--echo # other half of that execution and the half that writes rows. +--echo # +EXPLAIN SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; +SELECT DISTINCT SUM(seq), COUNT(*) FROM seq_1_to_100; + +--echo # +--echo # 3. Two gatherings ordered, so the rows are held and then read +--echo # back out of the table they were held in. +--echo # +EXPLAIN SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; +SELECT SUM(seq), COUNT(seq) FROM seq_1_to_5 ORDER BY 1; diff --git a/mysql-test/main/func_json_tmp_scan_count.result b/mysql-test/main/func_json_tmp_scan_count.result new file mode 100644 index 0000000000000..9d8c5f207b6c5 --- /dev/null +++ b/mysql-test/main/func_json_tmp_scan_count.result @@ -0,0 +1,707 @@ +# +# How many times a value out of the server's own temporary table is +# read to find out whether it is a document. +# +# No query can see the difference - the reading could only have found +# out what was already known - so nothing but Json_scans says whether +# it happened. The counts below are the record of it. A count that +# goes up is a reading that came back. +# +# A debug build reads values back to check what was claimed about +# them, and none of those readings are counted: they are the debug +# build's work and not the server's. A count of 0 below therefore +# means nothing read, not that a reading went uncounted. +# +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +# +# 1. TWO rows out of a base column, spliced into a new document. +# A column says nothing about itself, so each row is read once and +# that reading is not work that can be taken away. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +r +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 2. The same two rows, put through a function that attests to what +# it returns, and spliced without a temporary table in between. +# JSON_EXTRACT reads its input once; the splice reads nothing. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 3. The same, with a temporary table in between. What the function +# answered is what the column was built to hold, so the count is the +# function's own readings and nothing else. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 4. The same shape with a producer that attests for its class. +# The claim holds for the column at the time it is made, so no row +# is read on the way out. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 5. Through a GROUP BY, which writes a temporary table of its own. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1 GROUP BY id, j) d ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 6. A UNION, whose column has one producer per branch. The +# branches are where those producers are all in reach, so what the +# column answers is what every one of them answers, and the count +# is their own readings and nothing else. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT JSON_EXTRACT(j, '$') FROM t1) d; +r +[{"a": 1}] +[{"b": 2}] +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 7. A table the user asked for, which is not the server's own. +# +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_EXTRACT(j, '$') FROM t1; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM tt; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TEMPORARY TABLE tt; +# +# 8. A store that came up short takes back what was said about the +# column, so its rows are read on the way out like anything else. +# +# No function asks for less room than it goes on to use, so nothing +# arrives short of its own accord and the short store is arranged +# for here. The query is the one from 3, so the count below is what +# 3 would have come to with nothing said about its column. +# +SET @old_debug= @@SESSION.debug_dbug; +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +# +# 9. Two temporary tables one after the other. The second is built +# before a row of the first exists, so what the first says then is +# the most it will ever say. Every row put across asks it again, +# and what is left to count is the first table's own readings. +# +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 10. Asking what a materialized value IS, rather than splicing it. +# JSON_TYPE reads its argument whatever anyone says about it: what is +# promised is that the value is a document, not which one. +# +FLUSH STATUS; +SELECT JSON_TYPE(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +OBJECT +OBJECT +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 11. An aggregate gathers its elements in a table of its own and +# reads them back out of a record rather than off the item that put +# them there. The column is what attests to them. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS r FROM t1; +r +[{"a": 1},{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 12. The same, with a producer that attests for its class. The +# column takes that claim, so the elements are not read back and +# nothing is counted here - where 11 pays two readings for the +# values its producer parsed itself. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS r FROM t1; +r +[{"a": 1},{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 13. An aggregate whose arguments are buffered before the group can +# be built - a join with a grouping the rows do not arrive in. The +# argument is put into a table of its own on the way past, and read +# back off that column. +# +# Each group gathers the same row more than once, so what is +# gathered reads the same whichever order the rows arrive in - an +# order a grouping does not fix and this file must not depend on. +# +CREATE TABLE t13 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) +CHARSET utf8mb4; +INSERT INTO t13 VALUES (1, 1, '{"a":1}'), (2, 2, '{"b":2}'); +CREATE TABLE t13m (g INT); +INSERT INTO t13m VALUES (1), (1), (2), (2); +FLUSH STATUS; +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b +WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +r +{"2":{"b": 2}, "2":{"b": 2}} +{"1":{"a": 1}, "1":{"a": 1}} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 14. The same shape gathered into an array instead. The table the +# aggregate gathers in is built out of the buffer's column rather +# than out of the function that filled it, and asks that column +# again for every element it is handed. +# +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b +WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +r +[{"b": 2},{"b": 2}] +[{"a": 1},{"a": 1}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 15. The chain of 9, with a store into the FIRST table that comes +# up short. That column gives up its answer, and the second has to +# hear about it although it was built long before. The query is +# the one from 9, so the count is what 9 would have come to with +# neither column answering. +# +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +# +# 16. The same chain, with the row that crosses BETWEEN the two +# tables arriving short instead. The first column still answers +# for what it holds; it is the second that stops. +# +SET SESSION debug_dbug='+d,json_tmp_chain_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +# +# 17. The buffered aggregate of 14, with the store into the buffer +# coming up short. The aggregate's own table is filled from that +# buffer field to field rather than off an item, so this is the +# case that says the field-to-field route asks as well. +# +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b +WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +r +[{"b": 2},{"b": 2}] +[{"a": 1},{"a": 1}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +SET SESSION debug_dbug= @old_debug; +# +# 18. A grouping that needs no temporary table of its own still has +# to hold a value across the group it belongs to, and it holds it in +# a buffer rather than in a column. A HAVING or an ORDER BY reads +# what it reads out of that buffer. +# +# A buffer holds one value at a time and is filled right beside the +# item that made it, so what that item says about the value is put +# away together with the value and is about it still when it is read +# back. +# +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 +HAVING JSON_ARRAY(x) <> '[]'; +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 +ORDER BY JSON_ARRAY(x); +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# +# 19. The same with a producer that attests to the value it has +# just made rather than for every value it will ever make. A column +# cannot be told about one of those, being made before any of them +# exists; a buffer is filled after, so what it holds is attested to +# like anything else and nothing is read at all. +# +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t1 +HAVING JSON_ARRAY(x) <> '[]'; +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 0 +# +# 20. A buffer filled from a COLUMN rather than from an item, which +# is the route a value takes when it is wide enough to be kept apart +# from the row it belongs to. What the column says is what the +# buffer puts away. +# +CREATE TABLE t20 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t20 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +FLUSH STATUS; +SELECT MAX(d.id) AS m, d.x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t20) d +HAVING JSON_ARRAY(y) <> '[]'; +m y +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +DROP TABLE t20; +# +# 21. A buffer that could not take what it was handed keeps what it +# held before, which nothing here can attest to. Nothing arrives +# that way of its own accord - it takes a failure to find the room - +# so it is arranged for. The query is the one from 18, so the count +# is what 18 would have come to with the buffer saying nothing. +# +SET SESSION debug_dbug='+d,json_copy_not_kept'; +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 +HAVING JSON_ARRAY(x) <> '[]'; +m x +2 {"a": 1} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +SET SESSION debug_dbug= @old_debug; +# +# 22. The same UNION where both branches attest for their class, +# which is what a column needs: there is no telling which branch a +# row came out of once it is in there, so one branch not attesting +# would be the whole column not attesting. No row is read on the +# way out, and the two counted here are the ones the JSON_EXTRACT +# branch read itself. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT JSON_OBJECT('a', id) FROM t1) d; +r +[{"a": 1}] +[{"b": 2}] +[{"a": 1}] +[{"a": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 23. The same again with a branch that is a column, which says +# nothing about itself and so says nothing for the column it is +# written into either. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT j FROM t1) d; +r +[{"a": 1}] +[{"b": 2}] +[{"a":1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 24. Rows written one list at a time rather than one branch at a +# time, which is the same column with the same one producer per +# row and is asked the same way. +# +FLUSH STATUS; +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('{"b":2}', '$'))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +r +[{"a": 1}] +[{"b":2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# And a list whose producer attests for its class leaves the +# column attesting, exactly as a branch does. The one counted +# here is the JSON_EXTRACT list reading its own value; the +# constructor reads nothing. +# +FLUSH STATUS; +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +r +[{"a": 1}] +[{"b": 2}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 1 +# +# 25. A recursive CTE, whose column is built once the parts that do +# not go round have agreed on a type - the parts that DO go round +# not being reached at all, so nothing there can be asked and the +# ones that were reached do not answer in their place. +# +FLUSH STATUS; +WITH RECURSIVE r AS ( +SELECT 1 AS n, JSON_EXTRACT(j, '$') AS x FROM t1 +UNION ALL +SELECT n + 1, JSON_OBJECT('n', n) FROM r WHERE n < 2) +SELECT JSON_ARRAY(x) AS y FROM r ORDER BY y; +y +[{"a": 1}] +[{"b": 2}] +[{"n": 1}] +[{"n": 1}] +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 10 +# +# 26. A function that EDITS a document rather than building one. +# It composes its answer and then reads the whole of it back, that +# reading being where the spacing gets settled - unless what it was +# given was already spaced that way, which is a second thing such a +# column can say and says here. +# +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 27. The same over a column whose producer returns a document +# formatted some other way - JSON_QUERY copying a sub-document out as +# it stands rather than writing one. Attested, and not written +# for, so the reading back stays. +# +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 28. The formatting carried from one such table into the next, which +# the second column asks the first about at every row it is given. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# And taken back where what the first holds is formatted otherwise. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# 29. The same across the copier that fills a grouping's table, +# which is a different way of putting a row across and is asked the +# same thing. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 +ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 30. And that copier taking it back, for the same reason and by +# asking the same source. +# +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 +ORDER BY id; +r +{"a": 1, "z": 9} +{"b": 2, "z": 9} +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 6 +# +# Everything above is short enough that half its length is inside +# the limit, so nothing there was ever read to find out how deep it +# goes. These are not: at over a hundred characters the length says +# nothing useful, and what the column can say about the depth is the +# only thing left. +# +CREATE TABLE t31 (id INT, j VARCHAR(400) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t31 VALUES +(1, CONCAT('{"a":[1,2],"pad":"', REPEAT('x', 100), '"}')), +(2, CONCAT('{"b":[3,4],"pad":"', REPEAT('y', 100), '"}')); +# +# 31. Two long rows out of a base column. A column of a table the +# user asked for says nothing about how deep its values go any more +# than about anything else, so each row is read. +# +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t31 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 32. The same through a temporary table. The producer counted the +# depth while it wrote and the column kept the figure, so the splice +# has what it needs without the length being asked. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 33. The same carried from one such table into the next, which the +# second column takes from the first at every row it is given. +# +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d1) d2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 34. An aggregate gathering its elements out of such a column, +# which is the only thing left there to ask. +# +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 35. A producer that returns a piece of a document it was given +# rather than one it wrote counted nothing, so the column has no +# figure to keep and the reading stays. The count is what 32 would +# have come to with nothing said. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t31) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# 36. A column written by more than one producer keeps the deepest +# of all its rows, so where every one of them counted, every row +# gets the figure. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 +UNION ALL +SELECT id, JSON_EXTRACT(j, '$.a') FROM t31) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +# +# And one row that nobody counted is the whole column saying +# nothing, the deepest of the rows being unknown once one of them +# is. The same four rows, one pair of them uncounted. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 +UNION ALL +SELECT id, JSON_QUERY(j, '$') FROM t31) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +DROP TABLE t31; +# +# 37. A column filled with the names of a document's keys. Every +# row of one of those is an array of strings whatever the document +# held, and that is so before a row of it exists - so the column is +# built saying it, and the splice on the way out reads nothing. +# +# The names are long here for the same reason as above: the length +# says more levels than a document is allowed, so the depth the +# column keeps is what the splice has to go on. +# +CREATE TABLE t37 (id INT, j TEXT) CHARSET utf8mb4; +INSERT INTO t37 VALUES +(1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}')), +(2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}')); +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 2 +# +# 38. The same query with the store into that column coming up +# short, which is how 8 takes 3's column away. The count is what +# 37 would have come to with nothing said, so the difference +# between the two is the column and nothing else. +# +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +SET SESSION debug_dbug= @old_debug; +DROP TABLE t37; +# +# 39. Branches written in different character sets. A column takes +# one set for all of them, and a branch in another set is stored +# into it converting - except into the binary set, which converts +# nothing at all and keeps every byte under the other set's name. +# Rows that arrived that way are not the documents that were +# written, so the column takes its answer back and every row is +# read on the way out. +# +CREATE TABLE t39 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t39 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t39b (b BLOB); +INSERT INTO t39b VALUES ('{"b":2}'), ('[3,4]'); +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t39 +UNION ALL +SELECT JSON_QUERY(b, '$') FROM t39b) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 8 +# +# 40. The same two branches with the wide one alone, so that the +# column takes that same set and every character arrives as the one +# written. Nothing is taken back, and the count is the branches' +# own readings and nothing else - which is what 39 would have come +# to had the characters come across. +# +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t39 +UNION ALL +SELECT JSON_QUERY(j, '$') FROM t39) d; +SHOW STATUS LIKE 'Json_scans'; +Variable_name Value +Json_scans 4 +DROP TABLE t39, t39b; +DROP TABLE t1; +DROP TABLE t13, t13m; diff --git a/mysql-test/main/func_json_tmp_scan_count.test b/mysql-test/main/func_json_tmp_scan_count.test new file mode 100644 index 0000000000000..fd32af4f4bc10 --- /dev/null +++ b/mysql-test/main/func_json_tmp_scan_count.test @@ -0,0 +1,598 @@ +--source include/have_debug.inc + +--echo # +--echo # How many times a value out of the server's own temporary table is +--echo # read to find out whether it is a document. +--echo # +--echo # No query can see the difference - the reading could only have found +--echo # out what was already known - so nothing but Json_scans says whether +--echo # it happened. The counts below are the record of it. A count that +--echo # goes up is a reading that came back. +--echo # +--echo # A debug build reads values back to check what was claimed about +--echo # them, and none of those readings are counted: they are the debug +--echo # build's work and not the server's. A count of 0 below therefore +--echo # means nothing read, not that a reading went uncounted. +--echo # + +SET NAMES utf8mb4; + +# What is recorded here is work done rather than an answer given, so a +# statement run a second time to check that it repeats itself would be +# counted twice. +--disable_ps2_protocol +SET optimizer_switch='derived_merge=off'; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '{"b":2}'); + +--echo # +--echo # 1. TWO rows out of a base column, spliced into a new document. +--echo # A column says nothing about itself, so each row is read once and +--echo # that reading is not work that can be taken away. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(j) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 2. The same two rows, put through a function that attests to what +--echo # it returns, and spliced without a temporary table in between. +--echo # JSON_EXTRACT reads its input once; the splice reads nothing. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(JSON_EXTRACT(j, '$')) AS r FROM t1 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 3. The same, with a temporary table in between. What the function +--echo # answered is what the column was built to hold, so the count is the +--echo # function's own readings and nothing else. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 4. The same shape with a producer that attests for its class. +--echo # The claim holds for the column at the time it is made, so no row +--echo # is read on the way out. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 5. Through a GROUP BY, which writes a temporary table of its own. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1 GROUP BY id, j) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 6. A UNION, whose column has one producer per branch. The +--echo # branches are where those producers are all in reach, so what the +--echo # column answers is what every one of them answers, and the count +--echo # is their own readings and nothing else. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT JSON_EXTRACT(j, '$') FROM t1) d; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 7. A table the user asked for, which is not the server's own. +--echo # +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_EXTRACT(j, '$') FROM t1; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM tt; +SHOW STATUS LIKE 'Json_scans'; +DROP TEMPORARY TABLE tt; + +--echo # +--echo # 8. A store that came up short takes back what was said about the +--echo # column, so its rows are read on the way out like anything else. +--echo # +--echo # No function asks for less room than it goes on to use, so nothing +--echo # arrives short of its own accord and the short store is arranged +--echo # for here. The query is the one from 3, so the count below is what +--echo # 3 would have come to with nothing said about its column. +--echo # +SET @old_debug= @@SESSION.debug_dbug; +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 9. Two temporary tables one after the other. The second is built +--echo # before a row of the first exists, so what the first says then is +--echo # the most it will ever say. Every row put across asks it again, +--echo # and what is left to count is the first table's own readings. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 10. Asking what a materialized value IS, rather than splicing it. +--echo # JSON_TYPE reads its argument whatever anyone says about it: what is +--echo # promised is that the value is a document, not which one. +--echo # +FLUSH STATUS; +SELECT JSON_TYPE(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 11. An aggregate gathers its elements in a table of its own and +--echo # reads them back out of a record rather than off the item that put +--echo # them there. The column is what attests to them. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS r FROM t1; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 12. The same, with a producer that attests for its class. The +--echo # column takes that claim, so the elements are not read back and +--echo # nothing is counted here - where 11 pays two readings for the +--echo # values its producer parsed itself. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS r FROM t1; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 13. An aggregate whose arguments are buffered before the group can +--echo # be built - a join with a grouping the rows do not arrive in. The +--echo # argument is put into a table of its own on the way past, and read +--echo # back off that column. +--echo # +--echo # Each group gathers the same row more than once, so what is +--echo # gathered reads the same whichever order the rows arrive in - an +--echo # order a grouping does not fix and this file must not depend on. +--echo # +CREATE TABLE t13 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) + CHARSET utf8mb4; +INSERT INTO t13 VALUES (1, 1, '{"a":1}'), (2, 2, '{"b":2}'); +CREATE TABLE t13m (g INT); +INSERT INTO t13m VALUES (1), (1), (2), (2); +FLUSH STATUS; +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b + WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 14. The same shape gathered into an array instead. The table the +--echo # aggregate gathers in is built out of the buffer's column rather +--echo # than out of the function that filled it, and asks that column +--echo # again for every element it is handed. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b + WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 15. The chain of 9, with a store into the FIRST table that comes +--echo # up short. That column gives up its answer, and the second has to +--echo # hear about it although it was built long before. The query is +--echo # the one from 9, so the count is what 9 would have come to with +--echo # neither column answering. +--echo # +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 16. The same chain, with the row that crosses BETWEEN the two +--echo # tables arriving short instead. The first column still answers +--echo # for what it holds; it is the second that stops. +--echo # +SET SESSION debug_dbug='+d,json_tmp_chain_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAY(y) AS r FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 17. The buffered aggregate of 14, with the store into the buffer +--echo # coming up short. The aggregate's own table is filled from that +--echo # buffer field to field rather than off an item, so this is the +--echo # case that says the field-to-field route asks as well. +--echo # +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS r FROM t13 a, t13m b + WHERE a.g = b.g GROUP BY a.g ORDER BY a.g DESC; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 18. A grouping that needs no temporary table of its own still has +--echo # to hold a value across the group it belongs to, and it holds it in +--echo # a buffer rather than in a column. A HAVING or an ORDER BY reads +--echo # what it reads out of that buffer. +--echo # +--echo # A buffer holds one value at a time and is filled right beside the +--echo # item that made it, so what that item says about the value is put +--echo # away together with the value and is about it still when it is read +--echo # back. +--echo # +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 + HAVING JSON_ARRAY(x) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 + ORDER BY JSON_ARRAY(x); +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 19. The same with a producer that attests to the value it has +--echo # just made rather than for every value it will ever make. A column +--echo # cannot be told about one of those, being made before any of them +--echo # exists; a buffer is filled after, so what it holds is attested to +--echo # like anything else and nothing is read at all. +--echo # +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t1 + HAVING JSON_ARRAY(x) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 20. A buffer filled from a COLUMN rather than from an item, which +--echo # is the route a value takes when it is wide enough to be kept apart +--echo # from the row it belongs to. What the column says is what the +--echo # buffer puts away. +--echo # +CREATE TABLE t20 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t20 VALUES (1, '{"a":1}'), (2, '{"b":2}'); +FLUSH STATUS; +SELECT MAX(d.id) AS m, d.x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t20) d + HAVING JSON_ARRAY(y) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; +DROP TABLE t20; + +--echo # +--echo # 21. A buffer that could not take what it was handed keeps what it +--echo # held before, which nothing here can attest to. Nothing arrives +--echo # that way of its own accord - it takes a failure to find the room - +--echo # so it is arranged for. The query is the one from 18, so the count +--echo # is what 18 would have come to with the buffer saying nothing. +--echo # +SET SESSION debug_dbug='+d,json_copy_not_kept'; +FLUSH STATUS; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t1 + HAVING JSON_ARRAY(x) <> '[]'; +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +--echo # +--echo # 22. The same UNION where both branches attest for their class, +--echo # which is what a column needs: there is no telling which branch a +--echo # row came out of once it is in there, so one branch not attesting +--echo # would be the whole column not attesting. No row is read on the +--echo # way out, and the two counted here are the ones the JSON_EXTRACT +--echo # branch read itself. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT JSON_OBJECT('a', id) FROM t1) d; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 23. The same again with a branch that is a column, which says +--echo # nothing about itself and so says nothing for the column it is +--echo # written into either. +--echo # +FLUSH STATUS; +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT j FROM t1) d; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 24. Rows written one list at a time rather than one branch at a +--echo # time, which is the same column with the same one producer per +--echo # row and is asked the same way. +--echo # +FLUSH STATUS; +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('{"b":2}', '$'))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +SHOW STATUS LIKE 'Json_scans'; +--echo # +--echo # And a list whose producer attests for its class leaves the +--echo # column attesting, exactly as a branch does. The one counted +--echo # here is the JSON_EXTRACT list reading its own value; the +--echo # constructor reads nothing. +--echo # +FLUSH STATUS; +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS r FROM d ORDER BY r; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 25. A recursive CTE, whose column is built once the parts that do +--echo # not go round have agreed on a type - the parts that DO go round +--echo # not being reached at all, so nothing there can be asked and the +--echo # ones that were reached do not answer in their place. +--echo # +FLUSH STATUS; +WITH RECURSIVE r AS ( + SELECT 1 AS n, JSON_EXTRACT(j, '$') AS x FROM t1 + UNION ALL + SELECT n + 1, JSON_OBJECT('n', n) FROM r WHERE n < 2) +SELECT JSON_ARRAY(x) AS y FROM r ORDER BY y; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 26. A function that EDITS a document rather than building one. +--echo # It composes its answer and then reads the whole of it back, that +--echo # reading being where the spacing gets settled - unless what it was +--echo # given was already spaced that way, which is a second thing such a +--echo # column can say and says here. +--echo # +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 27. The same over a column whose producer returns a document +--echo # formatted some other way - JSON_QUERY copying a sub-document out as +--echo # it stands rather than writing one. Attested, and not written +--echo # for, so the reading back stays. +--echo # +FLUSH STATUS; +SELECT JSON_SET(x, '$.z', 9) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 28. The formatting carried from one such table into the next, which +--echo # the second column asks the first about at every row it is given. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; +--echo # +--echo # And taken back where what the first holds is formatted otherwise. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1) d2 ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 29. The same across the copier that fills a grouping's table, +--echo # which is a different way of putting a row across and is asked the +--echo # same thing. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 + ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 30. And that copier taking it back, for the same reason and by +--echo # asking the same source. +--echo # +FLUSH STATUS; +SELECT JSON_SET(y, '$.z', 9) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t1) d1 GROUP BY id, x) d2 + ORDER BY id; +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # Everything above is short enough that half its length is inside +--echo # the limit, so nothing there was ever read to find out how deep it +--echo # goes. These are not: at over a hundred characters the length says +--echo # nothing useful, and what the column can say about the depth is the +--echo # only thing left. +--echo # +CREATE TABLE t31 (id INT, j VARCHAR(400) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t31 VALUES + (1, CONCAT('{"a":[1,2],"pad":"', REPEAT('x', 100), '"}')), + (2, CONCAT('{"b":[3,4],"pad":"', REPEAT('y', 100), '"}')); + +--echo # +--echo # 31. Two long rows out of a base column. A column of a table the +--echo # user asked for says nothing about how deep its values go any more +--echo # than about anything else, so each row is read. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(j) AS r FROM t31 ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 32. The same through a temporary table. The producer counted the +--echo # depth while it wrote and the column kept the figure, so the splice +--echo # has what it needs without the length being asked. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 33. The same carried from one such table into the next, which the +--echo # second column takes from the first at every row it is given. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(y) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d1) d2 ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 34. An aggregate gathering its elements out of such a column, +--echo # which is the only thing left there to ask. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 35. A producer that returns a piece of a document it was given +--echo # rather than one it wrote counted nothing, so the column has no +--echo # figure to keep and the reading stays. The count is what 32 would +--echo # have come to with nothing said. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t31) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 36. A column written by more than one producer keeps the deepest +--echo # of all its rows, so where every one of them counted, every row +--echo # gets the figure. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 + UNION ALL + SELECT id, JSON_EXTRACT(j, '$.a') FROM t31) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # And one row that nobody counted is the whole column saying +--echo # nothing, the deepest of the rows being unknown once one of them +--echo # is. The same four rows, one pair of them uncounted. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t31 + UNION ALL + SELECT id, JSON_QUERY(j, '$') FROM t31) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t31; + +--echo # +--echo # 37. A column filled with the names of a document's keys. Every +--echo # row of one of those is an array of strings whatever the document +--echo # held, and that is so before a row of it exists - so the column is +--echo # built saying it, and the splice on the way out reads nothing. +--echo # +--echo # The names are long here for the same reason as above: the length +--echo # says more levels than a document is allowed, so the depth the +--echo # column keeps is what the splice has to go on. +--echo # +CREATE TABLE t37 (id INT, j TEXT) CHARSET utf8mb4; +INSERT INTO t37 VALUES + (1, CONCAT('{"', REPEAT('a', 40), '":1,"', REPEAT('b', 40), '":2}')), + (2, CONCAT('{"', REPEAT('c', 40), '":3,"', REPEAT('d', 40), '":4}')); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 38. The same query with the store into that column coming up +--echo # short, which is how 8 takes 3's column away. The count is what +--echo # 37 would have come to with nothing said, so the difference +--echo # between the two is the column and nothing else. +--echo # +SET SESSION debug_dbug='+d,json_tmp_store_kept_short'; +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT id, JSON_KEYS(j) AS x FROM t37) d ORDER BY id; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; +SET SESSION debug_dbug= @old_debug; + +DROP TABLE t37; + +--echo # +--echo # 39. Branches written in different character sets. A column takes +--echo # one set for all of them, and a branch in another set is stored +--echo # into it converting - except into the binary set, which converts +--echo # nothing at all and keeps every byte under the other set's name. +--echo # Rows that arrived that way are not the documents that were +--echo # written, so the column takes its answer back and every row is +--echo # read on the way out. +--echo # +CREATE TABLE t39 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t39 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t39b (b BLOB); +INSERT INTO t39b VALUES ('{"b":2}'), ('[3,4]'); +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t39 + UNION ALL + SELECT JSON_QUERY(b, '$') FROM t39b) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +--echo # +--echo # 40. The same two branches with the wide one alone, so that the +--echo # column takes that same set and every character arrives as the one +--echo # written. Nothing is taken back, and the count is the branches' +--echo # own readings and nothing else - which is what 39 would have come +--echo # to had the characters come across. +--echo # +FLUSH STATUS; +--disable_result_log +SELECT JSON_ARRAY(x) AS r FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t39 + UNION ALL + SELECT JSON_QUERY(j, '$') FROM t39) d; +--enable_result_log +SHOW STATUS LIKE 'Json_scans'; + +DROP TABLE t39, t39b; + +DROP TABLE t1; +DROP TABLE t13, t13m; + +--enable_ps2_protocol diff --git a/mysql-test/main/func_json_tmp_trust.result b/mysql-test/main/func_json_tmp_trust.result new file mode 100644 index 0000000000000..e43d40bae7024 --- /dev/null +++ b/mysql-test/main/func_json_tmp_trust.result @@ -0,0 +1,618 @@ +# +# What a JSON function reads back out of a temporary table the server +# built for itself. +# +# Such a table is written, read and thrown away inside one query, and +# no statement can name it, so what goes into a column of one is +# decided once, when the column is made. A column whose one producer +# always gives back a document therefore holds documents, and a +# reader of it can be spared finding that out again. +# +# A column of a table anybody can name says nothing of the sort: a row +# can hold bytes no check ever saw, and nothing in one records where +# they came from. The answers below are the same either way - the +# reading that is left out could only have found out what was already +# known - so this file is here to say that they stay the same. +# +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); +# +# 1. A document out of a temporary table, spliced into a new one. +# +SELECT id, JSON_ARRAY(x) AS spliced FROM +(SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +id spliced +1 [{"a": 1, "c": 1}] +2 [[1, 2]] +SELECT id, JSON_OBJECT('k', x) AS spliced FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +id spliced +1 {"k": {"a": 1}} +2 {"k": [1, 2]} +# +# 2. The same, edited rather than spliced. +# +SELECT id, JSON_SET(x, '$.d', 4) AS edited FROM +(SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +id edited +1 {"a": 1, "c": 1, "d": 4} +2 [1, 2] +SELECT id, JSON_MERGE_PATCH(x, '{"e":5}') AS merged FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; +id merged +1 {"a": 1, "e": 5} +2 {"e": 5} +# +# 3. Asked about, rather than read. +# +SELECT id, JSON_VALID(x) AS valid, JSON_TYPE(x) AS what, JSON_DEPTH(x) AS deep +FROM (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +id valid what deep +1 1 OBJECT 2 +2 1 ARRAY 2 +# +# 4. Through a GROUP BY, a DISTINCT and an ORDER BY, each of which +# writes a temporary table of its own. +# +SELECT JSON_ARRAY(x) AS grouped FROM +(SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1 GROUP BY id, j) d +ORDER BY x; +grouped +[[1, 2]] +[{"a": 1, "c": 1}] +SELECT DISTINCT JSON_ARRAY(x) AS distincted FROM +(SELECT JSON_SET(j, '$.c', 1) AS x FROM t1) d ORDER BY 1; +distincted +[[1, 2]] +[{"a": 1, "c": 1}] +# +# 5. Through a UNION, whose column has one producer per branch and +# answers what all of them answer - the branches being where they +# are all in reach. One that does not answer is the whole column +# not answering, there being no telling afterwards which branch a +# row came out of. +# +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 +UNION ALL +SELECT j FROM t1) d ORDER BY 1; +unioned +[[1, 2]] +[[1,2]] +[{"a": 1, "c": 1}] +[{"a":1}] +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 +UNION ALL +SELECT JSON_EXTRACT(j, '$') FROM t1) d ORDER BY 1; +unioned +[[1, 2]] +[[1, 2]] +[{"a": 1, "c": 1}] +[{"a": 1}] +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +UNION +SELECT JSON_QUERY(j, '$') FROM t1) d ORDER BY 1; +unioned +[[1, 2]] +[[1,2]] +[{"a": 1}] +[{"a":1}] +SELECT JSON_ARRAY(x) AS unioned FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1 +EXCEPT +SELECT JSON_EXTRACT(j, '$') FROM t1 WHERE id = 9) d ORDER BY 1; +unioned +[[1, 2]] +[{"a": 1}] +# +# The same column written one list of values at a time. +# +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('[1,2]', '$'))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +unioned +[[1,2]] +[{"a": 1}] +WITH d(x) AS +(VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +unioned +[{"a": 1}] +[{"b": 2}] +# +# A column of a recursive CTE is built once the parts that do not go +# round have agreed on a type, the parts that DO go round not being +# reached at all. So a value can arrive there from a producer +# nothing asked, and here one arrives that is no document: a column +# typed JSON holds whatever was put in it while its check was off. +# +CREATE TABLE t5 (good JSON, bad JSON); +SET @old_check= @@SESSION.check_constraint_checks; +SET SESSION check_constraint_checks= OFF; +INSERT INTO t5 VALUES ('{"a":1}', 'not a document'); +SET SESSION check_constraint_checks= @old_check; +WITH RECURSIVE r AS ( +SELECT 1 AS n, JSON_EXTRACT(good, '$') AS x FROM t5 +UNION ALL +SELECT n + 1, t5.bad FROM r, t5 WHERE n < 2) +SELECT n, JSON_ARRAY(x) AS unioned FROM r ORDER BY n; +n unioned +1 [{"a": 1}] +2 NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +DROP TABLE t5; +# +# Branches written in different character sets. The column takes +# one set for all of them, and a branch whose own set is not that +# one is stored into it converting. A store into the binary set +# converts nothing: it keeps every byte and calls them by the other +# set's name, so a document written in a wide set arrives as bytes +# that nothing reads as a document, and it arrives at exactly the +# length it was written at. A byte count is therefore the one +# measure that cannot tell this apart, and each branch is asked +# about its characters as well as its length. +# +CREATE TABLE t6 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t6 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t7 (b BLOB); +INSERT INTO t7 VALUES ('{"b":2}'), ('[3,4]'); +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, +HEX(x) AS bytes FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t6 +UNION ALL +SELECT JSON_QUERY(b, '$') FROM t7) d ORDER BY bytes; +packed still_a_document bytes +NULL 0 005B0031002C0032005D +NULL 0 007B002200610022003A0031007D +5B5B332C345D5D 1 5B332C345D +5B7B2262223A327D5D 1 7B2262223A327D +Warnings: +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +# +# The same two branches with the wide one on the right, so that it +# is the second producer rather than the first that has to move. +# +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, +HEX(x) AS bytes FROM +(SELECT JSON_QUERY(b, '$') AS x FROM t7 +UNION ALL +SELECT JSON_QUERY(j, '$') FROM t6) d ORDER BY bytes; +packed still_a_document bytes +NULL 0 005B0031002C0032005D +NULL 0 007B002200610022003A0031007D +5B5B332C345D5D 1 5B332C345D +5B7B2262223A327D5D 1 7B2262223A327D +Warnings: +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_array' at position 1 +Note 4036 Character disallowed in JSON in argument 1 to function 'json_valid' at position 1 +# +# And both branches in the wide set, where the column takes that +# same set and every character comes across as the one written. +# +SELECT JSON_ARRAY(x) AS unioned, JSON_VALID(x) AS still_a_document FROM +(SELECT JSON_QUERY(j, '$') AS x FROM t6 +UNION ALL +SELECT JSON_QUERY(j, '$') FROM t6 WHERE j <> '[1,2]') d ORDER BY 1; +unioned still_a_document +[[1,2]] 1 +[{"a":1}] 1 +[{"a":1}] 1 +DROP TABLE t6, t7; +# +# 6. A producer that attests to every value it will ever make, +# rather than to the one it has just made. A constructor returns a +# document or nothing whatever it is given, so a column fed by one +# is read without being asked. So does an aggregate, and so does a +# constructor composing from one. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY 1; +spliced +[{"a": 1}] +[{"a": 2}] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_ARRAY(id, j) AS x FROM t1) d ORDER BY 1; +spliced +[[1, {"a":1}]] +[[2, [1,2]]] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_ARRAYAGG(j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +spliced +[[[1,2]]] +[[{"a":1}]] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_OBJECTAGG(id, j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +spliced +[{"1":{"a":1}}] +[{"2":[1,2]}] +# +# And in a character set that cannot write the brackets it answers +# nothing, there being no document to promise - see section 12. +# +SELECT HEX(JSON_ARRAY(x)) AS spliced FROM +(SELECT JSON_ARRAY(CONVERT(j USING swe7)) AS x FROM t1) d ORDER BY 1; +spliced +NULL +NULL +Warnings: +Warning 1977 Cannot convert 'utf8mb4' character 0x7B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 1977 Cannot convert 'utf8mb4' character 0x5B to 'swe7' +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_array' at position 1 +# +# 7. A function that returns more than it was given. JSON_REMOVE +# writes a space after every separator it copies, so taking one +# element out of a dense array returns more than there was to begin +# with, and the column it goes into has to have been asked for wide +# enough to hold it. +# +CREATE TABLE t7 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t7 VALUES +('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), +('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); +SELECT LENGTH(j) AS given, LENGTH(JSON_REMOVE(j, '$[0]')) AS produced +FROM t7 ORDER BY 1, 2; +given produced +61 87 +61 87 +SELECT LENGTH(x) AS kept, JSON_VALID(x) AS still_a_document FROM +(SELECT JSON_REMOVE(j, '$[0]') AS x FROM t7) d ORDER BY 1; +kept still_a_document +87 1 +87 1 +DROP TABLE t7; +# +# The same function on a document that leaves room to spare. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_REMOVE(j, '$.a') AS x FROM t1 WHERE id = 1) d; +spliced +[{}] +# +# 8. A table the user asked for is not the server's own, whatever is +# written into it and whatever check it carries. +# +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_SET(j, '$.c', id) FROM t1; +SELECT JSON_ARRAY(x) AS spliced FROM tt ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +DROP TEMPORARY TABLE tt; +# +# 9. A materialized subquery. +# +SELECT id FROM t1 WHERE JSON_ARRAY(j) IN +(SELECT JSON_ARRAY(x) FROM (SELECT j AS x FROM t1) d) ORDER BY id; +id +1 +2 +# +# 10. Two temporary tables one after the other. The second was +# built before a row of the first existed, so what it was given +# then is only as much as the first can still say later, and it +# asks again for every row put across. +# +SELECT JSON_ARRAY(y) AS spliced FROM +(SELECT x AS y FROM +(SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2 ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +# +# The same three deep, and with a grouping in the middle so the row +# crosses by more than one route. +# +SELECT JSON_ARRAY(z) AS spliced FROM +(SELECT y AS z FROM +(SELECT x AS y FROM +(SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2) d3 ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +SELECT JSON_ARRAY(y) AS spliced FROM +(SELECT x AS y FROM +(SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY x) d2 ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1}] +# +# 11. A NULL, which is not a document and is not read as one. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_SET(NULL, '$.c', 1) AS x FROM t1) d ORDER BY 1; +spliced +[null] +[null] +# +# 12. A document written in one character set and read in another. +# +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT CONVERT(JSON_SET(j, '$.c', 1) USING latin1) AS x FROM t1) d +ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +SELECT JSON_ARRAY(x) AS spliced FROM +(SELECT JSON_SET(CONVERT(j USING latin1), '$.c', 1) AS x FROM t1) d +ORDER BY 1; +spliced +[[1, 2]] +[{"a": 1, "c": 1}] +DROP TABLE t1; +# +# 13. A window function's temporary table. +# +CREATE TABLE t2 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) +CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, 1, '{"a":1}'), (2, 1, '{"b":2}'), (3, 2, '[3]'); +SELECT id, JSON_ARRAY(x) AS spliced FROM +(SELECT id, JSON_SET(j, '$.n', +ROW_NUMBER() OVER (PARTITION BY g ORDER BY id)) AS x +FROM t2) d ORDER BY id; +id spliced +1 [{"a": 1, "n": 1}] +2 [{"b": 2, "n": 2}] +3 [[3]] +# +# 14. An aggregate, which gathers its elements in a table of its own +# and reads them back out of a record. +# +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS gathered FROM t2; +gathered +[{"a": 1},{"b": 2},[3]] +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS gathered FROM t2; +gathered +[{"a": 1},{"a": 2},{"a": 3}] +SELECT g, JSON_OBJECTAGG(id, JSON_EXTRACT(j, '$')) AS gathered FROM t2 +GROUP BY g ORDER BY g; +g gathered +1 {"1":{"a": 1}, "2":{"b": 2}} +2 {"3":[3]} +# +# 15. The same, with the arguments buffered before the group can be +# built. Each group gathers the same row more than once, so what is +# gathered reads the same whichever order the rows arrive in - an +# order a grouping does not fix and this file must not depend on. +# +CREATE TABLE t15m (g INT); +INSERT INTO t15m VALUES (1), (1), (2), (2); +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS gathered +FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) +GROUP BY a.g ORDER BY a.g DESC; +gathered +{"3":[3], "3":[3]} +{"1":{"a": 1}, "1":{"a": 1}} +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS gathered +FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) +GROUP BY a.g ORDER BY a.g DESC; +gathered +[[3],[3]] +[{"a": 1},{"a": 1}] +DROP TABLE t15m; +# +# 16. A value the aggregate cannot carry as a document. +# +SELECT JSON_ARRAYAGG(j) AS gathered FROM t2; +gathered +[{"a":1},{"b":2},[3]] +# +# 17. A grouping that writes no temporary table holds its values in +# buffers instead, and a HAVING or an ORDER BY reads them back out +# of those rather than out of a record. +# +SELECT MAX(id) AS m, JSON_SET(j, '$.c', 1) AS x FROM t2 +HAVING JSON_ARRAY(x) <> '[]'; +m x +3 {"a": 1, "c": 1} +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 +ORDER BY JSON_ARRAY(x); +m x +3 {"a": 1} +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t2 +HAVING JSON_ARRAY(x) <> '[]'; +m x +3 {"a": 1} +# +# The same where the buffer is filled from a column rather than +# from an item, which is the route a value wide enough to be kept +# apart from its row takes. +# +CREATE TABLE t17 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t17 VALUES (1, '{"a":1}'), (2, '[1,2]'); +SELECT MAX(d.id) AS m, d.x AS y FROM +(SELECT id, JSON_SET(j, '$.c', 1) AS x FROM t17) d +HAVING JSON_ARRAY(y) <> '[]'; +m y +2 {"a": 1, "c": 1} +SELECT MAX(d.id) AS m, d.j AS y FROM t17 d +HAVING JSON_ARRAY(y) <> '[]'; +m y +2 {"a":1} +DROP TABLE t17; +# +# A NULL, and a character set conversion around the producer - +# which carries the value through while the type stops at it, so the +# buffer attests to nothing and the value is quoted, not spliced. +# +SELECT MAX(id) AS m, JSON_SET(NULL, '$.c', 1) AS x FROM t2 +HAVING JSON_ARRAY(x) IS NULL; +m x +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 +ORDER BY JSON_ARRAY(x); +m x +3 {"a": 1} +# +# Quoted rather than spliced is the whole of what the buffer was +# asked, and a buffer is read from a HAVING or an ORDER BY and never +# from the select list - so what went into the document is only to +# be had through a comparison. Asking what its first element IS +# tells the two apart, and the row comes back for whichever +# happened. +# +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +m x +3 {"a": 1} +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +m x +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 +HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'STRING'; +m x +3 {"a": 1} +# +# And the same conversion with no buffer in the way IS looked +# through, which is what makes the two above a fact about the buffer +# rather than about the conversion. +# +SELECT id, JSON_ARRAY(CONVERT(JSON_EXTRACT(j, '$') USING latin1)) AS r +FROM t2 ORDER BY id; +id r +1 [{"a": 1}] +2 [{"b": 2}] +3 [[3]] +# +# 18. How a value is FORMATTED, which is a second thing such a column +# can attest to and is come by differently. Being a document is +# promised by the producer before a row exists; the formatting is +# settled one value at a time, so it is granted along with the first +# answer and spent by whatever gets written afterwards. +# +CREATE TABLE t18 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t18 VALUES (1, '{"a":1}'), (2, '[1,2]'); +# +# Edited out of a column whose producer writes the loose way, out of +# one that copies a sub-document in whatever way it found it, and out +# of one that writes a form of its own. +# +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM +(SELECT id, JSON_NORMALIZE(j) AS x FROM t18) d ORDER BY id; +id r +1 {"a": 1.0E0, "c": 1} +2 [1.0E0, 2.0E0] +# +# Carried from one such table into the next, and taken back there +# where what the first holds was written some other way. +# +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +# +# The same across the copier that fills a grouping's table. +# +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 +ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 +ORDER BY id; +id r +1 {"a": 1, "c": 1} +2 [1, 2] +# +# An aggregate reads its elements out of a record, the column being +# the only thing left there to ask - so what it says about their +# formatting is what the gathered document can say about its own. +# +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d; +r +[[{"a": 1},[1, 2]]] +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM +(SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d; +r +[[{"a":1},[1,2]]] +DROP TABLE t18; +# +# 19. How deep the values went, which is a figure rather than a yes. +# +# What a column keeps is the deepest of the rows written into it so +# far, not the depth of the row being read - more than that row +# needs and never less. More is the direction this one is allowed +# to be wrong in: a figure too large costs a reading that could have +# been skipped, while one too small would admit a document that +# nothing can read back. Nothing acts on the figure except to skip +# a reading, so the answers below are the ones the server has always +# given. +# +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), +'1', REPEAT('}', 30)); +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), +'1', REPEAT('}', 31)); +CREATE TABLE t19 (id INT, j LONGTEXT) CHARSET utf8mb4; +INSERT INTO t19 VALUES (1, @k30), (2, '{"a":1}'); +# +# One row at the limit and a shallow one beside it, put one level +# in. The column answers 30 for both, which is exact for the first +# and too large for the second, and each still ends up where it +# belongs. +# +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +id d +1 32 +2 3 +# +# A byte past it, where the same shape is refused - and the shallow +# row in the same column is not, the figure being about the column +# and the limit about each document that comes out of it. +# +UPDATE t19 SET j= @k31 WHERE id= 1; +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +id d +1 NULL +2 3 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array' at position 497 +# +# Each level of these is fourteen characters, so half the length is +# seven times the limit and the length can tell neither of them from +# the other. Only the figure the column kept can. +# +SELECT JSON_DEPTH(JSON_ARRAY(y)) AS d FROM +(SELECT id, x AS y FROM +(SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d1 WHERE id= 2) d2; +d +3 +DROP TABLE t19; +DROP TABLE t2; diff --git a/mysql-test/main/func_json_tmp_trust.test b/mysql-test/main/func_json_tmp_trust.test new file mode 100644 index 0000000000000..c1fdcff1c521d --- /dev/null +++ b/mysql-test/main/func_json_tmp_trust.test @@ -0,0 +1,435 @@ +--echo # +--echo # What a JSON function reads back out of a temporary table the server +--echo # built for itself. +--echo # +--echo # Such a table is written, read and thrown away inside one query, and +--echo # no statement can name it, so what goes into a column of one is +--echo # decided once, when the column is made. A column whose one producer +--echo # always gives back a document therefore holds documents, and a +--echo # reader of it can be spared finding that out again. +--echo # +--echo # A column of a table anybody can name says nothing of the sort: a row +--echo # can hold bytes no check ever saw, and nothing in one records where +--echo # they came from. The answers below are the same either way - the +--echo # reading that is left out could only have found out what was already +--echo # known - so this file is here to say that they stay the same. +--echo # + +SET NAMES utf8mb4; +SET optimizer_switch='derived_merge=off'; + +CREATE TABLE t1 (id INT, j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t1 VALUES (1, '{"a":1}'), (2, '[1,2]'); + +--echo # +--echo # 1. A document out of a temporary table, spliced into a new one. +--echo # +SELECT id, JSON_ARRAY(x) AS spliced FROM + (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +SELECT id, JSON_OBJECT('k', x) AS spliced FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; + +--echo # +--echo # 2. The same, edited rather than spliced. +--echo # +SELECT id, JSON_SET(x, '$.d', 4) AS edited FROM + (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; +SELECT id, JSON_MERGE_PATCH(x, '{"e":5}') AS merged FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t1) d ORDER BY id; + +--echo # +--echo # 3. Asked about, rather than read. +--echo # +SELECT id, JSON_VALID(x) AS valid, JSON_TYPE(x) AS what, JSON_DEPTH(x) AS deep + FROM (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1) d ORDER BY id; + +--echo # +--echo # 4. Through a GROUP BY, a DISTINCT and an ORDER BY, each of which +--echo # writes a temporary table of its own. +--echo # +SELECT JSON_ARRAY(x) AS grouped FROM + (SELECT id, JSON_SET(j, '$.c', id) AS x FROM t1 GROUP BY id, j) d + ORDER BY x; +SELECT DISTINCT JSON_ARRAY(x) AS distincted FROM + (SELECT JSON_SET(j, '$.c', 1) AS x FROM t1) d ORDER BY 1; + +--echo # +--echo # 5. Through a UNION, whose column has one producer per branch and +--echo # answers what all of them answer - the branches being where they +--echo # are all in reach. One that does not answer is the whole column +--echo # not answering, there being no telling afterwards which branch a +--echo # row came out of. +--echo # +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 + UNION ALL + SELECT j FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_SET(j, '$.c', 1) AS x FROM t1 + UNION ALL + SELECT JSON_EXTRACT(j, '$') FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + UNION + SELECT JSON_QUERY(j, '$') FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS unioned FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1 + EXCEPT + SELECT JSON_EXTRACT(j, '$') FROM t1 WHERE id = 9) d ORDER BY 1; +--echo # +--echo # The same column written one list of values at a time. +--echo # +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_QUERY('[1,2]', '$'))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +WITH d(x) AS + (VALUES (JSON_EXTRACT('{"a":1}', '$')), (JSON_OBJECT('b', 2))) +SELECT JSON_ARRAY(x) AS unioned FROM d ORDER BY 1; +--echo # +--echo # A column of a recursive CTE is built once the parts that do not go +--echo # round have agreed on a type, the parts that DO go round not being +--echo # reached at all. So a value can arrive there from a producer +--echo # nothing asked, and here one arrives that is no document: a column +--echo # typed JSON holds whatever was put in it while its check was off. +--echo # +CREATE TABLE t5 (good JSON, bad JSON); +SET @old_check= @@SESSION.check_constraint_checks; +SET SESSION check_constraint_checks= OFF; +INSERT INTO t5 VALUES ('{"a":1}', 'not a document'); +SET SESSION check_constraint_checks= @old_check; +WITH RECURSIVE r AS ( + SELECT 1 AS n, JSON_EXTRACT(good, '$') AS x FROM t5 + UNION ALL + SELECT n + 1, t5.bad FROM r, t5 WHERE n < 2) +SELECT n, JSON_ARRAY(x) AS unioned FROM r ORDER BY n; +DROP TABLE t5; +--echo # +--echo # Branches written in different character sets. The column takes +--echo # one set for all of them, and a branch whose own set is not that +--echo # one is stored into it converting. A store into the binary set +--echo # converts nothing: it keeps every byte and calls them by the other +--echo # set's name, so a document written in a wide set arrives as bytes +--echo # that nothing reads as a document, and it arrives at exactly the +--echo # length it was written at. A byte count is therefore the one +--echo # measure that cannot tell this apart, and each branch is asked +--echo # about its characters as well as its length. +--echo # +CREATE TABLE t6 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET ucs2; +INSERT INTO t6 VALUES ('{"a":1}'), ('[1,2]'); +CREATE TABLE t7 (b BLOB); +INSERT INTO t7 VALUES ('{"b":2}'), ('[3,4]'); +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, + HEX(x) AS bytes FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t6 + UNION ALL + SELECT JSON_QUERY(b, '$') FROM t7) d ORDER BY bytes; +--echo # +--echo # The same two branches with the wide one on the right, so that it +--echo # is the second producer rather than the first that has to move. +--echo # +SELECT HEX(JSON_ARRAY(x)) AS packed, JSON_VALID(x) AS still_a_document, + HEX(x) AS bytes FROM + (SELECT JSON_QUERY(b, '$') AS x FROM t7 + UNION ALL + SELECT JSON_QUERY(j, '$') FROM t6) d ORDER BY bytes; +--echo # +--echo # And both branches in the wide set, where the column takes that +--echo # same set and every character comes across as the one written. +--echo # +SELECT JSON_ARRAY(x) AS unioned, JSON_VALID(x) AS still_a_document FROM + (SELECT JSON_QUERY(j, '$') AS x FROM t6 + UNION ALL + SELECT JSON_QUERY(j, '$') FROM t6 WHERE j <> '[1,2]') d ORDER BY 1; +DROP TABLE t6, t7; + +--echo # +--echo # 6. A producer that attests to every value it will ever make, +--echo # rather than to the one it has just made. A constructor returns a +--echo # document or nothing whatever it is given, so a column fed by one +--echo # is read without being asked. So does an aggregate, and so does a +--echo # constructor composing from one. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_OBJECT('a', id) AS x FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_ARRAY(id, j) AS x FROM t1) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_ARRAYAGG(j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_OBJECTAGG(id, j) AS x FROM t1 GROUP BY id) d ORDER BY 1; +--echo # +--echo # And in a character set that cannot write the brackets it answers +--echo # nothing, there being no document to promise - see section 12. +--echo # +SELECT HEX(JSON_ARRAY(x)) AS spliced FROM + (SELECT JSON_ARRAY(CONVERT(j USING swe7)) AS x FROM t1) d ORDER BY 1; + +--echo # +--echo # 7. A function that returns more than it was given. JSON_REMOVE +--echo # writes a space after every separator it copies, so taking one +--echo # element out of a dense array returns more than there was to begin +--echo # with, and the column it goes into has to have been asked for wide +--echo # enough to hold it. +--echo # +CREATE TABLE t7 (j VARCHAR(64) CHECK (JSON_VALID(j))) CHARSET utf8mb4; +INSERT INTO t7 VALUES + ('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), + ('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); +SELECT LENGTH(j) AS given, LENGTH(JSON_REMOVE(j, '$[0]')) AS produced + FROM t7 ORDER BY 1, 2; +SELECT LENGTH(x) AS kept, JSON_VALID(x) AS still_a_document FROM + (SELECT JSON_REMOVE(j, '$[0]') AS x FROM t7) d ORDER BY 1; +DROP TABLE t7; +--echo # +--echo # The same function on a document that leaves room to spare. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_REMOVE(j, '$.a') AS x FROM t1 WHERE id = 1) d; + +--echo # +--echo # 8. A table the user asked for is not the server's own, whatever is +--echo # written into it and whatever check it carries. +--echo # +CREATE TEMPORARY TABLE tt (x VARCHAR(64) CHECK (JSON_VALID(x))) CHARSET utf8mb4; +INSERT INTO tt SELECT JSON_SET(j, '$.c', id) FROM t1; +SELECT JSON_ARRAY(x) AS spliced FROM tt ORDER BY 1; +DROP TEMPORARY TABLE tt; + +--echo # +--echo # 9. A materialized subquery. +--echo # +SELECT id FROM t1 WHERE JSON_ARRAY(j) IN + (SELECT JSON_ARRAY(x) FROM (SELECT j AS x FROM t1) d) ORDER BY id; + +--echo # +--echo # 10. Two temporary tables one after the other. The second was +--echo # built before a row of the first existed, so what it was given +--echo # then is only as much as the first can still say later, and it +--echo # asks again for every row put across. +--echo # +SELECT JSON_ARRAY(y) AS spliced FROM + (SELECT x AS y FROM + (SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2 ORDER BY 1; +--echo # +--echo # The same three deep, and with a grouping in the middle so the row +--echo # crosses by more than one route. +--echo # +SELECT JSON_ARRAY(z) AS spliced FROM + (SELECT y AS z FROM + (SELECT x AS y FROM + (SELECT JSON_SET(j, '$.c', id) AS x FROM t1) d1) d2) d3 ORDER BY 1; +SELECT JSON_ARRAY(y) AS spliced FROM + (SELECT x AS y FROM + (SELECT JSON_EXTRACT(j, '$') AS x FROM t1) d1 GROUP BY x) d2 ORDER BY 1; + +--echo # +--echo # 11. A NULL, which is not a document and is not read as one. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_SET(NULL, '$.c', 1) AS x FROM t1) d ORDER BY 1; + +--echo # +--echo # 12. A document written in one character set and read in another. +--echo # +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT CONVERT(JSON_SET(j, '$.c', 1) USING latin1) AS x FROM t1) d + ORDER BY 1; +SELECT JSON_ARRAY(x) AS spliced FROM + (SELECT JSON_SET(CONVERT(j USING latin1), '$.c', 1) AS x FROM t1) d + ORDER BY 1; + +DROP TABLE t1; + +--echo # +--echo # 13. A window function's temporary table. +--echo # +CREATE TABLE t2 (id INT, g INT, j VARCHAR(64) CHECK (JSON_VALID(j))) + CHARSET utf8mb4; +INSERT INTO t2 VALUES (1, 1, '{"a":1}'), (2, 1, '{"b":2}'), (3, 2, '[3]'); +SELECT id, JSON_ARRAY(x) AS spliced FROM + (SELECT id, JSON_SET(j, '$.n', + ROW_NUMBER() OVER (PARTITION BY g ORDER BY id)) AS x + FROM t2) d ORDER BY id; + +--echo # +--echo # 14. An aggregate, which gathers its elements in a table of its own +--echo # and reads them back out of a record. +--echo # +SELECT JSON_ARRAYAGG(JSON_EXTRACT(j, '$')) AS gathered FROM t2; +SELECT JSON_ARRAYAGG(JSON_OBJECT('a', id)) AS gathered FROM t2; +SELECT g, JSON_OBJECTAGG(id, JSON_EXTRACT(j, '$')) AS gathered FROM t2 + GROUP BY g ORDER BY g; + +--echo # +--echo # 15. The same, with the arguments buffered before the group can be +--echo # built. Each group gathers the same row more than once, so what is +--echo # gathered reads the same whichever order the rows arrive in - an +--echo # order a grouping does not fix and this file must not depend on. +--echo # +CREATE TABLE t15m (g INT); +INSERT INTO t15m VALUES (1), (1), (2), (2); +SELECT JSON_OBJECTAGG(a.id, JSON_EXTRACT(a.j, '$')) AS gathered + FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) + GROUP BY a.g ORDER BY a.g DESC; +SELECT JSON_ARRAYAGG(JSON_EXTRACT(a.j, '$')) AS gathered + FROM t2 a, t15m b WHERE a.g = b.g AND a.id IN (1, 3) + GROUP BY a.g ORDER BY a.g DESC; +DROP TABLE t15m; + +--echo # +--echo # 16. A value the aggregate cannot carry as a document. +--echo # +SELECT JSON_ARRAYAGG(j) AS gathered FROM t2; + +--echo # +--echo # 17. A grouping that writes no temporary table holds its values in +--echo # buffers instead, and a HAVING or an ORDER BY reads them back out +--echo # of those rather than out of a record. +--echo # +SELECT MAX(id) AS m, JSON_SET(j, '$.c', 1) AS x FROM t2 + HAVING JSON_ARRAY(x) <> '[]'; +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 + ORDER BY JSON_ARRAY(x); +SELECT MAX(id) AS m, JSON_OBJECT('a', id) AS x FROM t2 + HAVING JSON_ARRAY(x) <> '[]'; +--echo # +--echo # The same where the buffer is filled from a column rather than +--echo # from an item, which is the route a value wide enough to be kept +--echo # apart from its row takes. +--echo # +CREATE TABLE t17 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t17 VALUES (1, '{"a":1}'), (2, '[1,2]'); +SELECT MAX(d.id) AS m, d.x AS y FROM + (SELECT id, JSON_SET(j, '$.c', 1) AS x FROM t17) d + HAVING JSON_ARRAY(y) <> '[]'; +SELECT MAX(d.id) AS m, d.j AS y FROM t17 d + HAVING JSON_ARRAY(y) <> '[]'; +DROP TABLE t17; +--echo # +--echo # A NULL, and a character set conversion around the producer - +--echo # which carries the value through while the type stops at it, so the +--echo # buffer attests to nothing and the value is quoted, not spliced. +--echo # +SELECT MAX(id) AS m, JSON_SET(NULL, '$.c', 1) AS x FROM t2 + HAVING JSON_ARRAY(x) IS NULL; +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 + ORDER BY JSON_ARRAY(x); +--echo # +--echo # Quoted rather than spliced is the whole of what the buffer was +--echo # asked, and a buffer is read from a HAVING or an ORDER BY and never +--echo # from the select list - so what went into the document is only to +--echo # be had through a comparison. Asking what its first element IS +--echo # tells the two apart, and the row comes back for whichever +--echo # happened. +--echo # +SELECT MAX(id) AS m, JSON_EXTRACT(j, '$') AS x FROM t2 + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'OBJECT'; +SELECT MAX(id) AS m, CONVERT(JSON_EXTRACT(j, '$') USING latin1) AS x FROM t2 + HAVING JSON_TYPE(JSON_EXTRACT(JSON_ARRAY(x), '$[0]')) = 'STRING'; +--echo # +--echo # And the same conversion with no buffer in the way IS looked +--echo # through, which is what makes the two above a fact about the buffer +--echo # rather than about the conversion. +--echo # +SELECT id, JSON_ARRAY(CONVERT(JSON_EXTRACT(j, '$') USING latin1)) AS r + FROM t2 ORDER BY id; + +--echo # +--echo # 18. How a value is FORMATTED, which is a second thing such a column +--echo # can attest to and is come by differently. Being a document is +--echo # promised by the producer before a row exists; the formatting is +--echo # settled one value at a time, so it is granted along with the first +--echo # answer and spent by whatever gets written afterwards. +--echo # +CREATE TABLE t18 (id INT, j JSON) CHARSET utf8mb4; +INSERT INTO t18 VALUES (1, '{"a":1}'), (2, '[1,2]'); +--echo # +--echo # Edited out of a column whose producer writes the loose way, out of +--echo # one that copies a sub-document in whatever way it found it, and out +--echo # of one that writes a form of its own. +--echo # +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d ORDER BY id; +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d ORDER BY id; +SELECT id, JSON_SET(x, '$.c', 1) AS r FROM + (SELECT id, JSON_NORMALIZE(j) AS x FROM t18) d ORDER BY id; +--echo # +--echo # Carried from one such table into the next, and taken back there +--echo # where what the first holds was written some other way. +--echo # +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1) d2 ORDER BY id; +--echo # +--echo # The same across the copier that fills a grouping's table. +--echo # +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 + ORDER BY id; +SELECT id, JSON_SET(y, '$.c', 1) AS r FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d1 GROUP BY id, x) d2 + ORDER BY id; +--echo # +--echo # An aggregate reads its elements out of a record, the column being +--echo # the only thing left there to ask - so what it says about their +--echo # formatting is what the gathered document can say about its own. +--echo # +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t18) d; +SELECT JSON_ARRAY(JSON_ARRAYAGG(x)) AS r FROM + (SELECT id, JSON_QUERY(j, '$') AS x FROM t18) d; +DROP TABLE t18; + +--echo # +--echo # 19. How deep the values went, which is a figure rather than a yes. +--echo # +--echo # What a column keeps is the deepest of the rows written into it so +--echo # far, not the depth of the row being read - more than that row +--echo # needs and never less. More is the direction this one is allowed +--echo # to be wrong in: a figure too large costs a reading that could have +--echo # been skipped, while one too small would admit a document that +--echo # nothing can read back. Nothing acts on the figure except to skip +--echo # a reading, so the answers below are the ones the server has always +--echo # given. +--echo # +SET @k30= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 30), + '1', REPEAT('}', 30)); +SET @k31= CONCAT(REPEAT(CONCAT('{"', REPEAT('k', 10), '":'), 31), + '1', REPEAT('}', 31)); +CREATE TABLE t19 (id INT, j LONGTEXT) CHARSET utf8mb4; +INSERT INTO t19 VALUES (1, @k30), (2, '{"a":1}'); +--echo # +--echo # One row at the limit and a shallow one beside it, put one level +--echo # in. The column answers 30 for both, which is exact for the first +--echo # and too large for the second, and each still ends up where it +--echo # belongs. +--echo # +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +--echo # +--echo # A byte past it, where the same shape is refused - and the shallow +--echo # row in the same column is not, the figure being about the column +--echo # and the limit about each document that comes out of it. +--echo # +UPDATE t19 SET j= @k31 WHERE id= 1; +SELECT id, JSON_DEPTH(JSON_ARRAY(x)) AS d FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d ORDER BY id; +--echo # +--echo # Each level of these is fourteen characters, so half the length is +--echo # seven times the limit and the length can tell neither of them from +--echo # the other. Only the figure the column kept can. +--echo # +SELECT JSON_DEPTH(JSON_ARRAY(y)) AS d FROM + (SELECT id, x AS y FROM + (SELECT id, JSON_EXTRACT(j, '$') AS x FROM t19) d1 WHERE id= 2) d2; +DROP TABLE t19; + +DROP TABLE t2; diff --git a/mysql-test/main/func_json_trusted.result b/mysql-test/main/func_json_trusted.result new file mode 100644 index 0000000000000..efddccfc697c8 --- /dev/null +++ b/mysql-test/main/func_json_trusted.result @@ -0,0 +1,301 @@ +# +# 1. A key both documents hold +# +# Merging writes the value of such a key itself rather than +# copying it out of either document, so the punctuation in +# front of it is the composer's to get right. +# +SELECT JSON_MERGE_PRESERVE('{"a": 1}', '{"a": 2}') AS literal; +literal +{"a": [1, 2]} +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('a', 2)) AS built; +built +{"a": [1, 2]} +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), +JSON_EXTRACT('{"a": 2, "b": 3}', '$')) AS cut; +cut +{"a": [1, 2], "b": 3} +# +# The same where the shared value is a container, which the +# merging walks into and composes a level further down. +# +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_OBJECT('x', 1)), +JSON_OBJECT('a', JSON_OBJECT('y', 2))) AS objects; +objects +{"a": {"x": 1, "y": 2}} +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_ARRAY(1, 2)), +JSON_OBJECT('a', JSON_ARRAY(3, 4))) AS arrays; +arrays +{"a": [1, 2, 3, 4]} +# +# A key only one of them holds is copied across with the +# spacing it was written with, and is the control for the +# three above. +# +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) AS apart; +apart +{"a": 1, "b": 2} +# +# 2. Removing the first element of an array +# +# What is taken out reaches to the comma; the space after it +# belongs to the element that stays, and has to end up in +# front of it and not behind the bracket. +# +SELECT JSON_REMOVE('[1, 2, 3]', '$[0]') AS literal; +literal +[2, 3] +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') AS cut; +cut +[2, 3] +SELECT JSON_REMOVE(JSON_ARRAY(1, 2, 3), '$[0]') AS built; +built +[2, 3] +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": [1, 2, 3]}', '$'), '$.a[0]') AS nested; +nested +{"a": [2, 3]} +SELECT JSON_REMOVE(JSON_EXTRACT('[[1], 2, 3]', '$'), '$[0]') AS container; +container +[2, 3] +# +# Taking out anything other than the first element ends at a +# comma with nothing after it, so those are the controls. +# +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[1]') AS middle; +middle +[1, 3] +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[2]') AS last; +last +[1, 2] +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": 1, "b": 2}', '$'), '$.a') AS first_key; +first_key +{"b": 2} +# +# Where the document is nobody's word, what is composed is not +# the answer but what the reading back at the end is made +# from, and that reading is what complains about anything +# written after the document. It counts from the start of the +# text it was handed, so that text has to be the one a +# released server would have handed it - including the space +# the taking out would otherwise carry off, which that server +# leaves standing and drops later while writing the answer. +# +SELECT JSON_REMOVE('[1, 2] x', '$[0]') AS pos_first; +pos_first +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 6 +SELECT JSON_REMOVE('[1, 2, 3] x', '$[0]') AS pos_first_three; +pos_first_three +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 9 +# +# The controls, none of which has such a space to lose: one +# document written with no spacing at all, one where what is +# taken out is not the first piece, and one where it is a key. +# +SELECT JSON_REMOVE('[1,2] x', '$[0]') AS pos_compact; +pos_compact +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 5 +SELECT JSON_REMOVE('[1, 2] x', '$[1]') AS pos_second; +pos_second +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 5 +SELECT JSON_REMOVE('{"a": 1, "b": 2} x', '$.a') AS pos_key; +pos_key +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_remove' at position 10 +# +# 3. A value put where the document would end up too deep +# +# How deep the value ends up is the level it is put at plus +# its own; the level it is put at is read off the scanner, +# which by then has already left the thing being put into. +# +SET @deep30= CONCAT(REPEAT('[', 30), '{"k": 1}', REPEAT(']', 30)); +SET @newkey= CONCAT('$', REPEAT('[0]', 30), '.n'); +SET @pastend= CONCAT('$', REPEAT('[0]', 30), '[1]'); +SELECT JSON_INSERT(@deep30, @newkey, JSON_EXTRACT('[5]', '$')) +IS NULL AS literal_isnull; +literal_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 44 +SELECT JSON_INSERT(JSON_EXTRACT(@deep30, '$'), @newkey, +JSON_EXTRACT('[5]', '$')) IS NULL AS cut_isnull; +cut_isnull +1 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_insert' at position 3 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 45 +SELECT JSON_SET(JSON_EXTRACT(@deep30, '$'), @newkey, +JSON_EXTRACT('[5]', '$')) IS NULL AS set_isnull; +set_isnull +1 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_set' at position 3 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_set' at position 45 +SET @arr30= CONCAT(REPEAT('[', 30), '[1]', REPEAT(']', 30)); +SELECT JSON_INSERT(@arr30, @pastend, JSON_EXTRACT('[5]', '$')) +IS NULL AS end_literal_isnull; +end_literal_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 35 +SELECT JSON_INSERT(JSON_EXTRACT(@arr30, '$'), @pastend, +JSON_EXTRACT('[5]', '$')) IS NULL AS end_cut_isnull; +end_cut_isnull +1 +Warnings: +Note 4040 Limit of 32 on JSON nested structures depth is reached in argument 3 to function 'json_insert' at position 3 +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 35 +# +# One level shallower, the same statements answer, and that +# is what says the two above are refused for their depth and +# not for their shape. +# +SET @deep20= CONCAT(REPEAT('[', 20), '{"k": 1}', REPEAT(']', 20)); +SET @newkey20= CONCAT('$', REPEAT('[0]', 20), '.n'); +SELECT JSON_INSERT(JSON_EXTRACT(@deep20, '$'), @newkey20, +JSON_EXTRACT('[5]', '$')) IS NULL AS shallow_isnull; +shallow_isnull +0 +# +# And what it actually composed, which is where a document +# that came out a character longer or shorter than a released +# server's would show itself. Asking only whether an answer +# came back would not. +# +SELECT JSON_INSERT(JSON_EXTRACT('[[{"k": 1}]]', '$'), '$[0][0].n', +JSON_EXTRACT('[5]', '$')) AS composed; +composed +[[{"k": 1, "n": [5]}]] +SELECT JSON_INSERT(JSON_EXTRACT('[[1, 2]]', '$'), '$[0][2]', +JSON_EXTRACT('[5]', '$')) AS composed_past_end; +composed_past_end +[[1, 2, [5]]] +SELECT JSON_SET(JSON_EXTRACT('{"a": {"b": 1}}', '$'), '$.a.c', +JSON_EXTRACT('{"z": 9}', '$')) AS composed_set; +composed_set +{"a": {"b": 1, "c": {"z": 9}}} +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', +JSON_EXTRACT('[3, 4]', '$')) AS composed_append; +composed_append +[1, 2, [3, 4]] +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', +JSON_EXTRACT('[3, 4]', '$')) AS composed_ainsert; +composed_ainsert +[[3, 4], 1, 2] +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('{"a": 1}', '$'), '$', +JSON_EXTRACT('[3, 4]', '$')) AS composed_autowrap; +composed_autowrap +[{"a": 1}, [3, 4]] +SELECT JSON_EXTRACT('{"a": [1, 2], "b": {"c": 3}}', '$.a', '$.b') +AS composed_two_paths; +composed_two_paths +[[1, 2], {"c": 3}] +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), JSON_EXTRACT('2', '$')) +AS composed_merge_wrap; +composed_merge_wrap +[{"a": 1}, 2] +# +# 4. The bracket a reading of more than one path adds +# +# Two matches are returned wrapped in an array, and that +# wrapper is a level nobody asked the document about. +# +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SELECT JSON_VALID(@d31) AS legal_on_its_own; +legal_on_its_own +1 +SELECT JSON_EXTRACT(@d31, '$') IS NULL AS one_path_isnull; +one_path_isnull +0 +SELECT JSON_EXTRACT(@d31, '$', '$') IS NULL AS two_paths_isnull; +two_paths_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_extract' at position 32 +SET @d30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_EXTRACT(@d30, '$', '$') IS NULL AS two_paths_fit_isnull; +two_paths_fit_isnull +0 +# +# 5. The bracket a merge adds +# +# Merging a document with something it cannot be merged into +# puts both in a new array, which is a level as well. +# +SET @o31= CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SELECT JSON_VALID(@o31) AS legal_on_its_own; +legal_on_its_own +1 +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('1', '$')) +IS NULL AS scalar_wrap_isnull; +scalar_wrap_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 182 +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('{"a": 1}', '$')) +IS NULL AS shared_key_isnull; +shared_key_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_merge_preserve' at position 182 +SET @o30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SELECT JSON_MERGE(JSON_EXTRACT(@o30, '$'), JSON_EXTRACT('1', '$')) +IS NULL AS wrap_fits_isnull; +wrap_fits_isnull +0 +# +# 6. The value an autowrap keeps +# +# Appending to something that is not an array wraps what is +# already there, so the KEPT value goes a level down too - +# not only the one being appended. +# +SET @p30= CONCAT('$', REPEAT('.a', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o31, '$'), @p30, 1) +IS NULL AS append_wrap_isnull; +append_wrap_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_array_append' at position 182 +SELECT JSON_INSERT(JSON_EXTRACT(@o31, '$'), CONCAT(@p30, '[1]'), 1) +IS NULL AS insert_wrap_isnull; +insert_wrap_isnull +1 +Warnings: +Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_insert' at position 182 +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o30, '$'), @p29, 1) +IS NULL AS append_wrap_fits_isnull; +append_wrap_fits_isnull +0 +# +# 7. A document that fits, and the diagnostic it must not get +# +# The level a value is appended at is read off the scanner +# after it has already been counted, so counting it again +# refuses a document that fits - or lets it through with a +# note saying it did not fit, which is worse, the note being +# the only thing an application would go by. +# +SET @nest30= CONCAT(REPEAT('[', 30), REPEAT(']', 30)); +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_EXTRACT(@nest30, '$')) +IS NULL AS literal_isnull; +literal_isnull +0 +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1]', '$'), '$', +JSON_EXTRACT(@nest30, '$')) IS NULL AS cut_isnull; +cut_isnull +0 +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1]', '$'), '$[0]', +JSON_EXTRACT(@nest30, '$')) IS NULL AS insert_isnull; +insert_isnull +0 diff --git a/mysql-test/main/func_json_trusted.test b/mysql-test/main/func_json_trusted.test new file mode 100644 index 0000000000000..57f8c8b7d8c37 --- /dev/null +++ b/mysql-test/main/func_json_trusted.test @@ -0,0 +1,230 @@ +# +# The seven functions that used to read their own answer back now hand +# it as it was composed, whenever the item behind every document +# they were given attests is_valid and is_nice. +# +# Nothing else in the suite goes down that path. A string literal +# attests is_valid false for its document, so the reading back still +# happens and the composition is never the answer; a suite built out of +# literals can be entirely green while the composing is entirely wrong. +# Every case here is therefore built from a producer that attests to +# what it passes - JSON_OBJECT, JSON_ARRAY or JSON_EXTRACT - and is +# paired with the same statement written with a literal, which is what +# a released server runs. The pair has to agree. +# +# The answers in this file were recorded on a released 10.11 server, so +# what they say is what that server says, not what this one does. Five +# lines of it are not, and they are the only five: +# +# - Three of the refusals below carry an added Note naming the argument +# whose value would not fit, in front of the warning a released +# server gives on its own. Nothing is said less; something is said +# more, and what it says is true. +# +# - Two warning positions differ by one character from a released +# server's. A position is an offset into +# the text that was composed, and the text composed here is already +# written the loose way where a released server composed it compact +# and tidied it afterwards. Since these are the documents that are +# refused, the tidying never happened there, so its offsets are +# into a document that server threw away. Ours are into the one it +# would have returned. +# +# Every answer, and every one of the compositions written out below in +# full, is byte for byte what a released server gives. +# + +--enable_warnings + +--echo # +--echo # 1. A key both documents hold +--echo # +--echo # Merging writes the value of such a key itself rather than +--echo # copying it out of either document, so the punctuation in +--echo # front of it is the composer's to get right. +--echo # +SELECT JSON_MERGE_PRESERVE('{"a": 1}', '{"a": 2}') AS literal; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('a', 2)) AS built; +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), + JSON_EXTRACT('{"a": 2, "b": 3}', '$')) AS cut; + +--echo # +--echo # The same where the shared value is a container, which the +--echo # merging walks into and composes a level further down. +--echo # +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_OBJECT('x', 1)), + JSON_OBJECT('a', JSON_OBJECT('y', 2))) AS objects; +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', JSON_ARRAY(1, 2)), + JSON_OBJECT('a', JSON_ARRAY(3, 4))) AS arrays; + +--echo # +--echo # A key only one of them holds is copied across with the +--echo # spacing it was written with, and is the control for the +--echo # three above. +--echo # +SELECT JSON_MERGE_PRESERVE(JSON_OBJECT('a', 1), JSON_OBJECT('b', 2)) AS apart; + +--echo # +--echo # 2. Removing the first element of an array +--echo # +--echo # What is taken out reaches to the comma; the space after it +--echo # belongs to the element that stays, and has to end up in +--echo # front of it and not behind the bracket. +--echo # +SELECT JSON_REMOVE('[1, 2, 3]', '$[0]') AS literal; +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') AS cut; +SELECT JSON_REMOVE(JSON_ARRAY(1, 2, 3), '$[0]') AS built; +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": [1, 2, 3]}', '$'), '$.a[0]') AS nested; +SELECT JSON_REMOVE(JSON_EXTRACT('[[1], 2, 3]', '$'), '$[0]') AS container; + +--echo # +--echo # Taking out anything other than the first element ends at a +--echo # comma with nothing after it, so those are the controls. +--echo # +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[1]') AS middle; +SELECT JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[2]') AS last; +SELECT JSON_REMOVE(JSON_EXTRACT('{"a": 1, "b": 2}', '$'), '$.a') AS first_key; + +--echo # +--echo # Where the document is nobody's word, what is composed is not +--echo # the answer but what the reading back at the end is made +--echo # from, and that reading is what complains about anything +--echo # written after the document. It counts from the start of the +--echo # text it was handed, so that text has to be the one a +--echo # released server would have handed it - including the space +--echo # the taking out would otherwise carry off, which that server +--echo # leaves standing and drops later while writing the answer. +--echo # +SELECT JSON_REMOVE('[1, 2] x', '$[0]') AS pos_first; +SELECT JSON_REMOVE('[1, 2, 3] x', '$[0]') AS pos_first_three; + +--echo # +--echo # The controls, none of which has such a space to lose: one +--echo # document written with no spacing at all, one where what is +--echo # taken out is not the first piece, and one where it is a key. +--echo # +SELECT JSON_REMOVE('[1,2] x', '$[0]') AS pos_compact; +SELECT JSON_REMOVE('[1, 2] x', '$[1]') AS pos_second; +SELECT JSON_REMOVE('{"a": 1, "b": 2} x', '$.a') AS pos_key; + +--echo # +--echo # 3. A value put where the document would end up too deep +--echo # +--echo # How deep the value ends up is the level it is put at plus +--echo # its own; the level it is put at is read off the scanner, +--echo # which by then has already left the thing being put into. +--echo # +SET @deep30= CONCAT(REPEAT('[', 30), '{"k": 1}', REPEAT(']', 30)); +SET @newkey= CONCAT('$', REPEAT('[0]', 30), '.n'); +SET @pastend= CONCAT('$', REPEAT('[0]', 30), '[1]'); + +SELECT JSON_INSERT(@deep30, @newkey, JSON_EXTRACT('[5]', '$')) + IS NULL AS literal_isnull; +SELECT JSON_INSERT(JSON_EXTRACT(@deep30, '$'), @newkey, + JSON_EXTRACT('[5]', '$')) IS NULL AS cut_isnull; +SELECT JSON_SET(JSON_EXTRACT(@deep30, '$'), @newkey, + JSON_EXTRACT('[5]', '$')) IS NULL AS set_isnull; + +SET @arr30= CONCAT(REPEAT('[', 30), '[1]', REPEAT(']', 30)); +SELECT JSON_INSERT(@arr30, @pastend, JSON_EXTRACT('[5]', '$')) + IS NULL AS end_literal_isnull; +SELECT JSON_INSERT(JSON_EXTRACT(@arr30, '$'), @pastend, + JSON_EXTRACT('[5]', '$')) IS NULL AS end_cut_isnull; + +--echo # +--echo # One level shallower, the same statements answer, and that +--echo # is what says the two above are refused for their depth and +--echo # not for their shape. +--echo # +SET @deep20= CONCAT(REPEAT('[', 20), '{"k": 1}', REPEAT(']', 20)); +SET @newkey20= CONCAT('$', REPEAT('[0]', 20), '.n'); +SELECT JSON_INSERT(JSON_EXTRACT(@deep20, '$'), @newkey20, + JSON_EXTRACT('[5]', '$')) IS NULL AS shallow_isnull; + +--echo # +--echo # And what it actually composed, which is where a document +--echo # that came out a character longer or shorter than a released +--echo # server's would show itself. Asking only whether an answer +--echo # came back would not. +--echo # +SELECT JSON_INSERT(JSON_EXTRACT('[[{"k": 1}]]', '$'), '$[0][0].n', + JSON_EXTRACT('[5]', '$')) AS composed; +SELECT JSON_INSERT(JSON_EXTRACT('[[1, 2]]', '$'), '$[0][2]', + JSON_EXTRACT('[5]', '$')) AS composed_past_end; +SELECT JSON_SET(JSON_EXTRACT('{"a": {"b": 1}}', '$'), '$.a.c', + JSON_EXTRACT('{"z": 9}', '$')) AS composed_set; +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', + JSON_EXTRACT('[3, 4]', '$')) AS composed_append; +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', + JSON_EXTRACT('[3, 4]', '$')) AS composed_ainsert; +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('{"a": 1}', '$'), '$', + JSON_EXTRACT('[3, 4]', '$')) AS composed_autowrap; +SELECT JSON_EXTRACT('{"a": [1, 2], "b": {"c": 3}}', '$.a', '$.b') + AS composed_two_paths; +SELECT JSON_MERGE(JSON_EXTRACT('{"a": 1}', '$'), JSON_EXTRACT('2', '$')) + AS composed_merge_wrap; + +--echo # +--echo # 4. The bracket a reading of more than one path adds +--echo # +--echo # Two matches are returned wrapped in an array, and that +--echo # wrapper is a level nobody asked the document about. +--echo # +SET @d31= CONCAT(REPEAT('[', 31), '1', REPEAT(']', 31)); +SELECT JSON_VALID(@d31) AS legal_on_its_own; +SELECT JSON_EXTRACT(@d31, '$') IS NULL AS one_path_isnull; +SELECT JSON_EXTRACT(@d31, '$', '$') IS NULL AS two_paths_isnull; + +SET @d30= CONCAT(REPEAT('[', 30), '1', REPEAT(']', 30)); +SELECT JSON_EXTRACT(@d30, '$', '$') IS NULL AS two_paths_fit_isnull; + +--echo # +--echo # 5. The bracket a merge adds +--echo # +--echo # Merging a document with something it cannot be merged into +--echo # puts both in a new array, which is a level as well. +--echo # +SET @o31= CONCAT(REPEAT('{"a":', 31), '1', REPEAT('}', 31)); +SELECT JSON_VALID(@o31) AS legal_on_its_own; +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('1', '$')) + IS NULL AS scalar_wrap_isnull; +SELECT JSON_MERGE(JSON_EXTRACT(@o31, '$'), JSON_EXTRACT('{"a": 1}', '$')) + IS NULL AS shared_key_isnull; + +SET @o30= CONCAT(REPEAT('{"a":', 30), '1', REPEAT('}', 30)); +SELECT JSON_MERGE(JSON_EXTRACT(@o30, '$'), JSON_EXTRACT('1', '$')) + IS NULL AS wrap_fits_isnull; + +--echo # +--echo # 6. The value an autowrap keeps +--echo # +--echo # Appending to something that is not an array wraps what is +--echo # already there, so the KEPT value goes a level down too - +--echo # not only the one being appended. +--echo # +SET @p30= CONCAT('$', REPEAT('.a', 30)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o31, '$'), @p30, 1) + IS NULL AS append_wrap_isnull; +SELECT JSON_INSERT(JSON_EXTRACT(@o31, '$'), CONCAT(@p30, '[1]'), 1) + IS NULL AS insert_wrap_isnull; + +SET @p29= CONCAT('$', REPEAT('.a', 29)); +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT(@o30, '$'), @p29, 1) + IS NULL AS append_wrap_fits_isnull; + +--echo # +--echo # 7. A document that fits, and the diagnostic it must not get +--echo # +--echo # The level a value is appended at is read off the scanner +--echo # after it has already been counted, so counting it again +--echo # refuses a document that fits - or lets it through with a +--echo # note saying it did not fit, which is worse, the note being +--echo # the only thing an application would go by. +--echo # +SET @nest30= CONCAT(REPEAT('[', 30), REPEAT(']', 30)); +SELECT JSON_ARRAY_APPEND('[1]', '$', JSON_EXTRACT(@nest30, '$')) + IS NULL AS literal_isnull; +SELECT JSON_ARRAY_APPEND(JSON_EXTRACT('[1]', '$'), '$', + JSON_EXTRACT(@nest30, '$')) IS NULL AS cut_isnull; +SELECT JSON_ARRAY_INSERT(JSON_EXTRACT('[1]', '$'), '$[0]', + JSON_EXTRACT(@nest30, '$')) IS NULL AS insert_isnull; diff --git a/mysql-test/main/func_json_unfinished.result b/mysql-test/main/func_json_unfinished.result new file mode 100644 index 0000000000000..e7774234a2dcf --- /dev/null +++ b/mysql-test/main/func_json_unfinished.result @@ -0,0 +1,76 @@ +# +# A path counted from the end of an array in a document that does +# not end. +# +# A step written with 'last' is not known until the array it is +# counted in has been counted, so a document carrying one is read +# through twice: once to find out how many elements each array +# holds, and once to answer. The counting pass can run off the +# end of a document that stops in the middle, and what it finds +# there has to be carried out of the loop rather than left to the +# answering pass, which would otherwise answer about an array it +# never finished counting. +# +SET NAMES utf8mb4; +# +# 1. The counting pass reaching the end of the text +# +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'one', '$.a[last]') AS one_path; +one_path +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'all', '$.a[last-1]', '$.a[0]') +AS all_paths; +all_paths +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'one', '$[1][last]') AS nested; +nested +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'all', '$[0][last]', '$[1][last]') +AS nested_all; +nested_all +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_contains_path' +# +# 2. The same documents finished, which is what the answers above +# are read against +# +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'one', '$.a[last]') AS one_path; +one_path +1 +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'all', '$.a[last-1]', '$.a[0]') +AS all_paths; +all_paths +1 +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'one', '$[1][last]') AS nested; +nested +1 +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'all', '$[0][last]', '$[1][last]') +AS nested_all; +nested_all +1 +# +# 3. The reading functions over the same shapes +# +SELECT JSON_EXTRACT('{"a":[1,2,3', '$.a[last]') AS unfinished; +unfinished +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":[1,2,3]}', '$.a[last]') AS finished; +finished +3 +SELECT JSON_EXTRACT('[[1,2],[3', '$[1][last]') AS unfinished; +unfinished +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('[[1,2],[3]]', '$[1][last]') AS finished; +finished +3 diff --git a/mysql-test/main/func_json_unfinished.test b/mysql-test/main/func_json_unfinished.test new file mode 100644 index 0000000000000..94f2d3c1505be --- /dev/null +++ b/mysql-test/main/func_json_unfinished.test @@ -0,0 +1,44 @@ +--echo # +--echo # A path counted from the end of an array in a document that does +--echo # not end. +--echo # +--echo # A step written with 'last' is not known until the array it is +--echo # counted in has been counted, so a document carrying one is read +--echo # through twice: once to find out how many elements each array +--echo # holds, and once to answer. The counting pass can run off the +--echo # end of a document that stops in the middle, and what it finds +--echo # there has to be carried out of the loop rather than left to the +--echo # answering pass, which would otherwise answer about an array it +--echo # never finished counting. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. The counting pass reaching the end of the text +--echo # +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'one', '$.a[last]') AS one_path; +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3', 'all', '$.a[last-1]', '$.a[0]') + AS all_paths; +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'one', '$[1][last]') AS nested; +SELECT JSON_CONTAINS_PATH('[[1,2],[3', 'all', '$[0][last]', '$[1][last]') + AS nested_all; + +--echo # +--echo # 2. The same documents finished, which is what the answers above +--echo # are read against +--echo # +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'one', '$.a[last]') AS one_path; +SELECT JSON_CONTAINS_PATH('{"a":[1,2,3]}', 'all', '$.a[last-1]', '$.a[0]') + AS all_paths; +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'one', '$[1][last]') AS nested; +SELECT JSON_CONTAINS_PATH('[[1,2],[3]]', 'all', '$[0][last]', '$[1][last]') + AS nested_all; + +--echo # +--echo # 3. The reading functions over the same shapes +--echo # +SELECT JSON_EXTRACT('{"a":[1,2,3', '$.a[last]') AS unfinished; +SELECT JSON_EXTRACT('{"a":[1,2,3]}', '$.a[last]') AS finished; +SELECT JSON_EXTRACT('[[1,2],[3', '$[1][last]') AS unfinished; +SELECT JSON_EXTRACT('[[1,2],[3]]', '$[1][last]') AS finished; diff --git a/mysql-test/main/func_json_unquote.result b/mysql-test/main/func_json_unquote.result new file mode 100644 index 0000000000000..a7a158db8fa06 --- /dev/null +++ b/mysql-test/main/func_json_unquote.result @@ -0,0 +1,159 @@ +# +# What JSON_UNQUOTE reports its result to be, against what it +# actually returns. +# +# The function settles on utf8mb4_bin when the statement is prepared, +# before any argument has been read, so that is what every consumer +# downstream of it is told the bytes are. The bytes therefore have to +# arrive in that character set no matter what the argument was +# written in. Only one of the ways out of the function converts them. +# +SET NAMES utf8mb4; +# +# 1. What the function says about itself. +# +SELECT CHARSET(JSON_UNQUOTE(CONVERT('["a"]' USING latin1))) AS declared_cs, +COLLATION(JSON_UNQUOTE('["a"]')) AS declared_coll; +declared_cs declared_coll +utf8mb4 utf8mb4_bin +# +# 2. A string value. This is the path that unescapes, and it +# converts on the way out. +# +SELECT HEX(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS str_value; +str_value +C3A9 +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS chars; +chars +1 +# +# 3. A value that is not a string. There is nothing to unquote, so +# the argument is returned unchanged. +# +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS array_value; +array_value +5B22C3A9225D +SELECT HEX(JSON_UNQUOTE(CONVERT(X'7B2261223A22E9227D' USING latin1))) AS object_value; +object_value +7B2261223A22C3A9227D +SELECT HEX(JSON_UNQUOTE(CONVERT('123' USING latin1))) AS number_value; +number_value +313233 +SELECT HEX(JSON_UNQUOTE(CONVERT('true' USING latin1))) AS bool_value; +bool_value +74727565 +# +# 4. A document that does not parse. Also returned unchanged. +# +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E922' USING latin1))) AS broken_value; +broken_value +5B22C3A922 +# +# 5. The result used where its declared character set is taken at +# its word. +# +SELECT HEX(CONCAT(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)), 'z')) AS concatenated; +concatenated +5B22C3A9225D7A +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS chars; +chars +5 +SELECT JSON_VALID(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS still_json; +still_json +1 +SELECT HEX(UPPER(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)))) AS uppercased; +uppercased +5B22C389225D +# +# 6. A document held as bytes, which has no character set to +# convert from. +# +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22FF225D' AS BINARY))) AS from_binary; +from_binary +5B22C3BF225D +SELECT HEX(JSON_UNQUOTE(CAST(X'5B226122' AS BINARY))) AS from_binary_broken; +from_binary_broken +5B226122 +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22615D' AS BINARY))) AS from_binary_ascii; +from_binary_ascii +5B22615D +# the two ways out of the function have to say the same thing about +# the same byte. A value that IS a string is unescaped, reading each +# byte as the character of the same number; anything else is handed +# back converted, and it has to be read the same way. 0x80 and 0x9F +# are two of the 27 positions in 0x80-0x9F where latin1 would not: +# MariaDB's latin1 is cp1252 and makes 0x80 the euro sign. +SELECT HEX(JSON_UNQUOTE(CAST(X'228022' AS BINARY))) AS str_80; +str_80 +C280 +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2280225D' AS BINARY))) AS doc_80; +doc_80 +5B22C280225D +SELECT HEX(JSON_UNQUOTE(CAST(X'229F22' AS BINARY))) AS str_9f; +str_9f +C29F +SELECT HEX(JSON_UNQUOTE(CAST(X'5B229F225D' AS BINARY))) AS doc_9f; +doc_9f +5B22C29F225D +# and one of the five where it would agree, for contrast +SELECT HEX(JSON_UNQUOTE(CAST(X'228122' AS BINARY))) AS str_81; +str_81 +C281 +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2281225D' AS BINARY))) AS doc_81; +doc_81 +5B22C281225D +# +# 7. An argument already in the declared character set, which is +# the common case and must not move. +# +SELECT HEX(JSON_UNQUOTE(_utf8mb4'["a"]')) AS same_cs; +same_cs +5B2261225D +SELECT HEX(JSON_UNQUOTE(_utf8mb4'"a"')) AS same_cs_str; +same_cs_str +61 +# a document holding U+00E9, written here as its utf8mb4 bytes C3 A9 +SELECT HEX(JSON_UNQUOTE(_utf8mb4 0x5B22C3A9225D)) AS same_cs_wide; +same_cs_wide +5B22C3A9225D +# +# 8. An argument in a character set that admits every byte but has +# no character at some of them. +# +# The conversion has nowhere to put such a byte. What it does put +# there instead is not the value it was given, so the argument is +# returned as it arrived, which is the answer this call has +# always given, and the difference is said in a note. A note and +# not a warning: a warning becomes an error inside a statement +# running under strict mode, and a statement that used to finish +# would then stop finishing. +# +# cp1250 has no character at 0x81. +# +CREATE TABLE t1 (id INT, c TEXT CHARACTER SET cp1250); +INSERT INTO t1 VALUES (1, X'5B3181'), (2, X'5B2261225D'), (3, X'2281'); +SELECT id, HEX(c) AS stored, HEX(JSON_UNQUOTE(c)) AS unquoted +FROM t1 ORDER BY id; +id stored unquoted +1 5B3181 5B3181 +2 5B2261225D 5B2261225D +3 2281 2281 +Warnings: +Note 4035 Broken JSON string in argument 0 to function 'unquote' at position 0 +Warning 4035 Broken JSON string in argument 1 to function 'json_unquote' at position 1 +Note 4035 Broken JSON string in argument 0 to function 'unquote' at position 0 +# the same values inside a statement that stops on a warning. Row 3 +# is left out of it: reading a STRING that will not scan has always +# raised a warning of its own, from before any of this, and strict +# mode has always stopped on that one. +CREATE TABLE t2 (b BLOB); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t2 SELECT JSON_UNQUOTE(c) FROM t1 WHERE id < 3; +Warnings: +Note 4035 Broken JSON string in argument 0 to function 'unquote' at position 0 +SELECT HEX(b) AS stored FROM t2 ORDER BY b; +stored +5B2261225D +5B3181 +SET SESSION sql_mode=DEFAULT; +DROP TABLE t1, t2; diff --git a/mysql-test/main/func_json_unquote.test b/mysql-test/main/func_json_unquote.test new file mode 100644 index 0000000000000..66974764e7029 --- /dev/null +++ b/mysql-test/main/func_json_unquote.test @@ -0,0 +1,120 @@ +--source include/have_utf8mb4.inc + +--echo # +--echo # What JSON_UNQUOTE reports its result to be, against what it +--echo # actually returns. +--echo # +--echo # The function settles on utf8mb4_bin when the statement is prepared, +--echo # before any argument has been read, so that is what every consumer +--echo # downstream of it is told the bytes are. The bytes therefore have to +--echo # arrive in that character set no matter what the argument was +--echo # written in. Only one of the ways out of the function converts them. +--echo # + +SET NAMES utf8mb4; + +--echo # +--echo # 1. What the function says about itself. +--echo # + +SELECT CHARSET(JSON_UNQUOTE(CONVERT('["a"]' USING latin1))) AS declared_cs, + COLLATION(JSON_UNQUOTE('["a"]')) AS declared_coll; + +--echo # +--echo # 2. A string value. This is the path that unescapes, and it +--echo # converts on the way out. +--echo # + +SELECT HEX(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS str_value; +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'22E922' USING latin1))) AS chars; + +--echo # +--echo # 3. A value that is not a string. There is nothing to unquote, so +--echo # the argument is returned unchanged. +--echo # + +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS array_value; +SELECT HEX(JSON_UNQUOTE(CONVERT(X'7B2261223A22E9227D' USING latin1))) AS object_value; +SELECT HEX(JSON_UNQUOTE(CONVERT('123' USING latin1))) AS number_value; +SELECT HEX(JSON_UNQUOTE(CONVERT('true' USING latin1))) AS bool_value; + +--echo # +--echo # 4. A document that does not parse. Also returned unchanged. +--echo # + +SELECT HEX(JSON_UNQUOTE(CONVERT(X'5B22E922' USING latin1))) AS broken_value; + +--echo # +--echo # 5. The result used where its declared character set is taken at +--echo # its word. +--echo # + +SELECT HEX(CONCAT(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)), 'z')) AS concatenated; +SELECT CHAR_LENGTH(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS chars; +SELECT JSON_VALID(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1))) AS still_json; +SELECT HEX(UPPER(JSON_UNQUOTE(CONVERT(X'5B22E9225D' USING latin1)))) AS uppercased; + +--echo # +--echo # 6. A document held as bytes, which has no character set to +--echo # convert from. +--echo # + +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22FF225D' AS BINARY))) AS from_binary; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B226122' AS BINARY))) AS from_binary_broken; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B22615D' AS BINARY))) AS from_binary_ascii; + +--echo # the two ways out of the function have to say the same thing about +--echo # the same byte. A value that IS a string is unescaped, reading each +--echo # byte as the character of the same number; anything else is handed +--echo # back converted, and it has to be read the same way. 0x80 and 0x9F +--echo # are two of the 27 positions in 0x80-0x9F where latin1 would not: +--echo # MariaDB's latin1 is cp1252 and makes 0x80 the euro sign. +SELECT HEX(JSON_UNQUOTE(CAST(X'228022' AS BINARY))) AS str_80; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2280225D' AS BINARY))) AS doc_80; +SELECT HEX(JSON_UNQUOTE(CAST(X'229F22' AS BINARY))) AS str_9f; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B229F225D' AS BINARY))) AS doc_9f; +--echo # and one of the five where it would agree, for contrast +SELECT HEX(JSON_UNQUOTE(CAST(X'228122' AS BINARY))) AS str_81; +SELECT HEX(JSON_UNQUOTE(CAST(X'5B2281225D' AS BINARY))) AS doc_81; + +--echo # +--echo # 7. An argument already in the declared character set, which is +--echo # the common case and must not move. +--echo # + +SELECT HEX(JSON_UNQUOTE(_utf8mb4'["a"]')) AS same_cs; +SELECT HEX(JSON_UNQUOTE(_utf8mb4'"a"')) AS same_cs_str; +--echo # a document holding U+00E9, written here as its utf8mb4 bytes C3 A9 +SELECT HEX(JSON_UNQUOTE(_utf8mb4 0x5B22C3A9225D)) AS same_cs_wide; + +--echo # +--echo # 8. An argument in a character set that admits every byte but has +--echo # no character at some of them. +--echo # +--echo # The conversion has nowhere to put such a byte. What it does put +--echo # there instead is not the value it was given, so the argument is +--echo # returned as it arrived, which is the answer this call has +--echo # always given, and the difference is said in a note. A note and +--echo # not a warning: a warning becomes an error inside a statement +--echo # running under strict mode, and a statement that used to finish +--echo # would then stop finishing. +--echo # +--echo # cp1250 has no character at 0x81. +--echo # + +CREATE TABLE t1 (id INT, c TEXT CHARACTER SET cp1250); +INSERT INTO t1 VALUES (1, X'5B3181'), (2, X'5B2261225D'), (3, X'2281'); +SELECT id, HEX(c) AS stored, HEX(JSON_UNQUOTE(c)) AS unquoted + FROM t1 ORDER BY id; + +--echo # the same values inside a statement that stops on a warning. Row 3 +--echo # is left out of it: reading a STRING that will not scan has always +--echo # raised a warning of its own, from before any of this, and strict +--echo # mode has always stopped on that one. +CREATE TABLE t2 (b BLOB); +SET SESSION sql_mode='STRICT_ALL_TABLES'; +INSERT INTO t2 SELECT JSON_UNQUOTE(c) FROM t1 WHERE id < 3; +SELECT HEX(b) AS stored FROM t2 ORDER BY b; +SET SESSION sql_mode=DEFAULT; + +DROP TABLE t1, t2; diff --git a/mysql-test/main/func_json_valid_constraint.result b/mysql-test/main/func_json_valid_constraint.result new file mode 100644 index 0000000000000..b3075dc614aea --- /dev/null +++ b/mysql-test/main/func_json_valid_constraint.result @@ -0,0 +1,271 @@ +# +# Which column a JSON_VALID check constraint makes a JSON column +# +# A string column is a JSON column when it carries a check constraint +# that says that column holds a document. A constraint that reads +# another column, or no column at all, says nothing about this one. +# +SET NAMES utf8mb4; +# +# A constraint reading another column +# +# d is the ordinary usage and e an ordinary JSON column, both controls. +CREATE TABLE t1 ( +a VARCHAR(100) CHECK (JSON_VALID(b)), +b VARCHAR(100), +d VARCHAR(100) CHECK (JSON_VALID(d)), +e JSON +); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` varchar(100) DEFAULT NULL CHECK (json_valid(`b`)), + `b` varchar(100) DEFAULT NULL, + `d` varchar(100) DEFAULT NULL CHECK (json_valid(`d`)), + `e` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`e`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +# what the server tells a client the columns are +SELECT a, b, d, e FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t1 t1 a a 253 400 0 Y 0 0 45 +def test t1 t1 b b 253 400 0 Y 0 0 45 +def test t1 t1 d d 253 (format=json) 400 0 Y 0 0 45 +def test t1 t1 e e 252 (format=json) 4294967295 0 Y 144 0 45 +a b d e +SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1' ORDER BY COLUMN_NAME; +COLUMN_NAME DATA_TYPE +a varchar +b varchar +d varchar +e longtext +# +# a is unconstrained, so it can hold text that is not a document +# +INSERT INTO t1 VALUES ('not json at all', '{"x":1}', '{"y":2}', '{"z":3}'); +SELECT a, JSON_VALID(a) AS a_is_a_document FROM t1; +a a_is_a_document +not json at all 0 +Warnings: +Note 4038 Syntax error in JSON text in argument 1 to function 'json_valid' at position 1 +# a is quoted like the ordinary column it is, not spliced +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t1; +from_a +{"k": "not json at all"} +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t1; +from_b +{"k": "{\"x\":1}"} +# the controls are spliced +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t1; +from_d +{"k": {"y": 2}} +SELECT JSON_SET('{}', '$.k', e) AS from_e FROM t1; +from_e +{"k": {"z": 3}} +DROP TABLE t1; +# +# The same column holding a document that IS well formed +# +# Whether a is spliced or quoted decides whether the document nests +# or arrives as a string, and it must not depend on b's constraint. +# +CREATE TABLE t2 ( +a VARCHAR(100) CHECK (JSON_VALID(b)), +b VARCHAR(100), +d VARCHAR(100) CHECK (JSON_VALID(d)) +); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +INSERT INTO t2 VALUES ('{"p":1}', '{"x":1}', '{"p":1}'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t2; +from_a +{"k": "{\"p\":1}"} +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t2; +from_b +{"k": "{\"x\":1}"} +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t2; +from_d +{"k": {"p": 1}} +DROP TABLE t2; +# +# A constraint that constrains nothing +# +# JSON_VALID('{}') is a constant true, so the column keeps no +# promise about its contents whatsoever. +# +CREATE TABLE t3 (a VARCHAR(100) CHECK (JSON_VALID('{}'))); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +SELECT a FROM t3; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t3 t3 a a 253 400 0 Y 0 0 45 +a +INSERT INTO t3 VALUES ('not json at all'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t3; +from_a +{"k": "not json at all"} +DROP TABLE t3; +# +# The constraint is still enforced, on the column it names +# +CREATE TABLE t4 (a VARCHAR(100) CHECK (JSON_VALID(b)), b VARCHAR(100)); +Warnings: +Warning 4269 CHECK constraint of column 'a' calls JSON_VALID() on something other than 'a'; the column is not a JSON column +INSERT INTO t4 VALUES ('anything', 'not json'); +ERROR 23000: CONSTRAINT `t4.a` failed for `test`.`t4` +INSERT INTO t4 VALUES ('anything', '{"x":1}'); +SELECT a, b FROM t4; +a b +anything {"x":1} +DROP TABLE t4; +# +# A table level constraint types neither column, as before +# +CREATE TABLE t5 ( +a VARCHAR(100), +b VARCHAR(100), +CONSTRAINT c CHECK (JSON_VALID(b)) +); +SELECT a, b FROM t5; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t5 t5 a a 253 400 0 Y 0 0 45 +def test t5 t5 b b 253 400 0 Y 0 0 45 +a b +DROP TABLE t5; +# +# A check that asks more than whether the column holds a document +# +# What types a column is JSON_VALID() of it and nothing else. An +# AND keeps the promise and an OR does not, but a promise kept is +# not a column typed, and neither shape is typed. Each is told so, +# in the words that fit it. +# +CREATE TABLE t7 ( +a TEXT CHECK (LENGTH(a) > 0 AND JSON_VALID(a)), +b TEXT CHECK (LENGTH(b) > 0 OR JSON_VALID(b)), +c TEXT CHECK (LENGTH(d) > 0 AND JSON_VALID(d)), +d TEXT +); +Warnings: +Warning 4271 CHECK constraint of column 'a' asks more than JSON_VALID() of it; the column is not a JSON column +Warning 4270 CHECK constraint of column 'b' can pass without JSON_VALID() holding; the column is not a JSON column +Warning 4269 CHECK constraint of column 'c' calls JSON_VALID() on something other than 'c'; the column is not a JSON column +CREATE TABLE t8 (d TEXT CHECK (JSON_VALID(d))); +# the protocol tells the client what the warnings told the writer: +# none of the three is a JSON column, and a column whose check is +# the call alone still is +SELECT a, b, c, d FROM t7; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t7 t7 a a 252 262140 0 Y 16 0 45 +def test t7 t7 b b 252 262140 0 Y 16 0 45 +def test t7 t7 c c 252 262140 0 Y 16 0 45 +def test t7 t7 d d 252 262140 0 Y 16 0 45 +a b c d +SELECT d FROM t8; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t8 t8 d d 252 (format=json) 262140 0 Y 16 0 45 +d +# and what each does with a document says the same: quoted, where a +# column whose check is the call alone is spliced +INSERT INTO t7 VALUES ('{"p":1}', '{"p":1}', '{"p":1}', '{"p":1}'), +('[1,2]', '[1,2]', '[1,2]', '[1,2]'); +INSERT INTO t8 VALUES ('{"p":1}'), ('[1,2]'); +SELECT JSON_SET('{}', '$.k', a) AS from_and_shape FROM t7; +from_and_shape +{"k": "{\"p\":1}"} +{"k": "[1,2]"} +SELECT JSON_SET('{}', '$.k', d) AS from_call_alone FROM t8; +from_call_alone +{"k": {"p": 1}} +{"k": [1, 2]} +DROP TABLE t7, t8; +# +# A column the server itself wrote the constraint for stays JSON +# +# Both a declared JSON column and the columns of a temporary table +# built for grouping carry a constraint over themselves. +# +CREATE TABLE t6 (j JSON, s VARCHAR(100) CHECK (JSON_VALID(s))); +INSERT INTO t6 VALUES ('{"x":1}', '{"y":2}'), ('{"x":3}', '{"y":4}'); +SELECT j, s FROM t6 GROUP BY j, s; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def test t6 t6 j j 252 (format=json) 4294967295 7 Y 32912 0 45 +def test t6 t6 s s 253 (format=json) 400 7 Y 32768 0 45 +j s +{"x":1} {"y":2} +{"x":3} {"y":4} +SELECT JSON_SET('{}', '$.k', j) AS from_j, +JSON_SET('{}', '$.k', s) AS from_s +FROM (SELECT j, s FROM t6 GROUP BY j, s) AS grouped; +from_j from_s +{"k": {"x": 1}} {"k": {"y": 2}} +{"k": {"x": 3}} {"k": {"y": 4}} +SELECT JSON_ARRAYAGG(j) AS agg_j, JSON_ARRAYAGG(s) AS agg_s FROM t6; +agg_j agg_s +[{"x":1},{"x":3}] [{"y":2},{"y":4}] +DROP TABLE t6; +# +# A row image a trigger reads through OLD. or NEW. is reached +# through a SECOND set of field objects, copied from the table's +# own so that they can point at the other record buffer. A copy is +# the same column of the same table and carries the same check, so +# it is the same JSON column, and what it holds goes into a +# document as a document. +# +# Which columns get copied is not the same on the two sides. The +# OLD. buffer is built whenever there is an update or delete +# trigger and copies every column; the NEW. buffer is built only +# where some column of the table cannot be null, and copies only +# the columns that cannot be. So the five cases below that read a +# copy are not the five a reader would guess, and the two that +# read the table's own fields are here to say which is which. +# +# Every answer below is what a released server gives. +# +CREATE TABLE lg (tag VARCHAR(24), v TEXT); +CREATE TABLE t7 (id INT, j JSON); +CREATE TABLE t8 (id INT, j JSON NOT NULL); +INSERT INTO t7 VALUES (1, '{"a":1}'); +INSERT INTO t8 VALUES (1, '{"a":1}'); +CREATE TRIGGER tr1 BEFORE UPDATE ON t7 FOR EACH ROW +INSERT INTO lg VALUES ('nullable OLD upd', JSON_SET('{}','$.k',OLD.j)), +('nullable NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr2 BEFORE UPDATE ON t8 FOR EACH ROW +INSERT INTO lg VALUES ('notnull OLD upd', JSON_SET('{}','$.k',OLD.j)), +('notnull NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr3 BEFORE INSERT ON t7 FOR EACH ROW +INSERT INTO lg VALUES ('nullable NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr4 BEFORE INSERT ON t8 FOR EACH ROW +INSERT INTO lg VALUES ('notnull NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr5 BEFORE DELETE ON t7 FOR EACH ROW +INSERT INTO lg VALUES ('nullable OLD del', JSON_SET('{}','$.k',OLD.j)); +UPDATE t7 SET id = 2; +UPDATE t8 SET id = 2; +INSERT INTO t7 VALUES (3, '{"b":2}'); +INSERT INTO t8 VALUES (3, '{"b":2}'); +DELETE FROM t7 WHERE id = 3; +SELECT tag, v FROM lg ORDER BY tag, v; +tag v +notnull NEW ins {"k": {"b": 2}} +notnull NEW upd {"k": {"a": 1}} +notnull OLD upd {"k": {"a": 1}} +nullable NEW ins {"k": {"b": 2}} +nullable NEW upd {"k": {"a": 1}} +nullable OLD del {"k": {"b": 2}} +nullable OLD upd {"k": {"a": 1}} +# +# THE CONTROLS: the same two columns read through the table's own +# fields, where there is no copy in the way. These were never in +# question, and they say that the answers above are about the +# copying and not about the columns. +# +SELECT JSON_SET('{}','$.k',j) AS direct_nullable FROM t7 ORDER BY 1; +direct_nullable +{"k": {"a": 1}} +SELECT JSON_SET('{}','$.k',j) AS direct_notnull FROM t8 ORDER BY 1; +direct_notnull +{"k": {"a": 1}} +{"k": {"b": 2}} +DROP TABLE lg, t7, t8; diff --git a/mysql-test/main/func_json_valid_constraint.test b/mysql-test/main/func_json_valid_constraint.test new file mode 100644 index 0000000000000..3fce3704902b2 --- /dev/null +++ b/mysql-test/main/func_json_valid_constraint.test @@ -0,0 +1,235 @@ +--echo # +--echo # Which column a JSON_VALID check constraint makes a JSON column +--echo # +--echo # A string column is a JSON column when it carries a check constraint +--echo # that says that column holds a document. A constraint that reads +--echo # another column, or no column at all, says nothing about this one. +--echo # + +SET NAMES utf8mb4; + +# The constraint is looked at while the column definition is validated, +# which for a prepared CREATE TABLE is prepare time, so the warnings below +# only reach the client when prepare warnings are asked for. +--enable_prepare_warnings + +--echo # +--echo # A constraint reading another column +--echo # +--echo # d is the ordinary usage and e an ordinary JSON column, both controls. + +CREATE TABLE t1 ( + a VARCHAR(100) CHECK (JSON_VALID(b)), + b VARCHAR(100), + d VARCHAR(100) CHECK (JSON_VALID(d)), + e JSON +); +SHOW CREATE TABLE t1; + +--echo # what the server tells a client the columns are +# Read through a view the columns belong to the view, so the names and the +# flags reported here would be the wrapper's rather than the table's. +--disable_view_protocol +--enable_metadata +SELECT a, b, d, e FROM t1; +--disable_metadata +--enable_view_protocol + +SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA='test' AND TABLE_NAME='t1' ORDER BY COLUMN_NAME; + +--echo # +--echo # a is unconstrained, so it can hold text that is not a document +--echo # +INSERT INTO t1 VALUES ('not json at all', '{"x":1}', '{"y":2}', '{"z":3}'); +SELECT a, JSON_VALID(a) AS a_is_a_document FROM t1; + +--echo # a is quoted like the ordinary column it is, not spliced +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t1; +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t1; + +--echo # the controls are spliced +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t1; +SELECT JSON_SET('{}', '$.k', e) AS from_e FROM t1; + +DROP TABLE t1; + +--echo # +--echo # The same column holding a document that IS well formed +--echo # +--echo # Whether a is spliced or quoted decides whether the document nests +--echo # or arrives as a string, and it must not depend on b's constraint. +--echo # + +CREATE TABLE t2 ( + a VARCHAR(100) CHECK (JSON_VALID(b)), + b VARCHAR(100), + d VARCHAR(100) CHECK (JSON_VALID(d)) +); +INSERT INTO t2 VALUES ('{"p":1}', '{"x":1}', '{"p":1}'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t2; +SELECT JSON_SET('{}', '$.k', b) AS from_b FROM t2; +SELECT JSON_SET('{}', '$.k', d) AS from_d FROM t2; +DROP TABLE t2; + +--echo # +--echo # A constraint that constrains nothing +--echo # +--echo # JSON_VALID('{}') is a constant true, so the column keeps no +--echo # promise about its contents whatsoever. +--echo # + +CREATE TABLE t3 (a VARCHAR(100) CHECK (JSON_VALID('{}'))); +--disable_view_protocol +--enable_metadata +SELECT a FROM t3; +--disable_metadata +--enable_view_protocol +INSERT INTO t3 VALUES ('not json at all'); +SELECT JSON_SET('{}', '$.k', a) AS from_a FROM t3; +DROP TABLE t3; + +--echo # +--echo # The constraint is still enforced, on the column it names +--echo # + +CREATE TABLE t4 (a VARCHAR(100) CHECK (JSON_VALID(b)), b VARCHAR(100)); +--error ER_CONSTRAINT_FAILED +INSERT INTO t4 VALUES ('anything', 'not json'); +INSERT INTO t4 VALUES ('anything', '{"x":1}'); +SELECT a, b FROM t4; +DROP TABLE t4; + +--echo # +--echo # A table level constraint types neither column, as before +--echo # + +CREATE TABLE t5 ( + a VARCHAR(100), + b VARCHAR(100), + CONSTRAINT c CHECK (JSON_VALID(b)) +); +--disable_view_protocol +--enable_metadata +SELECT a, b FROM t5; +--disable_metadata +--enable_view_protocol +DROP TABLE t5; + +--echo # +--echo # A check that asks more than whether the column holds a document +--echo # +--echo # What types a column is JSON_VALID() of it and nothing else. An +--echo # AND keeps the promise and an OR does not, but a promise kept is +--echo # not a column typed, and neither shape is typed. Each is told so, +--echo # in the words that fit it. +--echo # + +CREATE TABLE t7 ( + a TEXT CHECK (LENGTH(a) > 0 AND JSON_VALID(a)), + b TEXT CHECK (LENGTH(b) > 0 OR JSON_VALID(b)), + c TEXT CHECK (LENGTH(d) > 0 AND JSON_VALID(d)), + d TEXT +); +CREATE TABLE t8 (d TEXT CHECK (JSON_VALID(d))); + +--echo # the protocol tells the client what the warnings told the writer: +--echo # none of the three is a JSON column, and a column whose check is +--echo # the call alone still is +--disable_view_protocol +--enable_metadata +SELECT a, b, c, d FROM t7; +SELECT d FROM t8; +--disable_metadata +--enable_view_protocol + +--echo # and what each does with a document says the same: quoted, where a +--echo # column whose check is the call alone is spliced +INSERT INTO t7 VALUES ('{"p":1}', '{"p":1}', '{"p":1}', '{"p":1}'), + ('[1,2]', '[1,2]', '[1,2]', '[1,2]'); +INSERT INTO t8 VALUES ('{"p":1}'), ('[1,2]'); +SELECT JSON_SET('{}', '$.k', a) AS from_and_shape FROM t7; +SELECT JSON_SET('{}', '$.k', d) AS from_call_alone FROM t8; +DROP TABLE t7, t8; + +--echo # +--echo # A column the server itself wrote the constraint for stays JSON +--echo # +--echo # Both a declared JSON column and the columns of a temporary table +--echo # built for grouping carry a constraint over themselves. +--echo # + +CREATE TABLE t6 (j JSON, s VARCHAR(100) CHECK (JSON_VALID(s))); +INSERT INTO t6 VALUES ('{"x":1}', '{"y":2}'), ('{"x":3}', '{"y":4}'); + +--disable_view_protocol +--enable_metadata +SELECT j, s FROM t6 GROUP BY j, s; +--disable_metadata +--enable_view_protocol + +SELECT JSON_SET('{}', '$.k', j) AS from_j, + JSON_SET('{}', '$.k', s) AS from_s +FROM (SELECT j, s FROM t6 GROUP BY j, s) AS grouped; + +SELECT JSON_ARRAYAGG(j) AS agg_j, JSON_ARRAYAGG(s) AS agg_s FROM t6; + +DROP TABLE t6; + +--echo # +--echo # A row image a trigger reads through OLD. or NEW. is reached +--echo # through a SECOND set of field objects, copied from the table's +--echo # own so that they can point at the other record buffer. A copy is +--echo # the same column of the same table and carries the same check, so +--echo # it is the same JSON column, and what it holds goes into a +--echo # document as a document. +--echo # +--echo # Which columns get copied is not the same on the two sides. The +--echo # OLD. buffer is built whenever there is an update or delete +--echo # trigger and copies every column; the NEW. buffer is built only +--echo # where some column of the table cannot be null, and copies only +--echo # the columns that cannot be. So the five cases below that read a +--echo # copy are not the five a reader would guess, and the two that +--echo # read the table's own fields are here to say which is which. +--echo # +--echo # Every answer below is what a released server gives. +--echo # +CREATE TABLE lg (tag VARCHAR(24), v TEXT); +CREATE TABLE t7 (id INT, j JSON); +CREATE TABLE t8 (id INT, j JSON NOT NULL); +INSERT INTO t7 VALUES (1, '{"a":1}'); +INSERT INTO t8 VALUES (1, '{"a":1}'); + +CREATE TRIGGER tr1 BEFORE UPDATE ON t7 FOR EACH ROW + INSERT INTO lg VALUES ('nullable OLD upd', JSON_SET('{}','$.k',OLD.j)), + ('nullable NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr2 BEFORE UPDATE ON t8 FOR EACH ROW + INSERT INTO lg VALUES ('notnull OLD upd', JSON_SET('{}','$.k',OLD.j)), + ('notnull NEW upd', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr3 BEFORE INSERT ON t7 FOR EACH ROW + INSERT INTO lg VALUES ('nullable NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr4 BEFORE INSERT ON t8 FOR EACH ROW + INSERT INTO lg VALUES ('notnull NEW ins', JSON_SET('{}','$.k',NEW.j)); +CREATE TRIGGER tr5 BEFORE DELETE ON t7 FOR EACH ROW + INSERT INTO lg VALUES ('nullable OLD del', JSON_SET('{}','$.k',OLD.j)); + +UPDATE t7 SET id = 2; +UPDATE t8 SET id = 2; +INSERT INTO t7 VALUES (3, '{"b":2}'); +INSERT INTO t8 VALUES (3, '{"b":2}'); +DELETE FROM t7 WHERE id = 3; + +SELECT tag, v FROM lg ORDER BY tag, v; + +--echo # +--echo # THE CONTROLS: the same two columns read through the table's own +--echo # fields, where there is no copy in the way. These were never in +--echo # question, and they say that the answers above are about the +--echo # copying and not about the columns. +--echo # +SELECT JSON_SET('{}','$.k',j) AS direct_nullable FROM t7 ORDER BY 1; +SELECT JSON_SET('{}','$.k',j) AS direct_notnull FROM t8 ORDER BY 1; + +DROP TABLE lg, t7, t8; + +--disable_prepare_warnings diff --git a/mysql-test/main/func_json_value_reserve.result b/mysql-test/main/func_json_value_reserve.result new file mode 100644 index 0000000000000..d06ac662375a3 --- /dev/null +++ b/mysql-test/main/func_json_value_reserve.result @@ -0,0 +1,653 @@ +SET @save_optimizer_switch= @@optimizer_switch; +CREATE TABLE t1 (v VARCHAR(200)); +INSERT INTO t1 VALUES ('{"x":1}'), ('{"x":2}'); +CREATE TABLE t2 (v VARCHAR(10), s VARCHAR(20)); +INSERT INTO t2 VALUES ('{"x":1}', REPEAT(CHAR(1),20)), +('{"x":2}', REPEAT(CHAR(2),20)); +CREATE TABLE t3 (v VARCHAR(200)); +INSERT INTO t3 VALUES (CONCAT('{"x":"', REPEAT('a',190), '"}')), +(CONCAT('{"x":"', REPEAT('b',190), '"}')); +# +# What is asked for, taken off the result itself rather than off +# a column it was put in. Nothing about where the result goes can +# reach this number, so it says what the arithmetic came to and +# nothing else. The Length column is the one to read. +# +SELECT JSON_SET(v, '$.p', 'z') AS one_pair FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def one_pair 253 (format=json) 417 18 Y 0 39 8 +one_pair +{"x": 1, "p": "z"} +{"x": 2, "p": "z"} +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS two_pairs FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def two_pairs 253 (format=json) 434 28 Y 0 39 8 +two_pairs +{"x": 1, "p": "z", "q": "w"} +{"x": 2, "p": "z", "q": "w"} +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS three_pairs +FROM t1; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def three_pairs 253 (format=json) 451 38 Y 0 39 8 +three_pairs +{"x": 1, "p": "z", "q": "w", "s": "y"} +{"x": 2, "p": "z", "q": "w", "s": "y"} +SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS plain_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def plain_value 253 (format=json) 95 47 Y 0 39 8 +plain_value +{"x": 1, "p": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +{"x": 2, "p": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"} +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS escaped_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def escaped_value 253 (format=json) 75 57 Y 0 39 8 +escaped_value +{"x": 1, "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +{"x": 2, "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +SELECT JSON_SET(v, '$.p', s) AS wide_escape_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def wide_escape_value 253 (format=json) 75 137 Y 0 39 8 +wide_escape_value +{"x": 1, "p": "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"} +{"x": 2, "p": "\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"} +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS document_fills_column FROM t3; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def document_fills_column 253 (format=json) 455 248 Y 0 39 8 +document_fills_column +{"x": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +{"x": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "p": "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\""} +# +# The same numbers again, read off the column a result is put in. +# The document, then a path and a value for each pair. +# +CREATE TABLE d1 AS SELECT JSON_SET(v, '$.p', 'z') AS r FROM t1; +SHOW CREATE TABLE d1; +Table Create Table +d1 CREATE TABLE `d1` ( + `r` varchar(417) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d1; +CREATE TABLE d2 AS +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS r FROM t1; +SHOW CREATE TABLE d2; +Table Create Table +d2 CREATE TABLE `d2` ( + `r` varchar(434) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d2; +CREATE TABLE d3 AS +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS r FROM t1; +SHOW CREATE TABLE d3; +Table Create Table +d3 CREATE TABLE `d3` ( + `r` varchar(451) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d3; +SET @@optimizer_switch='derived_merge=off'; +# +# A value longer than the document it goes into. The result is +# put in a column here, so what is asked for is what is kept. +# +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +produced +5017 +5017 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5017 1 +5017 1 +# +# A small document, so that what is asked for stays under the +# width above which a result is given a blob to live in. The +# column stays a VARCHAR, so the arithmetic is what decides the +# answer rather than the change of type. +# +CREATE TABLE d4 AS SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2; +SHOW CREATE TABLE d4; +Table Create Table +d4 CREATE TABLE `d4` ( + `r` varchar(95) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d4; +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',30))) AS produced FROM t2; +produced +47 +47 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2) AS d; +kept valid +47 1 +47 1 +# +# A value whose characters are written with an escape apiece, so +# what goes in is twice what was passed. +# +CREATE TABLE d5 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2; +SHOW CREATE TABLE d5; +Table Create Table +d5 CREATE TABLE `d5` ( + `r` varchar(75) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d5; +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t2; +produced +57 +57 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2) AS d; +kept valid +57 1 +57 1 +# +# A value whose characters cannot be written as they stand and are +# escaped out instead, six characters for each one. Twice the +# characters does not cover that, and this is the room that is +# still asked for short. +# +# STILL BROKEN AND KNOWN TO BE. Pricing an escaping properly needs +# both character sets to have a say, and it widens what these +# functions declare by enough to move a result out of a memory +# temporary table onto disk. It is left until a declared width no +# longer decides where a temporary table lives, and what is +# recorded below is what that leaves standing. +# +# Written into a table of its own, the store says so and the +# statement stops. +# +CREATE TABLE d6 AS SELECT JSON_SET(v, '$.p', s) AS r FROM t2; +ERROR 22001: Data too long for column 'r' at row 1 +SELECT LENGTH(s) AS value_length FROM t2; +value_length +20 +20 +SELECT LENGTH(JSON_SET(v, '$.p', s)) AS produced FROM t2; +produced +137 +137 +# +# Written into an internal temporary table, nothing says anything: +# that store does not raise count_cuted_fields, so the document is +# cut where it stands and what is left of it is not one. 'valid' +# reading 0 here is the same defect going unremarked. +# +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', s) AS r FROM t2) AS d; +kept valid +75 0 +75 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# A document that fills the column it is held in, with a short +# value that has to be written with escapes. The document is +# allowed twice its own width and no more, so whatever the +# number below comes to over that is the room for the path and +# the value or it is nothing at all. +# +CREATE TABLE d7 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3; +SHOW CREATE TABLE d7; +Table Create Table +d7 CREATE TABLE `d7` ( + `r` varchar(455) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d7; +SELECT LENGTH(v) AS document FROM t3; +document +198 +198 +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t3; +produced +248 +248 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3) AS d; +kept valid +248 1 +248 1 +# +# The long value in the last pair of three, which is the one the +# loop reaches last. +# +SELECT LENGTH(JSON_SET(v, '$.p', 'x', '$.q', 'y', +'$.s', REPEAT('z',5000))) AS produced FROM t1; +produced +5037 +5037 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$.p', 'x', '$.q', 'y', +'$.s', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5037 1 +5037 1 +# +# JSON_INSERT and JSON_REPLACE, whose length the same function +# works out. +# +SELECT LENGTH(JSON_INSERT(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +produced +5017 +5017 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_INSERT(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5017 1 +5017 1 +SELECT LENGTH(JSON_REPLACE(v, '$.x', REPEAT('z',5000))) AS produced FROM t1; +produced +5009 +5009 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REPLACE(v, '$.x', REPEAT('z',5000)) AS r FROM t1) AS d; +kept valid +5009 1 +5009 1 +# +# A path long enough to be worth counting. The name it adds is +# what goes into the result, so the room for it is asked for once. +# +SELECT LENGTH(JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z')) AS produced +FROM t1; +produced +317 +317 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z') AS r +FROM t1) AS d; +kept valid +317 1 +317 1 +# +# The same room asked for where it is checked rather than cut. +# +CREATE TABLE d8 AS SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid FROM d8; +kept valid +5017 1 +5017 1 +DROP TABLE d8; +# +# A document held in a column wide enough that nothing it is asked +# to hold can overflow it. +# +CREATE TABLE t4 (j JSON); +INSERT INTO t4 VALUES ('{"x":1}'), ('{"x":2}'); +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(j, '$.p', REPEAT('z',5000)) AS r FROM t4) AS d; +kept valid +5017 1 +5017 1 +DROP TABLE t4; +# +# The same room, asked for by the functions that build a document +# rather than edit one, and by the two that append to an array. +# All four work it out the same way and all four are given a value +# whose characters have to be escaped. +# +SELECT JSON_ARRAY(s) AS array_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def array_value 253 (format=json) 48 124 Y 0 39 8 +array_value +["\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"] +["\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"] +SELECT JSON_OBJECT('k', s) AS object_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def object_value 253 (format=json) 56 129 Y 0 39 8 +object_value +{"k": "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"} +{"k": "\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"} +SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS appended FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def appended 253 (format=json) 66 134 Y 0 39 8 +appended +{"x": [1, "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"]} +{"x": [2, "\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002"]} +SELECT JSON_ARRAY_INSERT(JSON_SET(v, '$.x', JSON_ARRAY(1)), '$.x[0]', s) +AS inserted FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def inserted 253 (format=json) 152 134 Y 0 39 8 +inserted +{"x": ["\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001", 1]} +{"x": ["\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002\u0002", 1]} +SELECT LENGTH(JSON_ARRAY(s)) AS produced FROM t2; +produced +124 +124 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY(s) AS r FROM t2) AS d; +kept valid +48 0 +48 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT LENGTH(JSON_OBJECT('k', s)) AS produced FROM t2; +produced +129 +129 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_OBJECT('k', s) AS r FROM t2) AS d; +kept valid +56 0 +56 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$.x', s)) AS produced FROM t2; +produced +134 +134 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS r FROM t2) AS d; +kept valid +66 0 +66 0 +Warnings: +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +Note 4037 Unexpected end of JSON text in argument 1 to function 'json_valid' +# +# What an escape costs where it is JSON_QUOTE doing the writing. +# A character that cannot be written as it stands is escaped as a +# backslash, a 'u' and the hex of its UTF-16 form, which is four +# figures for a character of the first plane and a second escape +# after it for any other. +# +# The pair is only ever reached for a character the set being +# written into cannot carry, and this function writes into +# utf8mb4, which carries everything there is. So six apiece is +# the whole of it, and every width over that is room asked for +# that nothing can take. +# +# The constructor below is asked for twice its argument instead, +# which is the room that does not cover an escaping - see above. +# +CREATE TABLE t7 (s VARCHAR(45)); +INSERT INTO t7 VALUES (REPEAT('a',45)), (REPEAT('b',45)); +SELECT JSON_QUOTE(s) AS quoted FROM t7; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def quoted 253 272 47 Y 128 39 8 +quoted +"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +CREATE TABLE d9 AS SELECT JSON_ARRAY(s) AS r FROM t7; +SHOW CREATE TABLE d9; +Table Create Table +d9 CREATE TABLE `d9` ( + `r` varchar(98) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE d9; +DROP TABLE t7; +# +# A value that is already a document goes in as it stands, and it +# is written out again with the spacing, so the room for it is +# asked for twice over and no quotes are asked for round it. +# +SELECT JSON_ARRAY(JSON_QUERY(v, '$')) AS document_value FROM t2; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def document_value 253 (format=json) 26 9 Y 0 39 8 +document_value +[{"x":1}] +[{"x":2}] +# +# A function that takes something out and writes what is left out +# again, a space arriving after every separator it copies. One +# element of two characters goes and twenty-eight spaces come, so +# the room asked for has to cover the writing here as well. +# +CREATE TABLE t5 (v VARCHAR(64)) CHARSET utf8mb4; +INSERT INTO t5 VALUES +('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), +('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); +SELECT JSON_REMOVE(v, '$[0]') AS removed FROM t5; +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def removed 253 (format=json) 128 87 Y 0 39 8 +removed +[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] +[2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2] +SELECT LENGTH(v) AS document FROM t5; +document +61 +61 +SELECT LENGTH(JSON_REMOVE(v, '$[0]')) AS produced FROM t5; +produced +87 +87 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REMOVE(v, '$[0]') AS r FROM t5) AS d; +kept valid +87 1 +87 1 +# +# A path that matches nothing, where nothing is taken out at all +# and only the spacing arrives. +# +SELECT LENGTH(JSON_REMOVE(v, '$.nothing')) AS produced FROM t5; +produced +90 +90 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REMOVE(v, '$.nothing') AS r FROM t5) AS d; +kept valid +90 1 +90 1 +# +# The neighbour that adds the same spacing without taking anything +# out, and the one that only ever takes spacing away. +# +SELECT LENGTH(JSON_LOOSE(v)) AS produced FROM t5; +produced +90 +90 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_LOOSE(v) AS r FROM t5) AS d; +kept valid +90 1 +90 1 +SELECT LENGTH(JSON_COMPACT(v)) AS produced FROM t5; +produced +61 +61 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_COMPACT(v) AS r FROM t5) AS d; +kept valid +61 1 +61 1 +# +# The same spacing arrives when a document is edited rather than +# cut down, the whole of it being written out again around whatever +# was put in. A document packed tight enough that the spacing +# alone outgrows it is the shape that says whether the room for the +# document covers the writing or only the reading. +# +SELECT LENGTH(JSON_SET(v, '$[30]', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_SET(v, '$[30]', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +SELECT LENGTH(JSON_INSERT(v, '$[30]', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_INSERT(v, '$[30]', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +SELECT LENGTH(JSON_REPLACE(v, '$[0]', 9)) AS produced FROM t5; +produced +90 +90 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_REPLACE(v, '$[0]', 9) AS r FROM t5) AS d; +kept valid +90 1 +90 1 +# +# The two that append to an array, which write the document out +# again in the same way. +# +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY_APPEND(v, '$', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +SELECT LENGTH(JSON_ARRAY_INSERT(v, '$[0]', 1)) AS produced FROM t5; +produced +93 +93 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_ARRAY_INSERT(v, '$[0]', 1) AS r FROM t5) AS d; +kept valid +93 1 +93 1 +# +# A value that is already a document goes in as it stands, but what +# goes in is written out again with the rest, so the room for it +# has to cover the spacing too. +# +SELECT LENGTH(JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$'))) AS produced +FROM t5; +produced +180 +180 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$')) AS r +FROM t5) AS d; +kept valid +180 1 +180 1 +DROP TABLE t5; +# +# The same, patched rather than merged, over two documents that +# have no name in common so that what comes back holds all of both. +# The columns are no wider than what they hold, so the room asked +# for is what the two of them say and nothing is over from the +# declaring. +# +CREATE TABLE t6 (v1 VARCHAR(37), v2 VARCHAR(37)) CHARSET utf8mb4; +INSERT INTO t6 VALUES +('{"a":1,"b":1,"c":1,"d":1,"e":1,"f":1}', +'{"g":1,"h":1,"i":1,"j":1,"k":1,"l":1}'), +('{"m":2,"n":2,"o":2,"p":2,"q":2,"r":2}', +'{"s":2,"t":2,"u":2,"v":2,"w":2,"x":2}'); +SELECT LENGTH(v1) AS document FROM t6; +document +37 +37 +SELECT LENGTH(JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), +JSON_QUERY(v2, '$'))) AS produced FROM t6; +produced +96 +96 +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid +FROM (SELECT JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), +JSON_QUERY(v2, '$')) AS r FROM t6) AS d; +kept valid +96 1 +96 1 +DROP TABLE t6; +# +# The same document priced twice: once with every argument already in +# the character set the result is written in, and once with one of them +# having to be converted to get there. +# +# Aggregating the sets wraps the argument that has to move, and what is +# asked for has to see through that wrapper the same way the writing +# does. A document goes in as it stands whether it arrived wrapped or +# not, so it costs what a document costs; read as a plain string it +# would be charged for the two quote characters that the writing is +# never going to put round it. Both columns below are declared the +# same width, +# and hold the same answer - the miscounting was only ever room asked +# for and not taken, so no value was ever cut short by it. +# +CREATE TABLE t4 (v VARCHAR(20) CHARACTER SET latin1, +u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t4 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); +CREATE TABLE t7 (v VARCHAR(20) CHARACTER SET utf8mb4, +u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t7 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); +CREATE TABLE m1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t4; +SHOW CREATE TABLE m1; +Table Create Table +m1 CREATE TABLE `m1` ( + `r` varchar(72) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE TABLE s1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t7; +SHOW CREATE TABLE s1; +Table Create Table +s1 CREATE TABLE `s1` ( + `r` varchar(72) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT r FROM m1 ORDER BY r; +r +[{"x":1}, "{\"y\":1}"] +[{"x":2}, "{\"y\":2}"] +SELECT r FROM s1 ORDER BY r; +r +[{"x":1}, "{\"y\":1}"] +[{"x":2}, "{\"y\":2}"] +DROP TABLE m1, s1; +# +# An object is built by the same arithmetic and moves the same way. +# +CREATE TABLE m2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r +FROM t4; +SHOW CREATE TABLE m2; +Table Create Table +m2 CREATE TABLE `m2` ( + `r` varchar(88) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE TABLE s2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r +FROM t7; +SHOW CREATE TABLE s2; +Table Create Table +s2 CREATE TABLE `s2` ( + `r` varchar(88) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE m2, s2; +# +# THE CONTROL: the same two columns through a function that EDITS a +# document. Those take the set of the document they were given rather +# than aggregating over their arguments, so no wrapper is built and +# there is nothing to see through. Both are declared the same width in +# both worlds, which is what says the widths above move because of the +# wrapping and not because two sets are in play. +# +CREATE TABLE m3 AS +SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r +FROM t4; +SHOW CREATE TABLE m3; +Table Create Table +m3 CREATE TABLE `m3` ( + `r` varchar(73) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE TABLE s3 AS +SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r +FROM t7; +SHOW CREATE TABLE s3; +Table Create Table +s3 CREATE TABLE `s3` ( + `r` varchar(73) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT r FROM m3 ORDER BY r; +r +{"y": 1, "k": {"x": 1}} +{"y": 2, "k": {"x": 2}} +SELECT r FROM s3 ORDER BY r; +r +{"y": 1, "k": {"x": 1}} +{"y": 2, "k": {"x": 2}} +DROP TABLE m3, s3; +DROP TABLE t4, t7; +SET @@optimizer_switch= @save_optimizer_switch; +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/func_json_value_reserve.test b/mysql-test/main/func_json_value_reserve.test new file mode 100644 index 0000000000000..ce7a19536929c --- /dev/null +++ b/mysql-test/main/func_json_value_reserve.test @@ -0,0 +1,460 @@ +# +# The room a function asks for the document it is going to return. +# +# The length has to be named before any value has been seen, so it is +# asked for out of what the arguments say about themselves. A result +# that does not fit what was asked for is cut where it stands, and what +# is left of a document is not one. +# +# A value that is not already a document goes in as a JSON string, so +# what is asked for has to cover what the writing makes of it and not +# what was passed. Twice its characters is what is asked, and a value +# that is not there at all is still written as four. +# +# Twice does not cover an escaping - that costs six for a character, or +# twelve where the set being written into cannot carry it - and pricing +# it does not stand here yet. Where that leaves a value escaped +# throughout is recorded below, under the heading that says so. +# +# The document a function is given has the same to attest to. It is +# written out again with a space after every separator that is copied, +# so what comes back can be longer than what went in whether something +# was taken out of it, put into it, or neither. +# +SET @save_optimizer_switch= @@optimizer_switch; + +CREATE TABLE t1 (v VARCHAR(200)); +INSERT INTO t1 VALUES ('{"x":1}'), ('{"x":2}'); + +CREATE TABLE t2 (v VARCHAR(10), s VARCHAR(20)); +INSERT INTO t2 VALUES ('{"x":1}', REPEAT(CHAR(1),20)), + ('{"x":2}', REPEAT(CHAR(2),20)); + +CREATE TABLE t3 (v VARCHAR(200)); +INSERT INTO t3 VALUES (CONCAT('{"x":"', REPEAT('a',190), '"}')), + (CONCAT('{"x":"', REPEAT('b',190), '"}')); + +--echo # +--echo # What is asked for, taken off the result itself rather than off +--echo # a column it was put in. Nothing about where the result goes can +--echo # reach this number, so it says what the arithmetic came to and +--echo # nothing else. The Length column is the one to read. +--echo # +# What is read here is what the expression itself declares. A cursor +# reports the temporary table it materialised the answer into and a view +# reports its own columns, so neither would be answering the question. +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_SET(v, '$.p', 'z') AS one_pair FROM t1; +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS two_pairs FROM t1; +SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS three_pairs + FROM t1; +SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS plain_value FROM t2; +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS escaped_value FROM t2; +SELECT JSON_SET(v, '$.p', s) AS wide_escape_value FROM t2; +SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS document_fills_column FROM t3; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +--echo # +--echo # The same numbers again, read off the column a result is put in. +--echo # The document, then a path and a value for each pair. +--echo # +CREATE TABLE d1 AS SELECT JSON_SET(v, '$.p', 'z') AS r FROM t1; +SHOW CREATE TABLE d1; +DROP TABLE d1; + +CREATE TABLE d2 AS + SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w') AS r FROM t1; +SHOW CREATE TABLE d2; +DROP TABLE d2; + +CREATE TABLE d3 AS + SELECT JSON_SET(v, '$.p', 'z', '$.q', 'w', '$.s', 'y') AS r FROM t1; +SHOW CREATE TABLE d3; +DROP TABLE d3; + +SET @@optimizer_switch='derived_merge=off'; + +--echo # +--echo # A value longer than the document it goes into. The result is +--echo # put in a column here, so what is asked for is what is kept. +--echo # +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; + +--echo # +--echo # A small document, so that what is asked for stays under the +--echo # width above which a result is given a blob to live in. The +--echo # column stays a VARCHAR, so the arithmetic is what decides the +--echo # answer rather than the change of type. +--echo # +CREATE TABLE d4 AS SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2; +SHOW CREATE TABLE d4; +DROP TABLE d4; + +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('z',30))) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('z',30)) AS r FROM t2) AS d; + +--echo # +--echo # A value whose characters are written with an escape apiece, so +--echo # what goes in is twice what was passed. +--echo # +CREATE TABLE d5 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2; +SHOW CREATE TABLE d5; +DROP TABLE d5; + +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t2) AS d; + +--echo # +--echo # A value whose characters cannot be written as they stand and are +--echo # escaped out instead, six characters for each one. Twice the +--echo # characters does not cover that, and this is the room that is +--echo # still asked for short. +--echo # +--echo # STILL BROKEN AND KNOWN TO BE. Pricing an escaping properly needs +--echo # both character sets to have a say, and it widens what these +--echo # functions declare by enough to move a result out of a memory +--echo # temporary table onto disk. It is left until a declared width no +--echo # longer decides where a temporary table lives, and what is +--echo # recorded below is what that leaves standing. +--echo # +--echo # Written into a table of its own, the store says so and the +--echo # statement stops. +--echo # +--error ER_DATA_TOO_LONG +CREATE TABLE d6 AS SELECT JSON_SET(v, '$.p', s) AS r FROM t2; + +SELECT LENGTH(s) AS value_length FROM t2; +SELECT LENGTH(JSON_SET(v, '$.p', s)) AS produced FROM t2; + +--echo # +--echo # Written into an internal temporary table, nothing says anything: +--echo # that store does not raise count_cuted_fields, so the document is +--echo # cut where it stands and what is left of it is not one. 'valid' +--echo # reading 0 here is the same defect going unremarked. +--echo # +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', s) AS r FROM t2) AS d; + +--echo # +--echo # A document that fills the column it is held in, with a short +--echo # value that has to be written with escapes. The document is +--echo # allowed twice its own width and no more, so whatever the +--echo # number below comes to over that is the room for the path and +--echo # the value or it is nothing at all. +--echo # +CREATE TABLE d7 AS SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3; +SHOW CREATE TABLE d7; +DROP TABLE d7; + +SELECT LENGTH(v) AS document FROM t3; +SELECT LENGTH(JSON_SET(v, '$.p', REPEAT('"',20))) AS produced FROM t3; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', REPEAT('"',20)) AS r FROM t3) AS d; + +--echo # +--echo # The long value in the last pair of three, which is the one the +--echo # loop reaches last. +--echo # +SELECT LENGTH(JSON_SET(v, '$.p', 'x', '$.q', 'y', + '$.s', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$.p', 'x', '$.q', 'y', + '$.s', REPEAT('z',5000)) AS r FROM t1) AS d; + +--echo # +--echo # JSON_INSERT and JSON_REPLACE, whose length the same function +--echo # works out. +--echo # +SELECT LENGTH(JSON_INSERT(v, '$.p', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_INSERT(v, '$.p', REPEAT('z',5000)) AS r FROM t1) AS d; + +SELECT LENGTH(JSON_REPLACE(v, '$.x', REPEAT('z',5000))) AS produced FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REPLACE(v, '$.x', REPEAT('z',5000)) AS r FROM t1) AS d; + +--echo # +--echo # A path long enough to be worth counting. The name it adds is +--echo # what goes into the result, so the room for it is asked for once. +--echo # +SELECT LENGTH(JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z')) AS produced + FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, CONCAT('$.', REPEAT('p',300)), 'z') AS r + FROM t1) AS d; + +--echo # +--echo # The same room asked for where it is checked rather than cut. +--echo # +CREATE TABLE d8 AS SELECT JSON_SET(v, '$.p', REPEAT('z',5000)) AS r FROM t1; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid FROM d8; +DROP TABLE d8; + +--echo # +--echo # A document held in a column wide enough that nothing it is asked +--echo # to hold can overflow it. +--echo # +CREATE TABLE t4 (j JSON); +INSERT INTO t4 VALUES ('{"x":1}'), ('{"x":2}'); +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(j, '$.p', REPEAT('z',5000)) AS r FROM t4) AS d; +DROP TABLE t4; + +--echo # +--echo # The same room, asked for by the functions that build a document +--echo # rather than edit one, and by the two that append to an array. +--echo # All four work it out the same way and all four are given a value +--echo # whose characters have to be escaped. +--echo # +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_ARRAY(s) AS array_value FROM t2; +SELECT JSON_OBJECT('k', s) AS object_value FROM t2; +SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS appended FROM t2; +SELECT JSON_ARRAY_INSERT(JSON_SET(v, '$.x', JSON_ARRAY(1)), '$.x[0]', s) + AS inserted FROM t2; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +SELECT LENGTH(JSON_ARRAY(s)) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY(s) AS r FROM t2) AS d; + +SELECT LENGTH(JSON_OBJECT('k', s)) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_OBJECT('k', s) AS r FROM t2) AS d; + +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$.x', s)) AS produced FROM t2; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY_APPEND(v, '$.x', s) AS r FROM t2) AS d; + +--echo # +--echo # What an escape costs where it is JSON_QUOTE doing the writing. +--echo # A character that cannot be written as it stands is escaped as a +--echo # backslash, a 'u' and the hex of its UTF-16 form, which is four +--echo # figures for a character of the first plane and a second escape +--echo # after it for any other. +--echo # +--echo # The pair is only ever reached for a character the set being +--echo # written into cannot carry, and this function writes into +--echo # utf8mb4, which carries everything there is. So six apiece is +--echo # the whole of it, and every width over that is room asked for +--echo # that nothing can take. +--echo # +--echo # The constructor below is asked for twice its argument instead, +--echo # which is the room that does not cover an escaping - see above. +--echo # +CREATE TABLE t7 (s VARCHAR(45)); +INSERT INTO t7 VALUES (REPEAT('a',45)), (REPEAT('b',45)); + +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_QUOTE(s) AS quoted FROM t7; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +CREATE TABLE d9 AS SELECT JSON_ARRAY(s) AS r FROM t7; +SHOW CREATE TABLE d9; +DROP TABLE d9; +DROP TABLE t7; + +--echo # +--echo # A value that is already a document goes in as it stands, and it +--echo # is written out again with the spacing, so the room for it is +--echo # asked for twice over and no quotes are asked for round it. +--echo # +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_ARRAY(JSON_QUERY(v, '$')) AS document_value FROM t2; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +--echo # +--echo # A function that takes something out and writes what is left out +--echo # again, a space arriving after every separator it copies. One +--echo # element of two characters goes and twenty-eight spaces come, so +--echo # the room asked for has to cover the writing here as well. +--echo # +CREATE TABLE t5 (v VARCHAR(64)) CHARSET utf8mb4; +INSERT INTO t5 VALUES + ('[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'), + ('[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]'); + +--disable_cursor_protocol +--disable_view_protocol +--enable_metadata +SELECT JSON_REMOVE(v, '$[0]') AS removed FROM t5; +--disable_metadata +--enable_view_protocol +--enable_cursor_protocol + +SELECT LENGTH(v) AS document FROM t5; +SELECT LENGTH(JSON_REMOVE(v, '$[0]')) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REMOVE(v, '$[0]') AS r FROM t5) AS d; + +--echo # +--echo # A path that matches nothing, where nothing is taken out at all +--echo # and only the spacing arrives. +--echo # +SELECT LENGTH(JSON_REMOVE(v, '$.nothing')) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REMOVE(v, '$.nothing') AS r FROM t5) AS d; + +--echo # +--echo # The neighbour that adds the same spacing without taking anything +--echo # out, and the one that only ever takes spacing away. +--echo # +SELECT LENGTH(JSON_LOOSE(v)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_LOOSE(v) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_COMPACT(v)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_COMPACT(v) AS r FROM t5) AS d; + +--echo # +--echo # The same spacing arrives when a document is edited rather than +--echo # cut down, the whole of it being written out again around whatever +--echo # was put in. A document packed tight enough that the spacing +--echo # alone outgrows it is the shape that says whether the room for the +--echo # document covers the writing or only the reading. +--echo # +SELECT LENGTH(JSON_SET(v, '$[30]', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_SET(v, '$[30]', 1) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_INSERT(v, '$[30]', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_INSERT(v, '$[30]', 1) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_REPLACE(v, '$[0]', 9)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_REPLACE(v, '$[0]', 9) AS r FROM t5) AS d; + +--echo # +--echo # The two that append to an array, which write the document out +--echo # again in the same way. +--echo # +SELECT LENGTH(JSON_ARRAY_APPEND(v, '$', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY_APPEND(v, '$', 1) AS r FROM t5) AS d; + +SELECT LENGTH(JSON_ARRAY_INSERT(v, '$[0]', 1)) AS produced FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_ARRAY_INSERT(v, '$[0]', 1) AS r FROM t5) AS d; + +--echo # +--echo # A value that is already a document goes in as it stands, but what +--echo # goes in is written out again with the rest, so the room for it +--echo # has to cover the spacing too. +--echo # +SELECT LENGTH(JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$'))) AS produced + FROM t5; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_MERGE(JSON_QUERY(v, '$'), JSON_QUERY(v, '$')) AS r + FROM t5) AS d; + +DROP TABLE t5; + +--echo # +--echo # The same, patched rather than merged, over two documents that +--echo # have no name in common so that what comes back holds all of both. +--echo # The columns are no wider than what they hold, so the room asked +--echo # for is what the two of them say and nothing is over from the +--echo # declaring. +--echo # +CREATE TABLE t6 (v1 VARCHAR(37), v2 VARCHAR(37)) CHARSET utf8mb4; +INSERT INTO t6 VALUES + ('{"a":1,"b":1,"c":1,"d":1,"e":1,"f":1}', + '{"g":1,"h":1,"i":1,"j":1,"k":1,"l":1}'), + ('{"m":2,"n":2,"o":2,"p":2,"q":2,"r":2}', + '{"s":2,"t":2,"u":2,"v":2,"w":2,"x":2}'); + +SELECT LENGTH(v1) AS document FROM t6; +SELECT LENGTH(JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), + JSON_QUERY(v2, '$'))) AS produced FROM t6; +SELECT LENGTH(r) AS kept, JSON_VALID(r) AS valid + FROM (SELECT JSON_MERGE_PATCH(JSON_QUERY(v1, '$'), + JSON_QUERY(v2, '$')) AS r FROM t6) AS d; + +DROP TABLE t6; + +--echo # +--echo # The same document priced twice: once with every argument already in +--echo # the character set the result is written in, and once with one of them +--echo # having to be converted to get there. +--echo # +--echo # Aggregating the sets wraps the argument that has to move, and what is +--echo # asked for has to see through that wrapper the same way the writing +--echo # does. A document goes in as it stands whether it arrived wrapped or +--echo # not, so it costs what a document costs; read as a plain string it +--echo # would be charged for the two quote characters that the writing is +--echo # never going to put round it. Both columns below are declared the +--echo # same width, +--echo # and hold the same answer - the miscounting was only ever room asked +--echo # for and not taken, so no value was ever cut short by it. +--echo # +CREATE TABLE t4 (v VARCHAR(20) CHARACTER SET latin1, + u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t4 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); +CREATE TABLE t7 (v VARCHAR(20) CHARACTER SET utf8mb4, + u VARCHAR(10) CHARACTER SET utf8mb4); +INSERT INTO t7 VALUES ('{"x":1}', '{"y":1}'), ('{"x":2}', '{"y":2}'); + +CREATE TABLE m1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t4; +SHOW CREATE TABLE m1; +CREATE TABLE s1 AS SELECT JSON_ARRAY(JSON_QUERY(v,'$'), u) AS r FROM t7; +SHOW CREATE TABLE s1; +SELECT r FROM m1 ORDER BY r; +SELECT r FROM s1 ORDER BY r; +DROP TABLE m1, s1; + +--echo # +--echo # An object is built by the same arithmetic and moves the same way. +--echo # +CREATE TABLE m2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r + FROM t4; +SHOW CREATE TABLE m2; +CREATE TABLE s2 AS SELECT JSON_OBJECT('k', JSON_QUERY(v,'$'), 'l', u) AS r + FROM t7; +SHOW CREATE TABLE s2; +DROP TABLE m2, s2; + +--echo # +--echo # THE CONTROL: the same two columns through a function that EDITS a +--echo # document. Those take the set of the document they were given rather +--echo # than aggregating over their arguments, so no wrapper is built and +--echo # there is nothing to see through. Both are declared the same width in +--echo # both worlds, which is what says the widths above move because of the +--echo # wrapping and not because two sets are in play. +--echo # +CREATE TABLE m3 AS + SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r + FROM t4; +SHOW CREATE TABLE m3; +CREATE TABLE s3 AS + SELECT JSON_INSERT(JSON_QUERY(u,'$'), '$.k', JSON_QUERY(v,'$')) AS r + FROM t7; +SHOW CREATE TABLE s3; +SELECT r FROM m3 ORDER BY r; +SELECT r FROM s3 ORDER BY r; +DROP TABLE m3, s3; +DROP TABLE t4, t7; + +SET @@optimizer_switch= @save_optimizer_switch; +DROP TABLE t1, t2, t3; diff --git a/mysql-test/main/func_json_variable_copy_oom.result b/mysql-test/main/func_json_variable_copy_oom.result new file mode 100644 index 0000000000000..14d071081aab1 --- /dev/null +++ b/mysql-test/main/func_json_variable_copy_oom.result @@ -0,0 +1,36 @@ +# +# No room for the copy made of a variable's value. +# +# What a function is given is a copy and not a view, so that the +# bytes it works on cannot be written over while it is working on +# them. Making that copy is a write, and a write can run short of +# room. There is no ordinary question that reaches the arm which +# copes with that, so it is reached here, and what it must do is +# refuse rather than return the view it was avoiding. +# +# The calls the copy is made for, and everything else about +# reading a document from a variable while the variable is being +# written, are in func_json_aliasing. +# +CREATE PROCEDURE p_variable_copy() +BEGIN +DECLARE v LONGTEXT DEFAULT NULL; +SET v = JSON_OBJECT('a', 1, 'b', 2); +SELECT JSON_INSERT(v, '$.c', 3) AS added, +JSON_REMOVE(v, '$.b') AS removed, +JSON_ARRAY_APPEND(v, '$', 9) AS appended; +END $$ +CALL p_variable_copy(); +added removed appended +{"a": 1, "b": 2, "c": 3} {"a": 1} [{"a": 1, "b": 2}, 9] +SET SESSION debug_dbug = '+d,json_variable_copy_out_of_memory'; +CALL p_variable_copy(); +added removed appended +NULL NULL NULL +SET SESSION debug_dbug = DEFAULT; +# and again with the room there, which is what says the refusals +# are about the copy rather than about the calls +CALL p_variable_copy(); +added removed appended +{"a": 1, "b": 2, "c": 3} {"a": 1} [{"a": 1, "b": 2}, 9] +DROP PROCEDURE p_variable_copy; diff --git a/mysql-test/main/func_json_variable_copy_oom.test b/mysql-test/main/func_json_variable_copy_oom.test new file mode 100644 index 0000000000000..b22c7ad3e0df5 --- /dev/null +++ b/mysql-test/main/func_json_variable_copy_oom.test @@ -0,0 +1,39 @@ +--source include/have_debug.inc + +--echo # +--echo # No room for the copy made of a variable's value. +--echo # +--echo # What a function is given is a copy and not a view, so that the +--echo # bytes it works on cannot be written over while it is working on +--echo # them. Making that copy is a write, and a write can run short of +--echo # room. There is no ordinary question that reaches the arm which +--echo # copes with that, so it is reached here, and what it must do is +--echo # refuse rather than return the view it was avoiding. +--echo # +--echo # The calls the copy is made for, and everything else about +--echo # reading a document from a variable while the variable is being +--echo # written, are in func_json_aliasing. +--echo # + +--delimiter $$ +CREATE PROCEDURE p_variable_copy() +BEGIN + DECLARE v LONGTEXT DEFAULT NULL; + SET v = JSON_OBJECT('a', 1, 'b', 2); + SELECT JSON_INSERT(v, '$.c', 3) AS added, + JSON_REMOVE(v, '$.b') AS removed, + JSON_ARRAY_APPEND(v, '$', 9) AS appended; +END $$ +--delimiter ; + +CALL p_variable_copy(); + +SET SESSION debug_dbug = '+d,json_variable_copy_out_of_memory'; +CALL p_variable_copy(); +SET SESSION debug_dbug = DEFAULT; + +--echo # and again with the room there, which is what says the refusals +--echo # are about the copy rather than about the calls +CALL p_variable_copy(); + +DROP PROCEDURE p_variable_copy; diff --git a/mysql-test/main/func_json_walk_oom.result b/mysql-test/main/func_json_walk_oom.result new file mode 100644 index 0000000000000..51fab2d1e41e8 --- /dev/null +++ b/mysql-test/main/func_json_walk_oom.result @@ -0,0 +1,75 @@ +# +# The room running out at every point of writing a value out. +# +# A value that is a document but is not written the loose way is +# written out again as it joins a document that is, and that +# writing puts down five kinds of thing: the bracket the value +# opens with, a comma before an element, a key with its colon, +# the bracket a value inside it opens with, and the bracket each +# of them closes with. Every one of them can be the append that +# finds no room, and each has its own way of giving up. +# +# Reaching them needs the room to run out at a chosen place, and +# the place is not something a single statement can aim at: what +# is left for the value is what the document did not spend, and +# where inside the value that runs out depends on how much +# punctuation stands before the point of interest. So the value +# is swept past the point instead. Each element of the run +# before the tail spends a fixed number of characters more in the +# loose form, and the key in front of the run spends one per +# letter, so the two together move the stopping point one +# character at a time across a tail that carries all of them. +# +# A function that edits a document takes room for its answer up +# front and one that reads a document does not, so the two are +# swept apart: the first runs out of room a fixed distance into +# the value, the second wherever the buffer last happened to +# grow to. +# +# What is pinned is not which statement fails - that is a fact +# about allocation sizes and is allowed to move. It is that +# every answer that does come back parses, and that some of them +# were refused, so the sweep is known to have reached the +# failures at all. A document with a piece missing out of the +# middle is not an answer, and no caller checks for one. +# +SET NAMES utf8mb4; +# +# Nothing came back half written, and the sweep did reach the +# failures. +# +SELECT @bad AS answers_that_do_not_parse, @refused > 0 AS some_were_refused; +answers_that_do_not_parse some_were_refused +0 1 +# +# The same values with the room there, so what the sweeps above +# compare against is an answer that arrives. +# +SET @a = CONCAT('{"a":1,', REPEAT('"p":1,', 110), +'"kkkk":{"b":[]},"z":[[]],"y":2}'); +SET @d = CONCAT('{"a":[', REPEAT('1,', 40), +'{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('z',1), '$.d', +JSON_QUERY(@a,'$')), '')) AS parses; +parses +1 +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@d, '$**.k'), '')) AS parses; +parses +1 +SELECT JSON_EXTRACT(@d, '$.b') AS answer; +answer +{"c": [6, [7]]} +# +# A value that stops in the middle of a key, which is the one way +# the walk gives up that is not about room. +# +SELECT JSON_EXTRACT('{"a":{"bb', '$.a') AS unfinished_key; +unfinished_key +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' +SELECT JSON_EXTRACT('{"a":[1,{"c', '$.a') AS unfinished_key; +unfinished_key +NULL +Warnings: +Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_extract' diff --git a/mysql-test/main/func_json_walk_oom.test b/mysql-test/main/func_json_walk_oom.test new file mode 100644 index 0000000000000..f37018a085b8d --- /dev/null +++ b/mysql-test/main/func_json_walk_oom.test @@ -0,0 +1,188 @@ +--source include/have_debug.inc + +--echo # +--echo # The room running out at every point of writing a value out. +--echo # +--echo # A value that is a document but is not written the loose way is +--echo # written out again as it joins a document that is, and that +--echo # writing puts down five kinds of thing: the bracket the value +--echo # opens with, a comma before an element, a key with its colon, +--echo # the bracket a value inside it opens with, and the bracket each +--echo # of them closes with. Every one of them can be the append that +--echo # finds no room, and each has its own way of giving up. +--echo # +--echo # Reaching them needs the room to run out at a chosen place, and +--echo # the place is not something a single statement can aim at: what +--echo # is left for the value is what the document did not spend, and +--echo # where inside the value that runs out depends on how much +--echo # punctuation stands before the point of interest. So the value +--echo # is swept past the point instead. Each element of the run +--echo # before the tail spends a fixed number of characters more in the +--echo # loose form, and the key in front of the run spends one per +--echo # letter, so the two together move the stopping point one +--echo # character at a time across a tail that carries all of them. +--echo # +--echo # A function that edits a document takes room for its answer up +--echo # front and one that reads a document does not, so the two are +--echo # swept apart: the first runs out of room a fixed distance into +--echo # the value, the second wherever the buffer last happened to +--echo # grow to. +--echo # +--echo # What is pinned is not which statement fails - that is a fact +--echo # about allocation sizes and is allowed to move. It is that +--echo # every answer that does come back parses, and that some of them +--echo # were refused, so the sweep is known to have reached the +--echo # failures at all. A document with a piece missing out of the +--echo # middle is not an answer, and no caller checks for one. +--echo # + +SET NAMES utf8mb4; + +--disable_query_log +--disable_result_log +--disable_warnings + +let $m= 0; +while ($m < 9) +{ + let $n= 96; + while ($n < 122) + { + eval SET @a = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"kkkk":{"b":[]},"z":[[]],"y":2}'); + eval SET @b = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"c":{},"d":[],"e":{"f":{}},"g":3}'); + eval SET @c = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"h":[1,2,[3,[4]]],"i":4}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + --error 0,5 + SELECT JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@a,'$')); + --error 0,5 + SELECT JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@b,'$')); + --error 0,5 + SELECT JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@c,'$')); + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +let $m= 0; +while ($m < 4) +{ + let $n= 0; + while ($n < 200) + { + eval SET @d = CONCAT('{"a', REPEAT('x', $m), '":[', REPEAT('1,', $n), + '{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + --error 0,5 + SELECT JSON_EXTRACT(@d, '$**.k'); + --error 0,5 + SELECT JSON_EXTRACT(@d, '$.b'); + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +# The same shapes again, with each answer read back to see that it +# parses. It is a pass of its own because reading the answer back is +# itself work in the buffer, and that moves the point the room runs out +# at - so a sweep that checks is not the same sweep as one that does +# not, and both are wanted. +SET @bad = 0; +SET @refused = 0; + +let $m= 0; +while ($m < 4) +{ + let $n= 96; + while ($n < 122) + { + eval SET @a = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"kkkk":{"b":[]},"z":[[]],"y":2}'); + eval SET @c = CONCAT('{"a', REPEAT('x', $m), '":1,', + REPEAT('"p":1,', $n), + '"h":[1,2,[3,[4]]],"i":4}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + + SET @r = NULL; + --error 0,5 + SET @r = JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@a,'$')); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET @r = NULL; + --error 0,5 + SET @r = JSON_INSERT(JSON_OBJECT('z',1), '$.d', JSON_QUERY(@c,'$')); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +let $m= 0; +while ($m < 2) +{ + let $n= 0; + while ($n < 60) + { + eval SET @d = CONCAT('{"a', REPEAT('x', $m), '":[', REPEAT('1,', $n), + '{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); + SET SESSION debug_dbug = '+d,json_nice_append_out_of_memory'; + + SET @r = NULL; + --error 0,5 + SET @r = JSON_EXTRACT(@d, '$**.k'); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET @r = NULL; + --error 0,5 + SET @r = JSON_EXTRACT(@d, '$.b'); + SET @refused = @refused + (@r IS NULL); + SET @bad = @bad + IF(@r IS NULL, 0, 1 - JSON_VALID(CONCAT(@r, ''))); + + SET SESSION debug_dbug = DEFAULT; + inc $n; + } + inc $m; +} + +--enable_warnings +--enable_result_log +--enable_query_log + +--echo # +--echo # Nothing came back half written, and the sweep did reach the +--echo # failures. +--echo # +SELECT @bad AS answers_that_do_not_parse, @refused > 0 AS some_were_refused; + +--echo # +--echo # The same values with the room there, so what the sweeps above +--echo # compare against is an answer that arrives. +--echo # +SET @a = CONCAT('{"a":1,', REPEAT('"p":1,', 110), + '"kkkk":{"b":[]},"z":[[]],"y":2}'); +SET @d = CONCAT('{"a":[', REPEAT('1,', 40), + '{"k":[2,3]},[],{},[[4]],5],"b":{"c":[6,[7]]}}'); +SELECT JSON_VALID(CONCAT(JSON_INSERT(JSON_OBJECT('z',1), '$.d', + JSON_QUERY(@a,'$')), '')) AS parses; +SELECT JSON_VALID(CONCAT(JSON_EXTRACT(@d, '$**.k'), '')) AS parses; +SELECT JSON_EXTRACT(@d, '$.b') AS answer; + +--echo # +--echo # A value that stops in the middle of a key, which is the one way +--echo # the walk gives up that is not about room. +--echo # +SELECT JSON_EXTRACT('{"a":{"bb', '$.a') AS unfinished_key; +SELECT JSON_EXTRACT('{"a":[1,{"c', '$.a') AS unfinished_key; diff --git a/mysql-test/main/func_json_wide_charset.result b/mysql-test/main/func_json_wide_charset.result new file mode 100644 index 0000000000000..43855f3154aad --- /dev/null +++ b/mysql-test/main/func_json_wide_charset.result @@ -0,0 +1,74 @@ +SET @save_collation_connection= @@collation_connection; +# +# Reading a value out. The result holds the value itself, in the +# width the character set gives it. +# +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a')) AS scalar_bytes; +scalar_value scalar_bytes +1 0031 +SELECT JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a') AS object_value, +HEX(JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a')) AS object_bytes; +object_value object_bytes +{"x": 1} 007B002200780022003A00200031007D +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b') AS several_paths, +HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b')) AS several_bytes; +several_paths several_bytes +[1, 2] 005B0031002C00200032005D +SELECT JSON_VALID(JSON_EXTRACT('{"a":{"x":1}}','$.a')) AS result_is_a_document; +result_is_a_document +1 +# +# Appending to an array, and to something that is not one and has +# to be wrapped in one first. Both arms copy the document. +# +SELECT JSON_ARRAY_APPEND('{"a":[1,2]}','$.a',3) AS target_is_array; +target_is_array +{"a": [1, 2, 3]} +SELECT JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS target_is_scalar; +target_is_scalar +{"a": [1, 3]} +SELECT JSON_ARRAY_APPEND('{"a":{"x":1}}','$.a',3) AS target_is_object; +target_is_object +{"a": [{"x": 1}, 3]} +SELECT JSON_VALID(JSON_ARRAY_APPEND('{"a":1}','$.a',3)) AS result_is_a_document; +result_is_a_document +1 +# +# The same at two bytes to the character, and at four. +# +SET collation_connection='ucs2_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +SET collation_connection='utf32_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +# +# A character set of one byte answers as it always did. +# +SET collation_connection='utf8mb4_general_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +SET collation_connection='latin1_swedish_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, +JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; +scalar_value wrapped +1 {"a": [1, 3]} +# +# A document that is genuinely malformed is still refused, and the +# position named is a position in what the caller sent. +# +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":oops}','$.a') AS malformed; +malformed +NULL +Warnings: +Warning 4038 Syntax error in JSON text in argument 1 to function 'json_extract' at position 24 +SET @@collation_connection= @save_collation_connection; diff --git a/mysql-test/main/func_json_wide_charset.test b/mysql-test/main/func_json_wide_charset.test new file mode 100644 index 0000000000000..d425143e3e1cf --- /dev/null +++ b/mysql-test/main/func_json_wide_charset.test @@ -0,0 +1,67 @@ +# +# Reading and editing a document written in a character set whose +# characters are more than one byte wide. +# +# A value taken out of the document is already in the character set the +# result is being built in, so it has to be copied rather than written. +# Writing it converts it a second time, which used to turn the value +# into something that could not be read back - and, because it was read +# back, into NULL and a complaint about a character the caller never +# sent. +# +--disable_service_connection + +SET @save_collation_connection= @@collation_connection; + +--echo # +--echo # Reading a value out. The result holds the value itself, in the +--echo # width the character set gives it. +--echo # +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a')) AS scalar_bytes; +SELECT JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a') AS object_value, + HEX(JSON_EXTRACT('{"a":{"x":1},"b":2}','$.a')) AS object_bytes; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b') AS several_paths, + HEX(JSON_EXTRACT('{"a":1,"b":2}','$.a','$.b')) AS several_bytes; +SELECT JSON_VALID(JSON_EXTRACT('{"a":{"x":1}}','$.a')) AS result_is_a_document; + +--echo # +--echo # Appending to an array, and to something that is not one and has +--echo # to be wrapped in one first. Both arms copy the document. +--echo # +SELECT JSON_ARRAY_APPEND('{"a":[1,2]}','$.a',3) AS target_is_array; +SELECT JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS target_is_scalar; +SELECT JSON_ARRAY_APPEND('{"a":{"x":1}}','$.a',3) AS target_is_object; +SELECT JSON_VALID(JSON_ARRAY_APPEND('{"a":1}','$.a',3)) AS result_is_a_document; + +--echo # +--echo # The same at two bytes to the character, and at four. +--echo # +SET collation_connection='ucs2_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +SET collation_connection='utf32_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +--echo # +--echo # A character set of one byte answers as it always did. +--echo # +SET collation_connection='utf8mb4_general_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +SET collation_connection='latin1_swedish_ci'; +SELECT JSON_EXTRACT('{"a":1,"b":2}','$.a') AS scalar_value, + JSON_ARRAY_APPEND('{"a":1}','$.a',3) AS wrapped; + +--echo # +--echo # A document that is genuinely malformed is still refused, and the +--echo # position named is a position in what the caller sent. +--echo # +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a":1,"b":oops}','$.a') AS malformed; + +SET @@collation_connection= @save_collation_connection; diff --git a/mysql-test/main/func_json_wide_compose.result b/mysql-test/main/func_json_wide_compose.result new file mode 100644 index 0000000000000..d5d73b3abbb5d --- /dev/null +++ b/mysql-test/main/func_json_wide_compose.result @@ -0,0 +1,154 @@ +# +# 1. Merging, which writes its own punctuation between a key and +# the value copied in after it +# +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) +AS ucs2_patch_agrees; +ucs2_patch_agrees +1 +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf16), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING utf16), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) +AS utf16_patch_agrees; +utf16_patch_agrees +1 +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf32), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING utf32), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) +AS utf32_patch_agrees; +utf32_patch_agrees +1 +# +# A key both documents hold, where the value is written out +# rather than copied and there is no space to inherit. +# +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"a":9}', '$')) +AS ucs2_shared_key_agrees; +ucs2_shared_key_agrees +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"a":9}', '$')) +AS ucs2_preserve_agrees; +ucs2_preserve_agrees +1 +# +# Two arrays, and an array with something that is not one. +# These take the arm of the merging that copies whole spans of +# both documents rather than walking their keys, so they are +# where a copy put through a converting append shows up. +# +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('[3, 4]' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), +JSON_EXTRACT('[3, 4]', '$')) +AS ucs2_arrays_agree; +ucs2_arrays_agree +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('[1, 2]' USING utf32), '$'), +JSON_EXTRACT(CONVERT('[3, 4]' USING utf32), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), +JSON_EXTRACT('[3, 4]', '$')) +AS utf32_arrays_agree; +utf32_arrays_agree +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('{"a": 1}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('2' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a": 1}', '$'), +JSON_EXTRACT('2', '$')) +AS ucs2_wrap_agrees; +ucs2_wrap_agrees +1 +SELECT CONVERT(JSON_MERGE_PRESERVE( +JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"a": 3}' USING ucs2), '$')) +USING utf8mb4) += JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), +JSON_EXTRACT('{"a": 3}', '$')) +AS ucs2_array_object_agree; +ucs2_array_object_agree +1 +# +# 2. The other six, each composing the same document a different +# way, so that a one-byte assumption anywhere shows here +# +SELECT CONVERT(JSON_INSERT(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), +'$'), '$.c', 3) USING utf8mb4) += JSON_INSERT(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.c', 3) +AS ucs2_insert_agrees; +ucs2_insert_agrees +1 +SELECT CONVERT(JSON_SET(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), +'$'), '$.a', 9) USING utf8mb4) += JSON_SET(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) +AS ucs2_set_agrees; +ucs2_set_agrees +1 +SELECT CONVERT(JSON_REPLACE(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), +'$'), '$.a', 9) USING utf8mb4) += JSON_REPLACE(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) +AS ucs2_replace_agrees; +ucs2_replace_agrees +1 +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), +'$'), '$[0]') USING utf8mb4) += JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') +AS ucs2_remove_agrees; +ucs2_remove_agrees +1 +SELECT CONVERT(JSON_ARRAY_APPEND(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), +'$'), '$', 3) USING utf8mb4) += JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', 3) +AS ucs2_append_agrees; +ucs2_append_agrees +1 +SELECT CONVERT(JSON_ARRAY_INSERT(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), +'$'), '$[0]', 9) USING utf8mb4) += JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', 9) +AS ucs2_array_insert_agrees; +ucs2_array_insert_agrees +1 +# +# 3. What the wide answers actually say, written out, so that a +# change to BOTH sides at once cannot pass unseen +# +SELECT CONVERT(JSON_MERGE_PATCH( +JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), +JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) +USING utf8mb4) AS ucs2_patch_written; +ucs2_patch_written +{"a": 1, "b": 2, "c": 3} +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), +'$'), '$[0]') USING utf8mb4) +AS ucs2_remove_written; +ucs2_remove_written +[2, 3] +SELECT JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), +JSON_EXTRACT('{"c":3}', '$')) AS utf8_patch_written; +utf8_patch_written +{"a": 1, "b": 2, "c": 3} diff --git a/mysql-test/main/func_json_wide_compose.test b/mysql-test/main/func_json_wide_compose.test new file mode 100644 index 0000000000000..c96f6d8330c7b --- /dev/null +++ b/mysql-test/main/func_json_wide_compose.test @@ -0,0 +1,152 @@ +# +# Composing a document in a character set that writes a letter, and a +# space, in more than one byte. +# +# A set that cannot encode a document at all is refused elsewhere; these +# are the sets that CAN, and so are the ones where composing has to +# count characters rather than bytes. A space in ucs2 is two bytes, in +# utf32 four, and the first of those bytes is a zero - so anything that +# looks at one byte to decide whether it is standing on a space decides +# wrongly, and quietly, in exactly these sets. +# +# Every case is written as a comparison against the same composition in +# utf8, which is the answer these have to agree with. Recording this +# file cannot therefore write a defect down as expected: a doubled or +# missing space makes the two sides differ whatever is recorded. +# + +--enable_warnings + +--echo # +--echo # 1. Merging, which writes its own punctuation between a key and +--echo # the value copied in after it +--echo # +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) + AS ucs2_patch_agrees; + +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf16), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING utf16), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) + AS utf16_patch_agrees; + +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING utf32), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING utf32), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) + AS utf32_patch_agrees; + +--echo # +--echo # A key both documents hold, where the value is written out +--echo # rather than copied and there is no space to inherit. +--echo # +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"a":9}', '$')) + AS ucs2_shared_key_agrees; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"a":9}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"a":9}', '$')) + AS ucs2_preserve_agrees; + +--echo # +--echo # Two arrays, and an array with something that is not one. +--echo # These take the arm of the merging that copies whole spans of +--echo # both documents rather than walking their keys, so they are +--echo # where a copy put through a converting append shows up. +--echo # +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('[3, 4]' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), + JSON_EXTRACT('[3, 4]', '$')) + AS ucs2_arrays_agree; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('[1, 2]' USING utf32), '$'), + JSON_EXTRACT(CONVERT('[3, 4]' USING utf32), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), + JSON_EXTRACT('[3, 4]', '$')) + AS utf32_arrays_agree; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('{"a": 1}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('2' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('{"a": 1}', '$'), + JSON_EXTRACT('2', '$')) + AS ucs2_wrap_agrees; + +SELECT CONVERT(JSON_MERGE_PRESERVE( + JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"a": 3}' USING ucs2), '$')) + USING utf8mb4) + = JSON_MERGE_PRESERVE(JSON_EXTRACT('[1, 2]', '$'), + JSON_EXTRACT('{"a": 3}', '$')) + AS ucs2_array_object_agree; + +--echo # +--echo # 2. The other six, each composing the same document a different +--echo # way, so that a one-byte assumption anywhere shows here +--echo # +SELECT CONVERT(JSON_INSERT(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), + '$'), '$.c', 3) USING utf8mb4) + = JSON_INSERT(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.c', 3) + AS ucs2_insert_agrees; + +SELECT CONVERT(JSON_SET(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), + '$'), '$.a', 9) USING utf8mb4) + = JSON_SET(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) + AS ucs2_set_agrees; + +SELECT CONVERT(JSON_REPLACE(JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), + '$'), '$.a', 9) USING utf8mb4) + = JSON_REPLACE(JSON_EXTRACT('{"a":1,"b":2}', '$'), '$.a', 9) + AS ucs2_replace_agrees; + +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), + '$'), '$[0]') USING utf8mb4) + = JSON_REMOVE(JSON_EXTRACT('[1, 2, 3]', '$'), '$[0]') + AS ucs2_remove_agrees; + +SELECT CONVERT(JSON_ARRAY_APPEND(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), + '$'), '$', 3) USING utf8mb4) + = JSON_ARRAY_APPEND(JSON_EXTRACT('[1, 2]', '$'), '$', 3) + AS ucs2_append_agrees; + +SELECT CONVERT(JSON_ARRAY_INSERT(JSON_EXTRACT(CONVERT('[1, 2]' USING ucs2), + '$'), '$[0]', 9) USING utf8mb4) + = JSON_ARRAY_INSERT(JSON_EXTRACT('[1, 2]', '$'), '$[0]', 9) + AS ucs2_array_insert_agrees; + +--echo # +--echo # 3. What the wide answers actually say, written out, so that a +--echo # change to BOTH sides at once cannot pass unseen +--echo # +SELECT CONVERT(JSON_MERGE_PATCH( + JSON_EXTRACT(CONVERT('{"a":1,"b":2}' USING ucs2), '$'), + JSON_EXTRACT(CONVERT('{"c":3}' USING ucs2), '$')) + USING utf8mb4) AS ucs2_patch_written; +SELECT CONVERT(JSON_REMOVE(JSON_EXTRACT(CONVERT('[1, 2, 3]' USING ucs2), + '$'), '$[0]') USING utf8mb4) + AS ucs2_remove_written; +SELECT JSON_MERGE_PATCH(JSON_EXTRACT('{"a":1,"b":2}', '$'), + JSON_EXTRACT('{"c":3}', '$')) AS utf8_patch_written; diff --git a/mysql-test/main/mysql-metadata.result b/mysql-test/main/mysql-metadata.result index 1530465eaf8bd..09687f8b088c2 100644 --- a/mysql-test/main/mysql-metadata.result +++ b/mysql-test/main/mysql-metadata.result @@ -8,6 +8,9 @@ js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +Warnings: +Warning 4271 CHECK constraint of column 'js2' asks more than JSON_VALID() of it; the column is not a JSON column +Warning 4269 CHECK constraint of column 'js3' calls JSON_VALID() on something other than 'js3'; the column is not a JSON column -------------- SELECT * FROM t1 -------------- @@ -44,7 +47,7 @@ Catalog: `def` Database: `test` Table: `t1` Org_table: `t1` -Type: BLOB (format=json) +Type: BLOB Collation: latin1_swedish_ci (8) Length: 65535 Max_length: 0 diff --git a/mysql-test/main/mysql-metadata.test b/mysql-test/main/mysql-metadata.test index bab44496f7880..8978ce49d6832 100644 --- a/mysql-test/main/mysql-metadata.test +++ b/mysql-test/main/mysql-metadata.test @@ -6,12 +6,16 @@ --echo # SET NAMES utf8; +# The check constraints are looked at while the column definitions are +# validated, which for a prepared CREATE TABLE is prepare time. +--enable_prepare_warnings CREATE TABLE t1 ( js0 JSON, js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +--disable_prepare_warnings --replace_regex /0 rows in set [(].*[)]/0 rows in set (TIME)/ --exec $MYSQL -vvv --column-type-info --database=test -e "SELECT * FROM t1;" diff --git a/mysql-test/main/type_json.result b/mysql-test/main/type_json.result index 5c7e4a9718366..708097558341b 100644 --- a/mysql-test/main/type_json.result +++ b/mysql-test/main/type_json.result @@ -57,7 +57,7 @@ show create table t2; Table Create Table t2 CREATE TABLE `t2` ( `a` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`a`)), - `t` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL + `t` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci show create table t3; Table Create Table @@ -111,11 +111,14 @@ js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +Warnings: +Warning 4271 CHECK constraint of column 'js2' asks more than JSON_VALID() of it; the column is not a JSON column +Warning 4269 CHECK constraint of column 'js3' calls JSON_VALID() on something other than 'js3'; the column is not a JSON column SELECT * FROM t1; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr def test t1 t1 js0 js0 252 (format=json) 4294967295 0 Y 144 0 33 def test t1 t1 js1 js1 252 (format=json) 196605 0 Y 16 0 33 -def test t1 t1 js2 js2 252 (format=json) 196605 0 Y 16 0 33 +def test t1 t1 js2 js2 252 196605 0 Y 16 0 33 def test t1 t1 js3 js3 252 196605 0 Y 16 0 33 js0 js1 js2 js3 SELECT js0, JSON_COMPACT(js0), JSON_COMPACT('{}') FROM t1; @@ -145,8 +148,8 @@ JSON_ARRAYAGG(1) JSON_ARRAYAGG(a) [1] [{"a":"b"}] SELECT JSON_OBJECTAGG('a','b'), JSON_OBJECTAGG('a',a) FROM t1; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def JSON_OBJECTAGG('a','b') 252 (format=json) 9437184 9 Y 0 0 33 -def JSON_OBJECTAGG('a',a) 252 (format=json) 12582912 15 Y 128 0 33 +def JSON_OBJECTAGG('a','b') 252 (format=json) 9437190 9 Y 0 0 33 +def JSON_OBJECTAGG('a',a) 252 (format=json) 12582918 15 Y 128 0 33 JSON_OBJECTAGG('a','b') JSON_OBJECTAGG('a',a) {"a":"b"} {"a":{"a":"b"}} DROP TABLE t1; @@ -155,7 +158,7 @@ DROP TABLE t1; # SELECT json_object('a', (SELECT json_objectagg(b, c) FROM (SELECT 'b','c') d)) AS j FROM DUAL; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def j 250 (format=json) 9437310 16 Y 0 39 33 +def j 250 (format=json) 18874494 16 Y 0 39 33 j {"a": {"b":"c"}} # diff --git a/mysql-test/main/type_json.test b/mysql-test/main/type_json.test index 8effe78f803d4..7af47326d1f2a 100644 --- a/mysql-test/main/type_json.test +++ b/mysql-test/main/type_json.test @@ -74,12 +74,16 @@ select cast('{a:1}' as json); --echo # SET NAMES utf8; +# The check constraints are looked at while the column definitions are +# validated, which for a prepared CREATE TABLE is prepare time. +--enable_prepare_warnings CREATE TABLE t1 ( js0 JSON, js1 TEXT CHECK (JSON_VALID(js1)), js2 TEXT CHECK (LENGTH(js2) > 0 AND JSON_VALID(js2)), js3 TEXT CHECK (LENGTH(js2) > 0 OR JSON_VALID(js2)) ) CHARACTER SET utf8; +--disable_prepare_warnings --disable_view_protocol --disable_ps_protocol --enable_metadata diff --git a/mysql-test/suite/json/r/json_no_table.result b/mysql-test/suite/json/r/json_no_table.result index 5b838a5414765..eda51094356ff 100644 --- a/mysql-test/suite/json/r/json_no_table.result +++ b/mysql-test/suite/json/r/json_no_table.result @@ -2930,7 +2930,7 @@ Warnings: Warning 4036 Character disallowed in JSON in argument 1 to function 'json_compact' at position 1 select json_unquote(json_compact(st_geomfromtext('point(1 1)'))); json_unquote(json_compact(st_geomfromtext('point(1 1)'))) -ð?ð? +ð?ð? SELECT JSON_UNQUOTE( '"abc"' ); JSON_UNQUOTE( '"abc"' ) abc diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index 2509f8f63e2a2..021251ed0eb79 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -152,7 +152,26 @@ void *my_realloc(PSI_memory_key key, void *old_point, size_t size, myf my_flags) DBUG_ASSERT((old_flags & 1) == MY_TEST(my_flags & MY_THREAD_SPECIFIC)); size= ALIGN_SIZE(size); - mh= sf_realloc(old_mh, size + HEADER_SIZE, my_flags); + + /* + Growing an allocation can fail where the first one succeeded, and a + buffer that is already held is where most of the unchecked writes in + the server are: the code that asked for the memory is long gone, and + what is left is an append that assumes it will fit. my_malloc() + above can be made to fail for a test; without the same here, none of + those paths can be reached at all. + + Kept apart from "simulate_out_of_memory" rather than folded into it, + so that a test can fail the growing of a buffer without also failing + every first allocation that happens to fall in the same window - + which is usually the whole point, the two failing in different + places for different reasons. A request that shrinks is left alone, + keeping the guarantee below that a smaller size never fails. + */ + if (size > old_size && DBUG_IF("simulate_realloc_out_of_memory")) + mh= NULL; + else + mh= sf_realloc(old_mh, size + HEADER_SIZE, my_flags); if (mh == NULL) { diff --git a/sql/derived_handler.cc b/sql/derived_handler.cc index cddd1200f5d0f..131aacd150a84 100644 --- a/sql/derived_handler.cc +++ b/sql/derived_handler.cc @@ -53,6 +53,12 @@ int Pushdown_derived::execute() DBUG_ENTER("Pushdown_derived::execute"); + /* + Dropped where the table is given to the engine - see + Pushdown_query::execute(). + */ + DBUG_ASSERT(!table->is_valid_json_static_set); + if ((err= handler->init_scan())) goto error; @@ -106,6 +112,12 @@ void derived_handler::set_derived(TABLE_LIST *tbl) { derived= tbl; table= tbl->table; + /* + The materialisation table was built by the server and is written by + this engine, row by row into record[0] - see + TABLE::set_filled_by_engine(). + */ + table->set_filled_by_engine(); unit= tbl->derived; select= unit->first_select(); tmp_table_param= ((select_unit *)(unit->result))->get_tmp_table_param(); diff --git a/sql/field.cc b/sql/field.cc index 57a08128e6989..a497b15c02e0f 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -36,6 +36,11 @@ #include "tztime.h" // struct Time_zone #include "filesort.h" // change_double_for_sort #include "log_event.h" // class Table_map_log_event +#include "sql_type_json.h" // Type_handler_json_common +#ifndef DBUG_OFF +#include "item_jsonfunc.h" // json_value_reads_as_document, + // json_value_is_nice +#endif #include // Maximum allowed exponent value for converting string to decimal @@ -1505,6 +1510,7 @@ bool Field::sp_prepare_and_store_item(THD *thd, Item **value) DBUG_ASSERT(value); Item *expr_item; + int store_rc; if (!(expr_item= thd->sp_fix_func_item_for_assignment(this, value))) goto error; @@ -1514,10 +1520,18 @@ bool Field::sp_prepare_and_store_item(THD *thd, Item **value) /* Save the value in the field. Convert the value if needed. */ - expr_item->save_in_field(this, 0); + store_rc= expr_item->save_in_field(this, 0); if (likely(!thd->is_error())) + { + /* + The one place a stored program's variable is written, and so the + one place there is anything to say about what is in it - see + Field::set_json_held_marks(). + */ + set_json_held_marks(expr_item, store_rc); DBUG_RETURN(false); + } error: /* @@ -1527,6 +1541,13 @@ bool Field::sp_prepare_and_store_item(THD *thd, Item **value) set x = x + 1; */ set_null(); + /* + The assignment failed, so nothing attests to what this field holds: + the bytes are whatever the store put down before the error, under a + NULL that was set without clearing them. The marks the previous + assignment left go with them. + */ + clear_json_held_marks(); DBUG_ASSERT(thd->is_error()); DBUG_RETURN(true); } @@ -2687,6 +2708,472 @@ Field *Field::clone(MEM_ROOT *root, TABLE *new_table, my_ptrdiff_t diff) } +/* + Whether a store into this field that answered 0 puts down exactly the + characters it was handed. + + Only such a field can carry what an Item answered about those characters, + JSON being written in characters. CHAR and BINARY pad what they are + given and the padding is part of what is stored; ENUM and SET keep the + member they matched rather than the text that matched it; GEOMETRY and + the compressed types put down something else entirely; and the rest + convert. Every one of those answers 0 while doing it. + + Asked of real_type() rather than answered by a virtual, so that a field + type has to be NAMED here to be trusted. A field type nobody has looked + at is untrusted, and a new subclass of a trusted one does not inherit the + trust - which is the way round that costs a check that was going to pass, + rather than the way round that admits a value that is not a document. + + That holds for every subclass that says what it is: Field_geom derives + from Field_blob and answers MYSQL_TYPE_GEOMETRY, so naming the blobs + below does not name it. The compressed classes are the exception and + the reason for the question below the comment: they derive from a named + type and go on answering its real_type(), saying nothing anywhere that + a switch could read. They are asked about their compression instead, + which is the one thing they do say. + + A virtual is safe THAT way round. It is asked in order to REFUSE, so a + field type that has never heard of it is left where it started - refused + unless something below names it - and the inheritance that costs the + trust cannot grant any. +*/ + +bool Field::is_character_preserving() const +{ + if (compression_method()) + return false; + + switch (real_type()) { + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_TINY_BLOB: + case MYSQL_TYPE_MEDIUM_BLOB: + case MYSQL_TYPE_LONG_BLOB: + case MYSQL_TYPE_BLOB: + return true; + default: + return false; + } +} + + +/* + The one rule every store site applies, asked in one place. + + A store that returned 0 into a field that puts down what it is given + mapped every character across without putting anything in its place. + JSON is written in characters, so a document that arrived that way is + still a document; anything else leaves the caller with nothing to say. + + Mapped across, and not merely carried across. A store into or out of + a binary field keeps the bytes and calls them by the other set's name, + which is a different string of characters - so a document written in + ucs2 and put into a BLOB arrives as bytes that read back as nothing at + all. conversion_keeps_characters() is the question, rather than the + plain my_charset_same() the two standing grants ask: those are said + once about values not yet made, where a store that would convert has + not happened yet and cannot be counted on, and this one is said with + the store already done and its answer in hand. + + The item is asked before the field type is, the two being in the order + that refuses soonest. Every value a table is given arrives here, and + one virtual answering no for everything that is not a document costs + less than the two virtuals and the switch that work out what this + field would have done with it. +*/ + +bool Field::is_attestation_preserved(const Item *item, int store_rc) const +{ + CHARSET_INFO *from= item->collation.collation; + + return store_rc == 0 && item->is_valid_json() && + is_character_preserving() && + String_copier::conversion_keeps_characters(charset(), from) && + !table->in_use->is_error(); +} + + +void Field::set_is_valid_json(const Item *item, int store_rc) +{ + /* + The list of places that set a mark is closed, and every one of them + asks first whether any column of this table carries a check that + could ever read one. Said here as well as at each of them, because + the closed list is what the debug reading in TABLE rests on: a site + added later that sets a mark on a table nobody asked about is a site + that will go on setting marks nothing polices. + */ + DBUG_ASSERT(table->has_own_json_valid_check); + + if (is_attestation_preserved(item, store_rc)) + bitmap_set_bit(&table->is_valid_json_set, field_index); + else + bitmap_clear_bit(&table->is_valid_json_set, field_index); +} + + +/* + The three of these are here rather than beside their siblings in + field.h for one reason: JSON_DEPTH_UNKNOWN is declared in item.h, + which a field header does not see and should not have to. The call + costs nothing beside the reading it exists to save. + + Asked through the standing answer, so a column that has given that up + has given the depth up with it and cannot be asked to return a + figure about values it no longer attests to. +*/ + +uint Field::json_static_depth() const +{ + return is_valid_json_static() ? table->json_static_depth[field_index] + : JSON_DEPTH_UNKNOWN; +} + + +/* + What a value that has just arrived makes of the running figure. The + entry only ever rises, so this is asked in the same call as the store + and before anything reads the row - which is what makes a figure read + at row N no smaller than the depth of row N. + + A value nobody counted arrives as JSON_DEPTH_UNKNOWN, which is the + largest there is, so it takes the column there and nothing brings it + back. +*/ + +void Field::raise_json_static_depth(uint depth) +{ + if (table->json_static_depth && depth > table->json_static_depth[field_index]) + table->json_static_depth[field_index]= depth; +} + + +void Field::forget_json_static_depth() +{ + if (table->json_static_depth) + table->json_static_depth[field_index]= JSON_DEPTH_UNKNOWN; +} + + +/* + Said once, of a field of a temporary table the server is building for + itself, while the one item that will ever write it is in hand. + + Three things are asked, and all three are properties of the pair rather + than of any value: + + 1. the item returns a document EVERY time it is evaluated, not just + the time somebody happens to look - which is the question this field + needs, there being no evaluation to have looked at yet; + 2. the field puts down the characters it is given, by the same list + is_character_preserving() uses for the check skip; + 3. it writes them in the character set they arrive in, so no character + is encoded again on the way down. A document rewritten into another + character set is still a document, but only where every character of + it can be written there, and that is not a question about the pair. + + The field being JSON-typed is asked too, and not because it makes the + value any more of a document: it is what decides whether anybody will + ever ask; an attestation about a column nothing reads as JSON is + never looked at. + + The formatting is granted here too, as a yes, without anything being + asked. Nothing could be asked: the item is in hand, but the question + is about values it has not made yet, and unlike being a document the + formatting is settled one value at a time. So it is granted here and + spent afterwards - each value that arrives written another way takes + it back, and a field that is never written keeps it and is never read. + + How deep the values go is settled the same way and for the same + reason, but it runs the other direction: it starts at nothing, being + the deepest of no rows, and each value that arrives raises it. Where + no answer is given at all it starts at the largest figure there is, + so that a column nobody attested to cannot be read as a shallow one. +*/ + +void Field::set_is_valid_json_static(const Item *item) +{ + if (!table->is_valid_json_static_set) + return; + DBUG_ASSERT(table->is_nice_json_static_set && table->json_static_depth); + if (item->is_valid_json_static() && is_character_preserving() && + Type_handler_json_common::is_json_type_handler(type_handler()) && + my_charset_same(charset(), item->collation.collation)) + { + bitmap_set_bit(table->is_valid_json_static_set, field_index); + bitmap_set_bit(table->is_nice_json_static_set, field_index); + table->json_static_depth[field_index]= 0; + } + else + { + bitmap_clear_bit(table->is_valid_json_static_set, field_index); + bitmap_clear_bit(table->is_nice_json_static_set, field_index); + table->json_static_depth[field_index]= JSON_DEPTH_UNKNOWN; + } +} + + +/* + Asked after every store into such a field, because the promise above is + about characters and a store is where characters go missing. + + It is asked as "how much did you keep", not "did anything go wrong", + because going wrong is not reported here: Field_longstr:: + report_if_important_data() answers 0 for a truncation unless + count_cuted_fields is raised above CHECK_FIELD_EXPRESSION, and writing + a temporary table does not raise it. A store that dropped the tail of + a document would return success and warn nobody. + + Losing the mark is for good. The value that arrived short is not a + document, and one field of one row being wrong is enough - a reader + believes the field, not the row. + + No function asks for less room than it goes on to use, so nothing gets + here having lost anything; what this catches is one of them ever + doing so. A store that comes up short is arranged for rather than + waited for, so that the losing of the mark is exercised and not merely + reasoned about. + + The formatting is asked of the item instead of the store, that being + where the answer is: a store puts characters down and knows nothing + about the order they are in, while the item has just written them and + says how. The two are asked together because they are spent together + - a store that lost characters has lost the formatting with them, which + is why the length is settled first and the formatting only after. + + The depth comes off the item for the same reason and is taken the + same way round: a value the item did not count arrives as the largest + figure there is and takes the column with it, so nothing has to + decide here what an absent answer means. +*/ + +void Field::confirm_is_valid_json_static(uint32 handed_length, bool is_nice, + uint depth, CHARSET_INFO *cs) +{ + DBUG_EXECUTE_IF("json_tmp_store_kept_short", handed_length++;); + /* + The length alone is not the question, and both the other two ask the + rest of it. A store between a wide set and the binary one keeps + every byte and calls them by the other set's name, so the length + comes back unchanged over characters that are no longer the ones + that were written: a ucs2 document arrives as bytes beginning 00 7B, + which nothing reads as a document. + + Asked here rather than left to the grant, because on the union route + the grant cannot ask it. The field is made from a type holder, whose + own set is the aggregated one and so is always the field's - the + question answers itself there and decides nothing. The branches that + do the storing are the ones with something to say, and this is where + each of them says it. + */ + if (value_length() != handed_length || + !String_copier::conversion_keeps_characters(charset(), cs)) + { + clear_is_valid_json_static(); + return; + } + confirm_json_static_value(is_nice, depth); +} + + +/* + What is left of a confirm once the answer itself has survived it: the + formatting and the depth, which are said about the values rather than + about the column and so are taken one value at a time. + + Both confirms end here, and the debug reading is written once for the + same reason the rule above it is: two copies of a check are two chances + for one of them to stop matching what it is checking. +*/ + +void Field::confirm_json_static_value(bool is_nice, uint depth) +{ + if (!is_nice) + clear_is_nice_json_static(); + raise_json_static_depth(depth); +#ifndef DBUG_OFF + { + StringBuffer kept; + const String *v= val_str(&kept); + /* + Nothing else will ever read this back. The mark appears in no + result and changes no output, so a field wrongly carrying one goes + unnoticed until something acts on it - and by then the store that + made the promise is a long way from the code that believed it. + */ + DBUG_ASSERT(json_value_reads_as_document(v)); + DBUG_ASSERT(!is_nice_json_static() || json_value_is_nice(v)); + DBUG_ASSERT(json_static_depth() >= json_value_depth(v)); + } +#endif +} + + +/* + Asked after such a field is filled from another FIELD rather than from + an item, which is how a temporary table built out of an earlier one is + filled. + + The answer that field was given was copied at build time, before a row + of either table existed, so it is not the answer the source holds now. + It is an upper bound on it: the bit is set in one place, reached only + while such a table is being built, and everything that touches it + afterwards only ever clears, so a source that has given its answer up + cannot get it back and a copy of it can only be too generous. Asking + the source again at every fill is what turns the bound into the + answer, and it is asked where both tables are alive by construction + and the source's own store of this row is already done - so nothing + here rests on the order two tables are written in. + + The source is asked the same two things the store is asked: whether it + attests to its values at all, and whether this field kept every + character of what it holds. The character sets are asked about too, + the length being counted in bytes and two fields of different + character sets being able to agree on a length while disagreeing about + every character in it. A grant is only ever made where they match, so + this decides nothing today; it is here so that the byte count does not + quietly become the wrong question if a fill site ever brings a source + the grant did not look at. + + The formatting comes from the source as well, and here it has to: no + item was evaluated to fill this field, so the only thing that knows + how the characters are arranged is the field they were copied from. + It is a bound in the same way the answer above is, and turned into the + answer the same way, by asking again at every fill. + + The depth comes from the source too, and it is the source's figure + over all its rows rather than the depth of the one value being + copied - the source having kept a running deepest and not a per-row + one. That is too large rather than wrong, and too large is the + direction this figure is allowed to be wrong in. +*/ + +void Field::confirm_is_valid_json_static_from(Field *from) +{ + uint32 handed_length; + + /* + A NULL is not read as a document and nothing attests to it as one, + so a row that puts one here leaves the column's answer where it was. + */ + if (is_null()) + return; + + handed_length= from->value_length(); + DBUG_EXECUTE_IF("json_tmp_chain_kept_short", handed_length++;); + if (from->is_null() || !from->is_valid_json_static() || + !my_charset_same(charset(), from->charset()) || + value_length() != handed_length) + { + clear_is_valid_json_static(); + return; + } + confirm_json_static_value(from->is_nice_json_static(), + from->json_static_depth()); +} + + +/* + What this field is holding, where it is a stored program's variable and + something was said about the value put here - see TABLE::json_held_marks. + + A field with nowhere to keep an answer answers the same as one that was + never granted anything, which is the answer everything but a stored + program's variable gives and the answer those give until an assignment + says otherwise. +*/ + +bool Field::is_valid_json_held() const +{ + return table->json_held_marks && + table->json_held_marks[field_index].valid(); +} + + +bool Field::is_nice_json_held() const +{ + return table->json_held_marks && + table->json_held_marks[field_index].nice(); +} + + +uint Field::json_held_depth() const +{ + return table->json_held_marks ? table->json_held_marks[field_index].depth() + : JSON_DEPTH_UNKNOWN; +} + + +void Field::clear_json_held_marks() +{ + if (table->json_held_marks) + table->json_held_marks[field_index].clear(); +} + + +/* + Said once per assignment, with the item that was assigned in hand and + the store it just did already done. + + The rule is the one every store site applies - the item said it was a + document, the store kept every character of it, and nothing went wrong + while it happened - and it is applied here for the same reason it is + applied there. What is different is what the answer is about: a + column is attested once and read over many rows, while a variable + is attested as often as it is written and read only until it is + written again. So the formatting and the depth are taken from the item + as well, exactly rather than as a bound: there is one value here, the + item that made it is right here, and it is the only value the answer + has to cover. + + The store having returned 0 is asked about and not merely whether + anything went wrong, because in this direction the two are not the + same: a store that had to drop characters or write them another way + answers 2, and answers it while succeeding. Assigning a variable does + raise an error where a column store would not, but that is a property + of this path rather than of the rule, and the rule is the conservative + one either way. + + Nothing is cleared on the way in. Every way out of the funnel this is + called from ends either here or at the clear beside it, so a value + that never arrived leaves no answer standing over the bytes of the one + before it. Not clearing first is also what lets SET x = x keep an + answer: the item is this same field, and what it says is read after + the store rather than wiped before it. +*/ + +void Field::set_json_held_marks(const Item *item, int store_rc) +{ + if (!table->json_held_marks) + return; + Json_result_marks &marks= table->json_held_marks[field_index]; + /* + The same rule a column's store is attested by - see + Field::is_attestation_preserved() - and a NULL besides, which is + not read as a document and is not attested as one. + */ + if (!is_attestation_preserved(item, store_rc) || is_null()) + { + marks.clear(); + return; + } + { + /* + The value is read back out of the field rather than taken from the + item, so that what the answer is checked against is what a reader + of this field will find. Only the check reads it - see + Json_result_marks::set(), which uses the value for nothing else - + so the reading is the debug build's work, the same as at the + confirm above. + */ + IF_DBUG(StringBuffer kept;,) + marks.set(IF_DBUG(val_str(&kept), NULL), true, item->is_nice_json(), + item->last_depth()); + } +} + + int Field::set_default() { if (default_value) @@ -7397,17 +7884,22 @@ Field_longstr::report_if_important_data(const char *pstr, const char *end, void Field_longstr::make_send_field(Send_field *field) { Field_str::make_send_field(field); - if (check_constraint) - { - /* - Append the format that is implicitly implied by the CHECK CONSTRAINT. - For example: - CREATE TABLE t1 (js longtext DEFAULT NULL CHECK (json_valid(a))); - SELECT j FROM t1; - will add "format=json" to the extended type info metadata for t1.js. - */ - check_constraint->expr->set_format_by_check_constraint(field); - } + /* + Append the format that is implicitly implied by the CHECK CONSTRAINT. + For example: + CREATE TABLE t1 (js longtext DEFAULT NULL CHECK (json_valid(js))); + SELECT js FROM t1; + will add "format=json" to the extended type info metadata for t1.js. + + What is asked is what types the column, which is the same question + Field_string::type_handler() below asks and the same answer. A column + the client is told is a document is a column whose values it may pass + to a parser without quoting them, and that is what being typed JSON + means here; a check the column cannot pass without holding a document + is a weaker thing and does not type it. + */ + if (Type_handler_json_common::has_json_valid_constraint(this)) + Type_handler_json_common::set_format_name(field); } @@ -10845,8 +11337,12 @@ bool Column_definition::fix_attributes_temporal_with_time(uint int_part_length) bool Column_definition::validate_check_constraint(THD *thd) { - return check_constraint && - check_expression(check_constraint, &field_name, VCOL_CHECK_FIELD); + if (!check_constraint) + return false; + Type_handler_json_common::warn_if_json_valid_does_not_type(thd, + check_constraint, + field_name); + return check_expression(check_constraint, &field_name, VCOL_CHECK_FIELD); } @@ -11825,6 +12321,12 @@ Virtual_column_info* Virtual_column_info::clone(THD *thd) Virtual_column_info* dst= new (thd->mem_root) Virtual_column_info(*this); if (!dst) return NULL; + /* + The copy's expression reads other Item_fields than the ones this was + worked out against, so it has to be worked out again when the table + the copy belongs to is opened. + */ + dst->json_valid_field_index= NO_JSON_VALID_FIELD; if (expr) { dst->expr= expr->deep_copy_with_checks(thd); diff --git a/sql/field.h b/sql/field.h index 8ff4d90f38a57..71ae3d707e700 100644 --- a/sql/field.h +++ b/sql/field.h @@ -581,6 +581,9 @@ static inline const char *vcol_type_name(enum_vcol_info_type type) - whether the field is used in a partitioning expression */ +/* A check constraint that is not one column's JSON_VALID() over itself */ +#define NO_JSON_VALID_FIELD ((field_index_t) ~0U) + class Virtual_column_info: public Sql_alloc, private Type_handler_hybrid_field_type { @@ -603,12 +606,21 @@ class Virtual_column_info: public Sql_alloc, Lex_ident name; /* Name of constraint */ /* see VCOL_* (VCOL_FIELD_REF, ...) */ uint flags; + /* + Worked out once at open: for a column's own check constraint that asks + nothing but whether that very column holds a document, the index of the + column. A check over another column, one wrapped in anything else, and + a check belonging to the table rather than a column all leave this + NO_JSON_VALID_FIELD and are always run. See TABLE::is_valid_json_set. + */ + field_index_t json_valid_field_index; Virtual_column_info() :Type_handler_hybrid_field_type(&type_handler_null), vcol_type((enum_vcol_info_type)VCOL_TYPE_NONE), in_partitioning_expr(FALSE), stored_in_db(FALSE), - utf8(TRUE), automatic_name(FALSE), expr(NULL), flags(0) + utf8(TRUE), automatic_name(FALSE), expr(NULL), flags(0), + json_valid_field_index(NO_JSON_VALID_FIELD) { name.str= NULL; name.length= 0; @@ -1217,6 +1229,144 @@ class Field: public Value_source bitmap_clear_bit(&table->has_value_set, field_index); } + /* + Attest to the value just stored here, by the one rule every store site + applies: the Item said it was a document, the store kept every + character of it, and nothing went wrong while it happened. Anything + else clears the mark - a store that failed leaves bytes behind too. + See TABLE::is_valid_json_set. + + Written here and read nowhere here. The one place that reads a mark + is TABLE::verify_constraints, which has the bitmap in front of it and + asks it by name; giving this class the short question as well would + put "does this field attest to its bytes" within reach of anything + that wanted it, and this is not that question. It is true of one row + image between a store and the next check and false everywhere a + reader would think to ask, which is a thing to be read by the one + caller that knows that and by nobody who does not. + */ + bool is_character_preserving() const; + bool is_attestation_preserved(const Item *item, int store_rc) const; + void set_is_valid_json(const Item *item, int store_rc); + void clear_is_valid_json() + { + bitmap_clear_bit(&table->is_valid_json_set, field_index); + } + + /* + Attest to every value this field will ever hold, given once while + the table was being built. Only a temporary table the server built + for itself has anywhere to keep such an answer, so everything else + says no by having no bitmap at all. + See TABLE::is_valid_json_static_set. + */ + bool is_valid_json_static() const + { + return table->is_valid_json_static_set && + bitmap_is_set(table->is_valid_json_static_set, field_index); + } + /* + Whether every value that has reached this field was also written the + loose way - see TABLE::is_nice_json_static_set. Asked through the + answer above, a formatting being nothing to say about a column whose + values are not known to be documents. + */ + bool is_nice_json_static() const + { + return is_valid_json_static() && + bitmap_is_set(table->is_nice_json_static_set, field_index); + } + /* + How deep the deepest value that has reached this field goes, or + JSON_DEPTH_UNKNOWN where no such thing is being kept - see + TABLE::json_static_depth. Asked through the answer above like the + formatting is, and out of line because that constant is declared where + a field cannot see it. + */ + uint json_static_depth() const; + void raise_json_static_depth(uint depth); + void forget_json_static_depth(); + void set_is_valid_json_static(const Item *item); + void clear_is_nice_json_static() + { + if (table->is_nice_json_static_set) + bitmap_clear_bit(table->is_nice_json_static_set, field_index); + } + void clear_is_valid_json_static() + { + if (table->is_valid_json_static_set) + bitmap_clear_bit(table->is_valid_json_static_set, field_index); + /* + Kept in step rather than left to the reader above to hide, so that + the two bitmaps say the same thing as each other and not merely the + same thing when read in the one order. + */ + clear_is_nice_json_static(); + forget_json_static_depth(); + } + void confirm_is_valid_json_static(uint32 handed_length, bool is_nice, + uint depth, CHARSET_INFO *cs); + void confirm_is_valid_json_static_from(Field *from); + void confirm_json_static_value(bool is_nice, uint depth); + + /* + Attest to the one value this field is holding, where the field is a + stored program's variable - see TABLE::json_held_marks. Out of line + for the same reason the depth above is: what the answers are kept in + is declared where a field header cannot see it. + + Read without asking first whether there is anything here to read: a + field with nowhere to keep an answer gives the same answer as a + field that has one and was not granted it, and no caller has ever + had a reason to tell those two apart. + */ + bool is_valid_json_held() const; + bool is_nice_json_held() const; + uint json_held_depth() const; + void set_json_held_marks(const Item *item, int store_rc); + void clear_json_held_marks(); + + /* + What this field can say about the bytes it is holding, which is + whichever of the two channels above has anything to say. Only one + of them is ever there to be asked: a table cannot both be one the + server built for a query and be where a stored program keeps its + variables, so the two never disagree. The long form of why either + of them can be believed is at Item_field::is_valid_json(). + + Asked by everything that reads a value through a field rather than + through the item that made it - Item_field for the field it names, + and the Item_ref family for the result field it reads a copy out + of. + + A row holding no value at all is attested by neither, and that + is asked here rather than left to the reader. The standing grant is + about the column: it is given once, where the table is built, and + what it says is that every value the column comes to hold arrived + from something that attests to the values it makes. That is as + true of a row that came out SQL NULL as of any other, there being no + value in it to be wrong about - so the grant goes on standing over + such a row while there are no characters under it to have been + attested. A caller that reads it there puts the answer down + somewhere the NULL itself cannot go, the column taking whatever it + takes in place of one, and the answer is then about bytes nothing + ever saw. The held answers are put down one value at a time and are + already cleared for a NULL where they are set - see + Field::set_json_held_marks() - so the question is asked of both only + so that a reader never has to know which of the two it is reading. + */ + bool attests_is_valid_json() const + { return !is_null() && (is_valid_json_static() || is_valid_json_held()); } + bool attests_is_nice_json() const + { return !is_null() && (is_nice_json_static() || is_nice_json_held()); } + /* + The smaller of the two, which is the one that was answered: the + other is JSON_DEPTH_UNKNOWN, that being the largest there is and + what a field with nothing to say about a depth says. + */ + uint attested_json_depth() const + { return MY_MIN(json_static_depth(), json_held_depth()); } + virtual my_time_t get_timestamp(const uchar *pos, ulong *sec_part) const { DBUG_ASSERT(0); return 0; } my_time_t get_timestamp(ulong *sec_part) const @@ -5907,7 +6057,26 @@ class Copy_field :public Sql_alloc { Field *from_field,*to_field; String tmp; // For items - Copy_field() = default; + /* + set(uchar*, Field*) has no destination field to record and records + neither field, and set(Field*, Field*, bool) returns early for a + MYSQL_TYPE_NULL destination without recording them either. Both kinds + of entry go in the same array, so start them absent rather than + leaving them to read as whatever the memory held. + */ + /* + Whether the destination attests to every value it holds, worked out + where the entry is made rather than once a row. The bit behind it is + SET only while a temporary table is being built, which is before any + entry here is made; afterwards it is only ever cleared, and the + confirm the copy loop makes reads it again. So a yes recorded here + is an upper bound and costs at most a question that answers no, + while asking per row costs two loads and a test for every copied + column of every grouped query in the instance. + */ + bool to_needs_confirm; + Copy_field() + : from_field(NULL), to_field(NULL), to_needs_confirm(false) {} ~Copy_field() = default; void set(Field *to,Field *from,bool save); // Field to field void set(uchar *to,Field *from); // Field to string diff --git a/sql/field_conv.cc b/sql/field_conv.cc index b4e2c03013cd8..5d1c692245666 100644 --- a/sql/field_conv.cc +++ b/sql/field_conv.cc @@ -701,6 +701,7 @@ void Copy_field::set(Field *to,Field *from,bool save) } from_field=from; to_field=to; + to_needs_confirm= to->is_valid_json_static(); from_ptr=from->ptr; from_length=from->pack_length_in_rec(); to_ptr= to->ptr; @@ -909,9 +910,21 @@ static int field_conv_incompatible(Field *to, Field *from) int field_conv(Field *to,Field *from) { - return to->memcpy_field_possible(from) ? - field_conv_memcpy(to, from) : - field_conv_incompatible(to, from); + int rc= to->memcpy_field_possible(from) ? + field_conv_memcpy(to, from) : + field_conv_incompatible(to, from); + /* + A field of a temporary table the server built for itself attests to + every value it holds, and a value put here came from another FIELD + rather than from an item that could be asked. The source is asked + instead, and it is asked HERE so that the answer does not rest on a + list of callers being kept complete: the two routes that go around + this function ask for themselves, and everything that goes through it + is covered wherever it is written. + */ + if (unlikely(to->is_valid_json_static())) + to->confirm_is_valid_json_static_from(from); + return rc; } diff --git a/sql/group_by_handler.cc b/sql/group_by_handler.cc index 7b998494af9ae..2375e3da9538d 100644 --- a/sql/group_by_handler.cc +++ b/sql/group_by_handler.cc @@ -46,6 +46,15 @@ int Pushdown_query::execute(JOIN *join) TABLE *table= handler->table; DBUG_ENTER("Pushdown_query::execute"); + /* + Nothing attests to what an engine puts in a table it fills, so the + bitmaps are dropped where the table is given to the engine - see + TABLE::set_filled_by_engine(). Asserted here rather than there + because what has to hold is that EVERY way in has dropped them, and + the ways in all come through here. + */ + DBUG_ASSERT(!table->is_valid_json_static_set); + if ((err= handler->init_scan())) goto error; diff --git a/sql/item.cc b/sql/item.cc index b3fbf6b72e876..2d1b5c19b924a 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -43,6 +43,7 @@ // RESOLVED_AGAINST_ALIAS, ... #include "sql_expression_cache.h" #include "sql_lex.h" // empty_clex_str +#include "item_jsonfunc.h" // json_value_reads_as_document const String my_null_string("NULL", 4, default_charset_info); const String my_default_string("DEFAULT", 7, default_charset_info); @@ -1738,6 +1739,78 @@ String *Item_sp_variable::val_str(String *sp) } +/* + A JSON function keeps its document for the whole of its work. It + walks the document to find the place it was asked about, then works + out the rest of its arguments, and only then reads the pieces of the + document either side of that place. Working out an argument runs + whatever the caller wrote, and what the caller wrote can assign to + this very variable - the variables of a package body being in reach + of every routine in that package. Storing a longer value there + gives the variable a new buffer and lets go of the old one, which is + the buffer being read from. + + val_str() above returns a pointer INTO the variable rather than + bytes of the caller's own, and does so on purpose: a function that + builds into the buffer it is offered must not get to build into the + variable. What that leaves the caller holding is a view, good only + for as long as nothing writes where it points, so a caller that + means to keep it across working something else out cannot be given + one. Give that caller a copy. + + A user variable has always been read this way - user_var_entry:: + val_str() copies - which is why the same shape written with one of + those holds together. +*/ + +String *Item_sp_variable::val_json(String *str) +{ + DBUG_ASSERT(fixed()); + Item *it= this_item(); + String *res= it->val_json(str); + + null_value= it->null_value; + + /* + Nothing to copy, or worked out into the buffer that was offered + instead of read off the variable, in which case the bytes are the + caller's own already. + */ + if (!res || res == str) + return res; + + if (str->copy(res->ptr(), res->length(), res->charset()) || + DBUG_IF("json_variable_copy_out_of_memory")) + { + null_value= true; /* Out of memory. */ + return NULL; + } + + return str; +} + + +/* + A view into the variable is what val_str() above returns, and it + is enough for a caller that reads the bytes and is done with them + before anything else can run: nothing can write where they point + while nothing is running. So this is the one JSON caller that need + not be given a copy. See val_json() above for the caller that must + be. +*/ + +String *Item_sp_variable::val_json_at_once(String *str) +{ + DBUG_ASSERT(fixed()); + Item *it= this_item(); + String *res= it->val_json(str); + + null_value= it->null_value; + + return res; +} + + bool Item_sp_variable::val_native(THD *thd, Native *to) { return val_native_from_item(thd, this_item(), to); @@ -2281,6 +2354,24 @@ class Item_aggregate_ref : public Item_ref Item_ident::print(str, query_type); } Ref_Type ref_type() override final { return AGGREGATE_REF; } + + /* + The standing question, and only that one. split_sum_func2() puts + this in front of whatever it moves out of an expression - "or copy + it (in case of fields)", its own comment says - so an aggregate is + not the only thing under here and an aggregate's habits cannot be + assumed of it. The three answers about a VALUE are therefore left + to Item_ref, which asks the item what its result side passes + rather than what it made. + + This one is not about a value. It is asked while a temporary table + is being built, before a row exists, and by then the item underneath + has had its result field pointed at the very column being asked + about - so there is no result side to put the question to yet, and + the producer is the only thing there is to ask. + */ + bool is_valid_json_static() const override + { return (*ref)->is_valid_json_static(); } protected: Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } @@ -5347,11 +5438,46 @@ int Item_copy_string::save_in_field(Field *field, bool no_conversions) void Item_copy_string::copy() { + bool kept= true; String *res=item->val_str(&str_value); if (res && res != &str_value) - str_value.copy(*res); + kept= !str_value.copy(*res); null_value=item->null_value; -#ifndef DBUG_OFF + /* + The item was evaluated just above and nothing has evaluated it + since, so this is the one moment its answers are about the bytes + that are being kept. String::copy() takes the character set along + with them, so the characters those answers are about are the + characters kept here - where it kept them at all. A copy that could + not get the room leaves whatever was here before, which the value + side has always returned and which nothing here can attest to. + + A value that carries an answer is one that gets spliced into a + document rather than quoted into one, and which of the two happens + is decided by the type rather than by the answer. This item's type + is the one it was made over, so the two agree wherever that item is + the producer; where it is a character set conversion around the + producer they do not, the conversion carrying the answer through + while the type stops at it. Asking the same question the splice + will ask is what keeps this item from attesting to a value that is + going to be quoted. + + The splice asks is_json_type(), whose first question is this one and + whose second is whether the item is a conversion to look through. + A copy is not one and real_item() of a copy is the copy, so the + second question can only ever be answered no here - and answering it + is a dynamic_cast, on a path walked once a row for every copied + field in every grouped query, JSON or not. So the first question is + asked directly, once, where the type is settled, and the two are the + same question. + */ + DBUG_EXECUTE_IF("json_copy_not_kept", kept= false;); + if (null_value || !kept || !m_is_json) + m_marks.clear(); + else + m_marks.set(&str_value, item->is_valid_json(), item->is_nice_json(), + item->last_depth()); +#ifdef DBUG_ASSERT_EXISTS copied_in= 1; #endif } @@ -5386,7 +5512,7 @@ void Item_copy_real::copy() { cached_value= item->val_real(); null_value= item->null_value; -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS copied_in= 1; #endif } @@ -5504,6 +5630,24 @@ String* Item_ref_null_helper::val_str(String* s) } +String* Item_ref_null_helper::val_json(String* s) +{ + DBUG_ASSERT(fixed()); + String* tmp= (*ref)->val_json_result(s); + owner->was_null|= null_value= (*ref)->null_value; + return tmp; +} + + +String* Item_ref_null_helper::val_json_at_once(String* s) +{ + DBUG_ASSERT(fixed()); + String* tmp= (*ref)->val_json_at_once_result(s); + owner->was_null|= null_value= (*ref)->null_value; + return tmp; +} + + bool Item_ref_null_helper::val_native(THD *thd, Native *to) { return (owner->was_null|= val_native_from_item(thd, *ref, to)); @@ -7033,6 +7177,84 @@ void Item_field::make_send_field(THD *thd, Send_field *tmp_field) } +#ifdef DBUG_ASSERT_EXISTS +/* + What a field is attesting to, read back off the bytes it is holding. + A field that answers nothing is nothing to disagree with, and a row + with nothing in it is a field that answers nothing. + + Three bodies rather than one that settles all three at a stroke, + which is what a single walk over the value could do. The three are + asked one at a time, so one body would be walked three times rather + than once - and answering the formatting means writing the whole value + out again and comparing it, which the other two would then be paying + for as well. Reading each of them the cheapest way it can be read is + what keeps a debug build's checking proportional to what is checked. +*/ +static bool field_reads_back_as_document(Field *f) +{ + /* + The two channels read the way is_valid_json() reads them, and not by + calling it: what it does before answering is ask for this. + */ + if (!f->attests_is_valid_json()) + return true; + { + StringBuffer kept; + return json_value_reads_as_document(f->val_str(&kept)); + } +} + + +static bool field_reads_back_as_nice(Field *f) +{ + if (!f->attests_is_nice_json()) + return true; + { + StringBuffer kept; + return json_value_is_nice(f->val_str(&kept)); + } +} + + +static bool field_reads_back_no_deeper_than_claimed(Field *f) +{ + if (!f->attests_is_valid_json()) + return true; + { + StringBuffer kept; + return json_value_depth(f->val_str(&kept)) <= f->attested_json_depth(); + } +} + + +/* + Asked of each of the two fields an item has: the one it names, and - + wherever a temporary table has been built out of that one - the copy + in it, which is what the _result() answers are about. A fill site + nobody taught to ask its source announces itself in whichever of the + two it got wrong. +*/ +bool Item_field::reads_back_as_document() const +{ return field_reads_back_as_document(field); } + +bool Item_field::reads_back_as_nice() const +{ return field_reads_back_as_nice(field); } + +bool Item_field::reads_back_no_deeper_than_claimed() const +{ return field_reads_back_no_deeper_than_claimed(field); } + +bool Item_field::reads_back_as_document_result() const +{ return field_reads_back_as_document(result_field); } + +bool Item_field::reads_back_as_nice_result() const +{ return field_reads_back_as_nice(result_field); } + +bool Item_field::reads_back_no_deeper_than_claimed_result() const +{ return field_reads_back_no_deeper_than_claimed(result_field); } +#endif + + /** Save a field value in another field @@ -7076,6 +7298,12 @@ static int save_field_in_field(Field *from, bool *null_value, if (to == from) DBUG_RETURN(0); + /* + field_conv() confirms the destination against what the source + attests to - see there. A value that arrives off an ITEM goes through + Item::save_str_in_field instead, which asks the store the same + question this asks the source. + */ res= field_conv(to, from); DBUG_RETURN(res); } @@ -7117,6 +7345,16 @@ void Item_field::save_org_in_field(Field *to, DBUG_VOID_RETURN; } (*fast_field_copier_func)(to, field); + /* + One of the two routes that go around field_conv(), reaching what it + dispatches to without passing through it. It does write fields of + the server's own temporary tables, so the question belongs here; no + field carrying an answer is known to arrive, which is a fact about + the queries anybody has written rather than about the route, so it + is asked and not assumed. + */ + if (unlikely(to->is_valid_json_static())) + to->confirm_is_valid_json_static_from(field); } else save_field_in_field(field, &null_value, to, TRUE); @@ -7192,6 +7430,24 @@ int Item::save_str_in_field(Field *field, bool no_conversions) field->set_notnull(); int error= field->store(result->ptr(),result->length(),cs); + /* + A value that reaches a field carrying the standing answer off a + string ITEM comes through here, that being the one way one is put + into a field. A value that comes off another FIELD does not, and + is asked about at the three places save_field_in_field() names. + The answer is about characters, so it is kept only while the store + keeps them. + + How they are formatted is asked of the item rather than of the store, + and it is asked HERE rather than said at build time because it is + about the value just written and not about the item that wrote it. + How deep the value goes is asked here for the same reason and taken + the same way, an item that counted nothing answering with the + largest figure there is. + */ + if (unlikely(field->is_valid_json_static())) + field->confirm_is_valid_json_static(result->length(), is_nice_json(), + last_depth(), cs); str_value.set_buffer_if_not_allocated(0, 0, cs); return error; } @@ -8992,6 +9248,33 @@ String *Item_ref::val_str(String* tmp) } +/* + The two below are written for a caller that wants a document, reaching + the same side of the referenced item that val_str() does. Without + them Item::val_json() would send the request back through val_str(), + and an item that answers those two differently - a routine's variable + returns a view of itself from one and a copy from the other - would + have the difference thrown away by the reference in front of it. +*/ + +String *Item_ref::val_json(String* tmp) +{ + DBUG_ASSERT(fixed()); + tmp=(*ref)->val_json_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + +String *Item_ref::val_json_at_once(String* tmp) +{ + DBUG_ASSERT(fixed()); + tmp=(*ref)->val_json_at_once_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + bool Item_ref::is_null() { DBUG_ASSERT(fixed()); @@ -9140,6 +9423,24 @@ String *Item_direct_ref::val_str(String* tmp) } +/* The value side, val_str() above being of the value side here. */ + +String *Item_direct_ref::val_json(String* tmp) +{ + tmp=(*ref)->val_json(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + +String *Item_direct_ref::val_json_at_once(String* tmp) +{ + tmp=(*ref)->val_json_at_once(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + my_decimal *Item_direct_ref::val_decimal(my_decimal *decimal_value) { my_decimal *tmp= (*ref)->val_decimal(decimal_value); @@ -9459,21 +9760,21 @@ String *Item_cache_wrapper::val_str(String* str) DBUG_ENTER("Item_cache_wrapper::val_str"); if (!expr_cache) { - String *tmp= orig_item->val_str(str); + String *tmp= m_value_arm.read(orig_item, str); null_value= orig_item->null_value; DBUG_RETURN(tmp); } if ((cached_value= check_cache())) { - String *tmp= cached_value->val_str(str); + String *tmp= m_value_arm.read(cached_value, str); null_value= cached_value->null_value; DBUG_RETURN(tmp); } cache(); if ((null_value= expr_value->null_value)) DBUG_RETURN(NULL); - DBUG_RETURN(expr_value->val_str(str)); + DBUG_RETURN(m_value_arm.read(expr_value, str)); } @@ -9984,6 +10285,22 @@ String *Item_direct_view_ref::str_result(String* tmp) } +String *Item_direct_view_ref::val_json_result(String* tmp) +{ + tmp=(*ref)->val_json_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + +String *Item_direct_view_ref::val_json_at_once_result(String* tmp) +{ + tmp=(*ref)->val_json_at_once_result(tmp); + null_value=(*ref)->null_value; + return tmp; +} + + my_decimal *Item_direct_view_ref::val_decimal_result(my_decimal *val) { my_decimal *tmp= (*ref)->val_decimal_result(val); @@ -10441,6 +10758,8 @@ bool Item_trigger_field::set_value(THD *thd, sp_rcontext * /*ctx*/, Item **it) field->table->copy_blobs= copy_blobs_saved; field->set_has_explicit_value(); + if (field->table->has_own_json_valid_check) + field->set_is_valid_json(item, err_code); return err_code < 0; } @@ -11018,11 +11337,13 @@ Item *Item_cache_decimal::convert_to_basic_const_item(THD *thd) bool Item_cache_str::cache_value() { + bool kept= true; if (!example) { DBUG_ASSERT(value_cached == FALSE); return FALSE; } + m_marks.clear(); value_cached= TRUE; THD *thd= current_thd; const bool err= thd->is_error(); @@ -11042,12 +11363,24 @@ bool Item_cache_str::cache_value() (select c from t1 where a=t2.a) from t2; */ - value_buff.copy(*value); + kept= !value_buff.copy(*value); value= &value_buff; } else - value_buff.copy(); + kept= !value_buff.copy(); value_buff.mark_as_const(); + /* + The item was evaluated just above and nothing has evaluated it since, + so this is the one moment its answers are about the bytes being kept + here. They are asked of its result side because str_result() is what + was read. String::copy() takes the character set along with the + bytes, so the characters answered about are the characters kept - a + copy that could not get the room keeps nothing this can attest to. + */ + if (value && kept) + m_marks.set(value, example->is_valid_json_result(), + example->is_nice_json_result(), + example->last_depth_result()); return TRUE; } diff --git a/sql/item.h b/sql/item.h index e8ea7d205ae9b..e4432ecece8dd 100644 --- a/sql/item.h +++ b/sql/item.h @@ -27,6 +27,7 @@ #include "sql_const.h" /* RAND_TABLE_BIT, MAX_FIELD_NAME */ #include "field.h" /* Derivation */ #include "sql_type.h" +#include "sql_type_json.h" /* Type_handler_json_common */ #include "sql_time.h" #include "sql_schema.h" #include "mem_root_array.h" @@ -861,6 +862,121 @@ static inline item_with_t operator~(const item_with_t a) } +/* + What a function answers when it cannot say how many structures deep + the value it returned goes. It is a number no document reaches, + so a caller weighing it against the limit turns the value down + whatever it is being put inside, which is the answer that costs a + reading and nothing else. + + A caller with a bound of its own takes the smaller of the two before + adding anything to it, so this never enters a sum. +*/ +#define JSON_DEPTH_UNKNOWN UINT_MAX + + +/* + What a JSON function says about the value it has just returned: + is_valid, that the value is a document, and is_nice, that it is + formatted the way json_nice() writes one in its loose form. + + Both start out saying nothing, and a function says something only + where it hands a result back, so every other way out leaves them as + they were. Saying nothing is always safe - a caller that is told + nothing reads the value, which is what every caller does today. + + The two are kept together in one member rather than as two bools of + their own so that a class picking up one picks up the other, and so + that there is a single place to say what copying and setting them + mean, however many classes come to hold one. + + It lives here, and not with the JSON functions, because a class that + is not one of them can still carry a value through - see + Item_func_conv_charset - and going through this member is what puts + such a class under the debug check below. +*/ +class Json_result_marks +{ + bool m_valid; + bool m_nice; + /* + How deep the value goes, where the function that wrote it worked + that out on the way; JSON_DEPTH_UNKNOWN where it did not. Kept + with the other two because it is answered under the same rules and + over the same value, and because a class picking up one picks up + all three. + */ + uint m_depth; +public: + Json_result_marks() + : m_valid(false), m_nice(false), m_depth(JSON_DEPTH_UNKNOWN) {} + /* + A copy is a new item that has not been evaluated, so it has produced + no value and there is nothing for it to say. What the item it was + copied from can say is about bytes that item still owns and is free + to write over. + */ + Json_result_marks(const Json_result_marks &) + : m_valid(false), m_nice(false), m_depth(JSON_DEPTH_UNKNOWN) {} + bool valid() const { return m_valid; } + bool nice() const { return m_nice; } + uint depth() const { return m_depth; } + void clear() + { m_valid= m_nice= false; m_depth= JSON_DEPTH_UNKNOWN; } + /* + 'str' is the value 'valid', 'nice' and 'depth' are being set for. A + debug build reads it back and stops if it is not what they say it + is. Nothing else ever would: the marks appear in no result, so a + claim that was not true would go unnoticed until something acted on + it. + */ + void set(const String *str, bool valid, bool nice, + uint depth= JSON_DEPTH_UNKNOWN); +}; + + +/* + What an item that returns a value one of its arguments made needs in + order to pass on what that argument said about it. Such an item copies + the bytes out and adds nothing of its own to them, so the three + questions above are answered by putting them to the argument that made + the value. + + Which argument that was is written down while the value is being read. + It cannot be worked out afterwards: some of these items find their + argument by reading it, and the rest are told which one by a condition + that may hold a subquery or a stored program, so asking a second time + would run those a second time. +*/ +class Json_value_arm +{ + Item *m_arm; + /* + Whether the value is being read for a caller that asked for a + document. An argument that keeps a value of its own returns a view + of it from val_str() and a copy from val_json(), and an item that + passes values on has to pass that request on with them. + */ + bool m_read_as_json; +public: + Json_value_arm() : m_arm(NULL), m_read_as_json(false) {} + /* + A copy is a new item that has not been evaluated, so it has read no + argument and there is nothing for it to pass on. Json_result_marks + is not copied for the same reason. + */ + Json_value_arm(const Json_value_arm &) + : m_arm(NULL), m_read_as_json(false) {} + /* Read the value off the argument that is returning it. */ + String *read(Item *arm, String *str); + /* Have the reads above made with val_json() rather than val_str(). */ + void want_json(bool want) { m_read_as_json= want; } + bool valid() const; + bool nice() const; + uint depth() const; +}; + + class Item :public Value_source, public Type_all_attributes { @@ -1708,6 +1824,160 @@ class Item :public Value_source, String *val_str(String *str, String *converter, CHARSET_INFO *to); virtual String *val_json(String *str) { return val_str(str); } + + /* + The same value, for a caller that is finished with it before + anything else can run. + + val_json() may have to return bytes of the caller's own so that + they outlive whatever the caller does next. A caller that does + nothing next has no use for them, and this variant lets the item + say so cheaply. + + Only ask for it when BOTH hold: nothing else is worked out between + this call and the last read of what it returns, and no part of + what it returns reaches the caller's own caller. If either + fails, val_json() is what is wanted. Answering with val_json() is + always allowed, and is what an item that has no cheaper answer + does. + */ + virtual String *val_json_at_once(String *str) { return val_json(str); } + + /* + The same two of the result side, which is what a reference reads - + Item_ref::val_str() calls str_result(), so Item_ref::val_json() + calls these. Where an item has no result side of its own the two + sides are the same object and the defaults here say so; the five + that override str_result() attest to whatever their own + str_result() reads, exactly as they do for the marks below. + + Without them a reference would answer a request for a document by + falling back to Item::val_json(), which is val_str() - and an item + that hands out a view of somebody else's bytes from val_str() and a + copy from val_json() would be back to handing out the view. + */ + virtual String *val_json_result(String *str) { return val_json(str); } + virtual String *val_json_at_once_result(String *str) + { return val_json_at_once(str); } + + /* + Whether the value returned by the LAST evaluation of this item is + guaranteed to be a JSON document, written in the character set the + value says it is written in. + + The question is put to the item rather than to the value because a + value has nowhere to keep the answer: val_str() returns bytes, a + length and a character set, and that is all there is room for. So a + caller evaluates the item and asks it, in that order, and the answer + stands until the item is evaluated again. + + FALSE does not say the value is not a document. It says only that + nothing here attests to it, and a caller who needs to know must read + the value and find out - which is what every caller does today. + Answering TRUE for a value that is not guaranteed is the one answer + that must never be given; answering FALSE for one that is costs a + reading and nothing else. + */ + virtual bool is_valid_json() const { return false; } + + /* + Whether that same value is formatted the way json_nice() writes a + document in its LOOSE form, which is the formatting the JSON functions + return today. Same rules as above, and the same one-sided cost: + a document said not to be in that form is simply written out again. + */ + virtual bool is_nice_json() const { return false; } + + /* + How many structures that same value nests: 0 for a scalar, 1 for an + array or object holding nothing but scalars, and so on. Asked and + answered under the same rules as the two above, and read only + together with is_valid - a value that is not known to be a document + has no depth worth speaking of. + + An item works this out only where it was going to walk the value + anyway, so most of them say nothing and answer JSON_DEPTH_UNKNOWN. + That is the safe answer here as false is there: a caller that is + told nothing reads the value, which is what every caller did before + there was anything to ask. + + A number too small is the one answer that must never be given. The + caller adds it to the depth the value is being spliced at and holds + the total against the limit, so an item that under-counts lets a + document past that limit be composed and passed on as one inside + it. + */ + virtual uint last_depth() const { return JSON_DEPTH_UNKNOWN; } + + /* + Whether EVERY evaluation of this item returns a document or + nothing at all. A property of the class, so it can be asked before + anything has been evaluated - which is what a caller running at + create time needs, the two answers above having no meaning there. + */ + virtual bool is_valid_json_static() const { return false; } + + /* + The same three questions, asked of the value the RESULT side of this + item returns - str_result(), which is how an Item_ref reads it. + + For nearly every item the two sides are one function: + Item::str_result() IS val_str(), so the value asked about is the + same value and the answers are the same answers, which is what the + defaults here say. An item that keeps a result field of its own + returns a copy sitting in a record buffer instead, and what was + said about the value it made is not said about the copy. Those + items - the five that override str_result() - attest to whatever + their own str_result() reads, by putting the question to it. + + Which of the two sides a reference forwards is not a judgement it + makes. Item_ref::val_str() calls str_result(), so it asks these; + Item_direct_ref::val_str() calls val_str(), so it asks the three + above. Nothing is dropped for safety and nothing is guessed: the + marks describe the bytes because they are asked of whatever handed + the bytes over. + */ + virtual bool is_valid_json_result() const { return is_valid_json(); } + virtual bool is_nice_json_result() const { return is_nice_json(); } + virtual uint last_depth_result() const { return last_depth(); } + + /* + The argument of the wrapper that aggregating character sets puts + round an item whose value has to move to another one, and that + CONVERT(x USING cs) is written as, or NULL from every other item - + which is to say, this asks "are you that wrapper, and if so what + are you put round". + + It is here so that a caller that has to look through the wrapper to + reach the item underneath can be told, instead of working it out + from the object it was handed. is_json_type() is that caller: the + wrapper round a document is put there for the character set and + leaves a document, so a look that stopped at it would take the + documents moved between character sets for values that have to be + quoted. It is asked once per value written into a container and + once per argument sized, which between them is nearly every + argument the JSON functions are given. + + The two ways of working it out from the object are both worse. A + dynamic_cast that fails walks the base classes of what it was + handed and cannot stop early, absence being provable only by + exhausting the graph, and Item derives from two classes, so every + item that reaches it is walked as the multiple-inheritance case; + no is the answer for all but a few. A Functype of the wrapper's + own would be one comparison, but Functype is dispatched on by + engines outside the tree, several of which take UNKNOWN_FUNC to + mean "printed by name" - a wrapper that stopped answering it would + fall out of their dispatch, and what they build would lose the + conversion rather than refuse it. A virtual answers the one + question that is being asked and is read by nothing else. + + NULL is read as "not that wrapper, nothing to look through", never + as an error, and anything that comes to derive from the wrapper + inherits the answer - which is what the cast this replaces would + have said about it too. + */ + virtual Item *conv_charset_arg() const { return NULL; } + /* Return decimal representation of item with fixed point. @@ -2194,15 +2464,6 @@ class Item :public Value_source, /* This is to handle printing of default values */ virtual bool need_parentheses_in_default() { return false; } virtual void save_in_result_field(bool no_conversions) {} - /* - Data type format implied by the CHECK CONSTRAINT, - to be sent to the client in the result set metadata. - */ - virtual bool set_format_by_check_constraint(Send_field_extended_metadata *) - const - { - return false; - } /* set value of aggregate function in case of no rows for grouping were found */ @@ -2347,6 +2608,12 @@ class Item :public Value_source, virtual bool switch_to_nullable_fields_processor(void *arg) { return 0; } virtual bool find_function_processor (void *arg) { return 0; } + /* + Whether this is a JSON_VALID() call reading the column named by arg. + Used while a table is being defined, where the expression is not fixed + yet and a column can only be recognised by its name. + */ + virtual bool json_valid_of_column_processor (void *) { return 0; } /* Check if a partition function is allowed SYNOPSIS @@ -3201,12 +3468,45 @@ class Item_sp_variable :public Item_fixed_hybrid double val_real() override; longlong val_int() override; String *val_str(String *sp) override; + String *val_json(String *str) override; + String *val_json_at_once(String *str) override; my_decimal *val_decimal(my_decimal *decimal_value) override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; bool val_native(THD *thd, Native *to) override; bool is_null() override; public: + /* + What the variable this stands for is holding, asked of the field the + value is in - see TABLE::json_held_marks. + + Asked through this_item(), which finds the running frame the way + every other read of the variable does and never remembers what it + found. The same item is called upon twice over, from both sides: + reading the variable comes through here, and assigning it to another + one arrives at the store as that same field, dereferenced on the way + by THD::sp_fix_func_item(). So one answer serves both, and a value + moved from variable to variable carries what was said about it + without anything having to move it. + + A variable of a stored program is not typed as a document however it + was declared, so a value out of one is quoted into a document rather + than spliced into it, and these three answers change nothing about + that. What they are read for is the other thing attestation saves: + a function given this value as the document to work on need not + parse it again. + + The standing question is not answered here and cannot be. A + variable holds what the last assignment put in it and the next one + will put something else there, so nothing about the value in it now + holds for every evaluation of it. + */ + bool is_valid_json() const override + { return this_item()->is_valid_json(); } + bool is_nice_json() const override + { return this_item()->is_nice_json(); } + uint last_depth() const override + { return this_item()->last_depth(); } void make_send_field(THD *thd, Send_field *field) override; bool const_item() const override { return true; } Field *create_tmp_field_ex(MEM_ROOT *root, @@ -3948,6 +4248,147 @@ class Item_field :public Item_ident, return field->table->pos_in_table_list->outer_join; } bool check_index_dependence(void *arg) override; + /* + A column says nothing about itself: a row can hold bytes no check ever + saw, and nothing in it records where they came from. The one + exception is a field of a temporary table the server built for + itself, which was told at build time what would be written into it - + see TABLE::is_valid_json_static_set. Nobody can address such a table, + so nothing can put anything else in it. + + One bit answers both questions, and that is what it was given to + mean: it was said of every row the column will ever hold, so what is + true of this evaluation is true of all of them. Answering the + standing question is what lets a column of one such table be written + into a column of the next - a table built out of an earlier one is + built before either has a row, so what the earlier column says THEN + is an upper bound on what it says by the time the row arrives, and + the later column asks it again at every fill. See + Field::confirm_is_valid_json_static_from(). + + The other exception is a field a stored program keeps a variable in, + which was told what was put there by the assignment that put it - + see TABLE::json_held_marks. That is a different promise about a + different thing and it is asked here for one reason: an assignment + of one variable to another arrives at the store as a read of the + field behind the source, this item and no other, so a variable + attesting to its value at all means this item attesting to it. + + Only one of the two is ever there to be asked. A table cannot both + be built by the server for a query and be where a stored program + keeps its variables, so the two never disagree and nothing has to + decide which of them wins. + */ + bool is_valid_json() const override + { + DBUG_ASSERT(reads_back_as_document()); + return field->attests_is_valid_json(); + } + /* + The standing question, which only the first of the two answers. A + variable is written as many times as somebody assigns it, so what is + in it now says nothing about what will be in it when a column built + out of it comes to be filled. + */ + bool is_valid_json_static() const override + { return field->is_valid_json_static(); } + /* + Formatting, unlike the two above, is nothing the column was promised. + It was granted along with them and spent by whatever has been + written since, so what it answers is about the rows that are there - + which is what a reader with a row in front of it is asking about. + + There is no standing companion to this one and there could not be: + the standing question is asked while a table is being built out of an + earlier one, and what the earlier column will come to be written like + is not a thing anybody knows then. A column of the later table is + granted the formatting like any other and gets it from its source at + each fill - see Field::confirm_is_valid_json_static_from(). + */ + bool is_nice_json() const override + { + DBUG_ASSERT(reads_back_as_nice()); + return field->attests_is_nice_json(); + } + /* + And how deep it goes, which is what a caller splicing this value + into one of its own adds its own levels to. It is the deepest of + the rows written so far rather than the depth of this one, which is + more than this value needs and never less - and more is the + direction a depth is allowed to be wrong in, costing a reading that + could have been skipped rather than admitting a document nothing + can read back. + */ + uint last_depth() const override + { + DBUG_ASSERT(reads_back_no_deeper_than_claimed()); + /* + Where a variable is what is being read the figure is the depth of + the one value in it rather than the deepest of a column's rows, + and so is exact. + */ + return field->attested_json_depth(); + } + /* + A field has no document of its own to pass - val_json() here is + Item::val_json(), which is val_str() - so its result side is + str_result() and nothing more. The bytes it reads are a row's, and + a row outlives the working out of an argument, so there is nothing + for val_json_at_once() to be careful about, and the two are one + call. Item_default_value inherits both: str_result() is virtual, + so its own is the one reached and the value gets put in place first. + */ + String *val_json_result(String *str) override { return str_result(str); } + String *val_json_at_once_result(String *str) override + { return str_result(str); } + /* + str_result() reads result_field rather than field, so these read + result_field too. Which of the two an Item_field is asked for is + decided by whoever reads it: everything that goes through a value + side reaches the first, and an Item_ref reaches the second, the two + being the same object until a temporary table has been built out of + this one and the copy put in it - which is exactly the case where + they can disagree, a store into the copy being able to shorten or + reformat what the original holds. + */ + bool is_valid_json_result() const override + { + DBUG_ASSERT(reads_back_as_document_result()); + return result_field->attests_is_valid_json(); + } + bool is_nice_json_result() const override + { + DBUG_ASSERT(reads_back_as_nice_result()); + return result_field->attests_is_nice_json(); + } + uint last_depth_result() const override + { + DBUG_ASSERT(reads_back_no_deeper_than_claimed_result()); + return result_field->attested_json_depth(); + } +#ifdef DBUG_ASSERT_EXISTS + /* + Detectors rather than checks, and the only ones that do not care + which way the value arrived: whatever filled this field, if the + column attests to its values then the bytes in the row have to read + as a document, and if it attests to their formatting they have to be + formatted that way. A fill site nobody taught to ask its source + announces itself here, at the first read, rather than wherever + something later acts on the answer. + */ + bool reads_back_as_document() const; + bool reads_back_as_nice() const; + bool reads_back_no_deeper_than_claimed() const; + /* + And the same three over the other field, for the three answers that + are about it. A twin rather than a parameter because the assertions + read as the accessors do, each naming the field its own answer came + from. + */ + bool reads_back_as_document_result() const; + bool reads_back_as_nice_result() const; + bool reads_back_no_deeper_than_claimed_result() const; +#endif void set_refers_to_temp_table(); friend class Item_default_value; friend class Item_insert_value; @@ -5969,6 +6410,8 @@ class Item_ref :public Item_ident my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; String *val_str(String* tmp) override; + String *val_json(String* tmp) override; + String *val_json_at_once(String* tmp) override; bool val_native(THD *thd, Native *to) override; bool is_null() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; @@ -5981,6 +6424,45 @@ class Item_ref :public Item_ident my_decimal *val_decimal_result(my_decimal *) override; bool val_bool_result() override; bool is_null_result() override; + /* + val_str() here reads the referenced item through str_result(), so + what is asked of that item is what IT says about the value its + result side passes - which for most items is the value it made, + and for the few that keep a copy is the copy. The item is the one + that knows which; nothing is decided here. + */ + bool is_valid_json() const override + { return (*ref)->is_valid_json_result(); } + bool is_nice_json() const override + { return (*ref)->is_nice_json_result(); } + uint last_depth() const override + { return (*ref)->last_depth_result(); } + /* + And this item's own result side, which str_result() below reads out + of a result field where there is one and through val_str() where + there is not. Whether there is one is a fact about this instance + rather than about the class, which is why the condition is written + out here rather than answered once and for all somewhere. + */ + String *val_json_result(String* tmp) override + { return result_field ? str_result(tmp) : val_json(tmp); } + String *val_json_at_once_result(String* tmp) override + { return result_field ? str_result(tmp) : val_json_at_once(tmp); } + bool is_valid_json_result() const override + { + return result_field ? result_field->attests_is_valid_json() + : (*ref)->is_valid_json_result(); + } + bool is_nice_json_result() const override + { + return result_field ? result_field->attests_is_nice_json() + : (*ref)->is_nice_json_result(); + } + uint last_depth_result() const override + { + return result_field ? result_field->attested_json_depth() + : (*ref)->last_depth_result(); + } bool send(Protocol *prot, st_value *buffer) override; void make_send_field(THD *thd, Send_field *field) override; bool fix_fields(THD *, Item **) override; @@ -6196,6 +6678,8 @@ class Item_direct_ref :public Item_ref my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; String *val_str(String* tmp) override; + String *val_json(String* tmp) override; + String *val_json_at_once(String* tmp) override; bool val_native(THD *thd, Native *to) override; bool is_null() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; @@ -6203,6 +6687,20 @@ class Item_direct_ref :public Item_ref longlong val_time_packed(THD *) override; Ref_Type ref_type() override { return DIRECT_REF; } + /* + val_str() here returns what the referenced item returned, byte + for byte, so whatever could be said about that value can still be + said about this one. Item_ref cannot say the same: it reads through + str_result(), which for a field is the copy sitting in a record + buffer rather than the value the producing item made, and nothing + here knows what happened to it on the way. + */ + bool is_valid_json() const override { return (*ref)->is_valid_json(); } + bool is_nice_json() const override { return (*ref)->is_nice_json(); } + uint last_depth() const override { return (*ref)->last_depth(); } + bool is_valid_json_static() const override + { return (*ref)->is_valid_json_static(); } + /* Should be called if ref is changed */ inline void ref_changed() { @@ -6269,6 +6767,13 @@ class Item_cache_wrapper :public Item_result_field */ Item_cache *expr_value; + /* + What this item returns is what the expression returned or what the + cache kept of it, and it adds nothing of its own to either. See + Json_value_arm. + */ + Json_value_arm m_value_arm; + List parameters; Item *check_cache(); @@ -6292,6 +6797,30 @@ class Item_cache_wrapper :public Item_result_field double val_real() override; longlong val_int() override; String *val_str(String* tmp) override; + /* + Asked for a document, which is a request whatever returns the value + has to see - see Json_value_arm. val_str() below is what reads it. + */ + String *val_json(String* tmp) override + { + m_value_arm.want_json(true); + String *res= val_str(tmp); + m_value_arm.want_json(false); + return res; + } + bool is_valid_json() const override + { return !null_value && m_value_arm.valid(); } + bool is_nice_json() const override + { return !null_value && m_value_arm.nice(); } + uint last_depth() const override + { return null_value ? JSON_DEPTH_UNKNOWN : m_value_arm.depth(); } + /* + A property of the expression rather than of any value it has + produced. The cache keeps every character of what the expression + returns, so what holds of the expression holds of the cache. + */ + bool is_valid_json_static() const override + { return orig_item->is_valid_json_static(); } bool val_native(THD *thd, Native *to) override; my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; @@ -6401,8 +6930,7 @@ class Item_direct_view_ref :public Item_direct_ref bool check_null_ref() { - DBUG_ASSERT(null_ref_table); - if (null_ref_table != NO_NULL_TABLE && null_ref_table->null_row) + if (null_row_ref()) { null_value= 1; return TRUE; @@ -6410,6 +6938,16 @@ class Item_direct_view_ref :public Item_direct_ref return FALSE; } + /* + The same question without the answer being recorded, for callers that + only want to know and are not evaluating anything. + */ + bool null_row_ref() const + { + DBUG_ASSERT(null_ref_table); + return null_ref_table != NO_NULL_TABLE && null_ref_table->null_row; + } + public: Item_direct_view_ref(THD *thd, Name_resolution_context *context_arg, Item **item, @@ -6469,6 +7007,37 @@ class Item_direct_view_ref :public Item_direct_ref Item *in_subq_field_transformer_for_where(THD *thd, uchar *arg) override; Item *in_subq_field_transformer_for_having(THD *thd, uchar *arg) override; + /* + On a row the outer join filled in with NULLs the val_XXX below return + without evaluating the referenced item at all, so what that item can + say is about whatever row it was last asked about. Say nothing here + instead. The row that was just produced is the one being asked about, + which is why this reads null_row_ref() rather than remembering + anything: it is answering about the same row the val_XXX did. + */ + bool is_valid_json() const override + { return !null_row_ref() && Item_direct_ref::is_valid_json(); } + bool is_nice_json() const override + { return !null_row_ref() && Item_direct_ref::is_nice_json(); } + uint last_depth() const override + { + return null_row_ref() ? JSON_DEPTH_UNKNOWN + : Item_direct_ref::last_depth(); + } + /* + str_result() below passes on whatever the referenced item's result + side gave, and does not look at the row the way the val_XXX above + it do, so neither do these. + */ + String *val_json_result(String* tmp) override; + String *val_json_at_once_result(String* tmp) override; + bool is_valid_json_result() const override + { return (*ref)->is_valid_json_result(); } + bool is_nice_json_result() const override + { return (*ref)->is_nice_json_result(); } + uint last_depth_result() const override + { return (*ref)->last_depth_result(); } + void save_val(Field *to) override { if (check_null_ref()) @@ -6497,6 +7066,20 @@ class Item_direct_view_ref :public Item_direct_ref else return Item_direct_ref::val_str(tmp); } + String *val_json(String* tmp) override + { + if (check_null_ref()) + return NULL; + else + return Item_direct_ref::val_json(tmp); + } + String *val_json_at_once(String* tmp) override + { + if (check_null_ref()) + return NULL; + else + return Item_direct_ref::val_json_at_once(tmp); + } bool val_native(THD *thd, Native *to) override { if (check_null_ref()) @@ -6699,6 +7282,13 @@ class Item_ref_null_helper: public Item_ref double val_real() override; longlong val_int() override; String* val_str(String* s) override; + /* + Every val_XXX here notes a NULL down for the owner, so these have to + as well: they are how val_str() used to be reached, and a NULL that + goes unnoted is a subquery answering the wrong way. + */ + String* val_json(String* s) override; + String* val_json_at_once(String* s) override; my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; @@ -6804,12 +7394,30 @@ class Item_copy :public Item, Type_std_attributes::set(item); name= item->name; set_handler(item->type_handler()); -#ifndef DBUG_OFF + m_is_json= Type_handler_json_common::is_json_type_handler(type_handler()); +#ifdef DBUG_ASSERT_EXISTS copied_in= 0; #endif } -#ifndef DBUG_OFF + /* + Whether a value kept here is one a splice puts into a document rather + than quotes into one. The type settles that and the type is settled + here, so it is worked out once: the row path would otherwise ask two + virtuals per copied field per row to find out something that was true + before the query began. + */ + bool m_is_json; + + /* + Read by a DBUG_ASSERT and by nothing else, so it is compiled wherever + a DBUG_ASSERT is rather than wherever a debug build is. Those are not + the same set: DBUG_ASSERT_AS_PRINTF turns the assertion into a printed + complaint and leaves its expression standing, while setting DBUG_OFF, + so a guard written the other way round leaves that expression with + nothing to read and the build stops compiling. + */ +#ifdef DBUG_ASSERT_EXISTS bool copied_in; #endif @@ -6870,6 +7478,20 @@ class Item_copy :public Item, */ class Item_copy_string : public Item_copy { + /* + What the item this was made over said about the value kept here, + taken at the moment it was kept. + + The three questions are about the last value an item produced, and + this item's values are produced elsewhere and at another time: what + copy() puts away stands until the next copy(), while the item it + came from goes on evaluating. Asking that item when somebody asks + this one would pair an answer about a value it has since produced + with the bytes sitting here - which is why the answers are taken + once, beside the bytes they are about, exactly as + Item_func_conv_charset does over the value it freezes. + */ + Json_result_marks m_marks; public: Item_copy_string(THD *thd, Item *item_arg): Item_copy(thd, item_arg) {} @@ -6882,6 +7504,25 @@ class Item_copy_string : public Item_copy DBUG_ASSERT(copied_in); return get_date_from_string(thd, ltime, fuzzydate); } + /* + Asked next to val_str(), which returns the same bytes these are + about and which asserts the same thing about having been filled. + */ + bool is_valid_json() const override + { + DBUG_ASSERT(copied_in); + return m_marks.valid(); + } + bool is_nice_json() const override + { + DBUG_ASSERT(copied_in); + return m_marks.nice(); + } + uint last_depth() const override + { + DBUG_ASSERT(copied_in); + return m_marks.depth(); + } void copy() override; int save_in_field(Field *field, bool no_conversions) override; protected: @@ -6971,7 +7612,7 @@ class Item_copy_timestamp: public Item_copy null_value= tmp.is_null(); m_value= tmp.is_null() ? Timestamp_or_zero_datetime() : Timestamp_or_zero_datetime(tmp); -#ifndef DBUG_OFF +#ifdef DBUG_ASSERT_EXISTS copied_in=1; #endif } @@ -7176,7 +7817,14 @@ class Item_default_value : public Item_field longlong val_time_packed(THD *thd) override { return Item::val_time_packed(thd); } - /* Result variants */ + /* + Result variants. Each of these puts the default in place and then + does what Item_field does, and the marks that go with them need no + variant of their own: what they add is the putting in place, and by + the time anybody asks what a value was, the value has been made. + An accessor that called calculate() again would be making a second + one. + */ double val_result() override; longlong val_int_result() override; String *str_result(String* tmp) override; @@ -8050,7 +8698,18 @@ class Item_cache_str: public Item_cache char buffer[STRING_BUFFER_USUAL_SIZE]; String *value, value_buff; bool is_varbinary; - + /* + What the item this was cached from said about the value at the + moment it was cached, which is the moment the two are about the same + bytes. Asking that item later would pair an answer about whatever + it has gone on to produce with the copy kept here - the copy being + the point of this item, and being what val_str() below returns. + + Taken beside the copy exactly as Item_copy_string does over the value + it keeps. + */ + Json_result_marks m_marks; + public: Item_cache_str(THD *thd, const Item *item): Item_cache(thd, item->type_handler()), value(0), @@ -8070,6 +8729,25 @@ class Item_cache_str: public Item_cache int save_in_field(Field *field, bool no_conversions) override; bool cache_value() override; Item *convert_to_basic_const_item(THD *thd) override; + /* + Asked next to val_str(), which returns the bytes these are about. + A cache emptied by clear() or set_null() keeps no value at all, and + what was said about the one it used to keep is not said about that. + */ + bool is_valid_json() const override + { return !null_value && m_marks.valid(); } + bool is_nice_json() const override + { return !null_value && m_marks.nice(); } + uint last_depth() const override + { return null_value ? JSON_DEPTH_UNKNOWN : m_marks.depth(); } + /* + A property of the item being cached rather than of any value it has + produced, so it is answered whether or not anything has been cached + yet. The copy keeps every character of what that item returns, so + what holds of the one holds of the other. + */ + bool is_valid_json_static() const override + { return example && example->is_valid_json_static(); } protected: Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } @@ -8201,11 +8879,14 @@ class Item_type_holder: public Item, public Type_handler_hybrid_field_type { protected: const TYPELIB *enum_set_typelib; + bool m_is_valid_json_static; public: Item_type_holder(THD *thd, Item *item, const Type_handler *handler, - const Type_all_attributes *attr, bool maybe_null_arg) + const Type_all_attributes *attr, bool maybe_null_arg, + bool is_valid_json_static_arg) :Item(thd), Type_handler_hybrid_field_type(handler), - enum_set_typelib(attr->get_typelib()) + enum_set_typelib(attr->get_typelib()), + m_is_valid_json_static(is_valid_json_static_arg) { name= item->name; Type_std_attributes::set(*attr); @@ -8225,6 +8906,17 @@ class Item_type_holder: public Item, public Type_handler_hybrid_field_type } Type type() const override { return TYPE_HOLDER; } + /* + A holder stands in for a column of a UNION, which has one producer + per branch and a type they agree on. What is answered here is what + every one of those producers answers, asked of them all while the + types are being agreed - the branches being the one place they are + all in reach. Where they were not all reached, the caller hands + down a no rather than letting the ones it did reach speak for the + rest. + */ + bool is_valid_json_static() const override + { return m_is_valid_json_static; } const TYPELIB *get_typelib() const override { return enum_set_typelib; } /* When handling a query like this: diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index cfae1bae1da8c..dce85df48808e 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -2469,14 +2469,14 @@ String * Item_func_ifnull::str_op(String *str) { DBUG_ASSERT(fixed()); - String *res =args[0]->val_str(str); + String *res = read_arm(args[0], str); if (!args[0]->null_value) { null_value=0; res->set_charset(collation.collation); return res; } - res=args[1]->val_str(str); + res= read_arm(args[1], str); if ((null_value=args[1]->null_value)) return 0; res->set_charset(collation.collation); @@ -2975,7 +2975,7 @@ Item_func_nullif::str_op(String *str) null_value=1; return 0; } - res= args[2]->val_str(str); + res= read_arm(args[2], str); null_value= args[2]->null_value; return res; } @@ -3126,7 +3126,7 @@ String *Item_func_case::str_op(String *str) return 0; } null_value= 0; - if (!(res=item->val_str(str))) + if (!(res= read_arm(item, str))) null_value= 1; return res; } @@ -3516,7 +3516,7 @@ String *Item_func_coalesce::str_op(String *str) for (uint i=0 ; i < arg_count ; i++) { String *res; - if ((res=args[i]->val_str(str))) + if ((res= read_arm(args[i], str))) return res; } null_value=1; @@ -6761,21 +6761,6 @@ Item *Item_cond_and::neg_transformer(THD *thd) /* NOT(a AND b AND ...) -> */ } -bool -Item_cond_and::set_format_by_check_constraint( - Send_field_extended_metadata *to) const -{ - List_iterator_fast li(const_cast&>(list)); - Item *item; - while ((item= li++)) - { - if (item->set_format_by_check_constraint(to)) - return true; - } - return false; -} - - Item *Item_cond_or::neg_transformer(THD *thd) /* NOT(a OR b OR ...) -> */ /* NOT a AND NOT b AND ... */ { diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index a75b1eef510f9..25530bb66f3d6 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1218,6 +1218,9 @@ class Item_func_coalesce :public Item_func_case_expression return name; } table_map not_null_tables() const override { return 0; } + /* Any of the arguments can be the one whose value is returned. */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(0, arg_count); } protected: Item *shallow_copy(THD *thd) const override @@ -1305,6 +1308,9 @@ class Item_func_ifnull :public Item_func_case_abbreviation2 } table_map not_null_tables() const override { return 0; } + /* Either argument can be the one whose value is returned. */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(0, 2); } protected: Item *shallow_copy(THD *thd) const override @@ -1351,15 +1357,30 @@ class Item_func_case_abbreviation2_switch: public Item_func_case_abbreviation2 { return val_decimal_from_item(find_item(), decimal_value); } + /* + Item::val_str_from_item(), reading the argument through read_arm() so + that which argument it was is written down and a caller that asked + for a document has its request reach it. + */ String *str_op(String *str) override { - return val_str_from_item(find_item(), str); + DBUG_ASSERT(fixed()); + Item *arm= find_item(); + String *res= read_arm(arm, str); + if (res) + res->set_charset(collation.collation); + if ((null_value= arm->null_value)) + res= NULL; + return res; } bool native_op(THD *thd, Native *to) override { return val_native_with_conversion_from_item(thd, find_item(), to, type_handler()); } + /* args[0] is the switch; the value comes from one of the other two. */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(1, 2); } }; @@ -1489,6 +1510,13 @@ class Item_func_nullif :public Item_func_case_expression void update_used_tables() override; table_map not_null_tables() const override { return 0; } bool is_null() override; + /* + The value is always the right "a", args[2] - see the comment on the + constructor. The left "a" is only ever compared, and args[1] is only + ever compared against. + */ + bool is_valid_json_static() const override + { return args_are_valid_json_static(2, 1); } Item* propagate_equal_fields(THD *thd, const Context &ctx, COND_EQUAL *cond) override { @@ -2392,6 +2420,7 @@ class Item_func_case :public Item_func_case_expression DTCollation cmp_collation; bool aggregate_then_and_else_arguments(THD *thd, uint count); virtual Item **else_expr_addr() const= 0; + virtual uint when_count() const= 0; virtual Item *find_item()= 0; inline void print_when_then_arguments(String *str, enum_query_type query_type, @@ -2419,6 +2448,22 @@ class Item_func_case :public Item_func_case_expression } CHARSET_INFO *compare_collation() const { return cmp_collation.collation; } bool need_parentheses_in_default() override { return true; } + /* + The value comes from one of the THEN arguments or from the ELSE, and + reorder_args() has put exactly those last: the optional CASE + expression comes first, then the WHEN expressions, then the THEN + expressions and the ELSE. So the arguments that can be returned are + the last one per WHEN plus the ELSE where there is one. + + Asking the WHEN expressions too would answer no for every CASE ever + written - what they return is a condition, and this item never + returns it. + */ + bool is_valid_json_static() const override + { + uint count= when_count() + MY_TEST(else_expr_addr()); + return args_are_valid_json_static(arg_count - count, count); + } }; @@ -2431,7 +2476,7 @@ class Item_func_case :public Item_func_case_expression */ class Item_func_case_searched: public Item_func_case { - uint when_count() const { return arg_count / 2; } + uint when_count() const override { return arg_count / 2; } bool with_else() const { return arg_count % 2; } Item **else_expr_addr() const override { return with_else() ? &args[arg_count - 1] : 0; } @@ -2476,7 +2521,7 @@ class Item_func_case_simple: public Item_func_case, { protected: uint m_found_types; - uint when_count() const { return (arg_count - 1) / 2; } + uint when_count() const override { return (arg_count - 1) / 2; } bool with_else() const { return arg_count % 2 == 0; } Item **else_expr_addr() const override { return with_else() ? &args[arg_count - 1] : 0; } @@ -3695,8 +3740,6 @@ class Item_cond_and final :public Item_cond COND *build_equal_items(THD *thd, COND_EQUAL *inherited, bool link_item_fields, COND_EQUAL **cond_equal_ref) override; - bool set_format_by_check_constraint(Send_field_extended_metadata *to) const - override; void add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) override; diff --git a/sql/item_func.h b/sql/item_func.h index c668bbfe76826..37a4890c3fc3b 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -1092,6 +1092,31 @@ class Item_func_hybrid_field_type: public Item_hybrid_func */ class Item_func_case_expression: public Item_func_hybrid_field_type { +protected: + /* + What this item returns is what one of its arguments returned, byte + for byte, so the questions below are put to that argument. Every + path that returns a string value reads it through here. + */ + Json_value_arm m_value_arm; + String *read_arm(Item *arm, String *str) + { return m_value_arm.read(arm, str); } + + /* + Whether every one of 'count' arguments starting at 'first' returns a + document or nothing at all each time it is evaluated. One of them + will be the one this item returns and nothing here knows which, so + what can be said before any of them is evaluated is only what they + all say. + */ + bool args_are_valid_json_static(uint first, uint count) const + { + for (uint i= first; i < first + count; i++) + if (!args[i]->is_valid_json_static()) + return false; + return true; + } + public: Item_func_case_expression(THD *thd) :Item_func_hybrid_field_type(thd) @@ -1109,6 +1134,31 @@ class Item_func_case_expression: public Item_func_hybrid_field_type Item_func_hybrid_field_type(thd, list) { } bool find_not_null_fields(table_map allowed) override { return false; } + + /* + Asked for a document rather than for a string, which is a request the + argument has to see - see Json_value_arm. val_str() is what reaches + str_op(), and str_op() is what reads the argument. + */ + String *val_json(String *str) override + { + m_value_arm.want_json(true); + String *res= val_str(str); + m_value_arm.want_json(false); + return res; + } + + /* + Nothing is said where the evaluation produced no value: NULL is not a + document, and the argument written down would then be whichever one + was read last rather than one this item returned. + */ + bool is_valid_json() const override + { return !null_value && m_value_arm.valid(); } + bool is_nice_json() const override + { return !null_value && m_value_arm.nice(); } + uint last_depth() const override + { return null_value ? JSON_DEPTH_UNKNOWN : m_value_arm.depth(); } }; @@ -3602,6 +3652,29 @@ class Item_func_set_user_var :public Item_func_user_var String *str_result(String *str) override; my_decimal *val_decimal_result(my_decimal *) override; bool is_null_result() override; + /* + Reading the document out of str_result() is what happens today and + is what has to keep happening: the assignment it performs is this + item's whole point, and a document must not be fetched by any road + that performs it a second time or not at all. A user variable is + read out by copying, so what comes back is the caller's own bytes + and val_json_at_once() has nothing to save. + */ + String *val_json_result(String *str) override { return str_result(str); } + String *val_json_at_once_result(String *str) override + { return str_result(str); } + /* + str_result() above is the one of the five that does not merely read + somewhere else: it assigns the variable and then returns what is + in it. So these cannot be written the way the other four are, by + putting the question to whatever that reads - putting a question to + it would perform the assignment. They say nothing instead, which + is what there is to say: nothing keeps marks for a user variable, + so there would be nothing to pass on even if asking were free. + */ + bool is_valid_json_result() const override { return false; } + bool is_nice_json_result() const override { return false; } + uint last_depth_result() const override { return JSON_DEPTH_UNKNOWN; } bool update_hash(void *ptr, size_t length, const Type_handler *th, CHARSET_INFO *cs); bool send(Protocol *protocol, st_value *buffer) override; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 6acf31a8f6753..2b859f651e8c3 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -31,6 +31,106 @@ int dbug_json_check_min_stack_requirement() extern void pause_execution(THD *thd, double timeout); + +#ifndef DBUG_OFF +/* + Counts a reading of a JSON value. + + What these functions are being taught is not to read a value they have + already read, and a value read one time fewer returns exactly what + it gave back before - so there is nothing in any answer that says the + saving happened. The count is the only place it shows, which is why + it exists at all. + + A scanner started while there is no session to charge is left + uncounted rather than refused. + + Written the way json_lib calls it, that being what its address is + handed to. +*/ +extern "C" void json_count_scan() +{ + THD *thd= current_thd; + + if (thd) + status_var_increment(thd->status_var.json_scans); +} + + +/* + Asks json_lib to say when a value is read. + + The count is taken where the reading begins rather than where it is + asked for. Every way of asking - reading a path, asking whether a + value is a document, normalizing one - ends up at json_scan_start(), + so a count taken there is a count of readings. A count taken at the + asking would have to name every way of asking, and one such list has + already been wrong: normalizing was reading twice and being counted + none, because it does its reading from inside json_lib where a list + kept here cannot see it. + + What is counted is therefore every reading this session does, and not + only the ones these functions ask for by name. +*/ +static struct Json_scan_count_hook +{ + Json_scan_count_hook() { json_scan_start_hook= json_count_scan; } +} json_scan_count_hook; + + +Json_scans_unbilled::Json_scans_unbilled(THD *thd) + :m_thd(thd), m_scans(thd->status_var.json_scans) +{ } + + +Json_scans_unbilled::~Json_scans_unbilled() +{ + m_thd->status_var.json_scans= m_scans; +} +#endif + + +#ifdef DBUG_ASSERT_EXISTS +/* + Takes a reading back off the count. + + A value is read back here to check what was claimed about it, and that + reading exists only because the assertion checking the claim does. + Counting it would put work no released server does into a number kept + to watch the work it does, and the number would then move for reasons + that have nothing to do with anybody's query. + + Where the count is not kept there is nothing to take anything off: + Json_scans belongs to a debug build, while the assertions that make + these readings are compiled by one build more than that. +*/ +static inline void json_uncount_scan() +{ +#ifndef DBUG_OFF + THD *thd= current_thd; + + if (thd) + status_var_decrement(thd->status_var.json_scans); +#endif +} + + +/* + The way to the scanner for a reading that is not the server's work: + counted like any other and then taken back off again, so that the one + place the counting happens stays the only place it happens. +*/ +static inline int json_scan_start_unbilled(json_engine_t *je, + CHARSET_INFO *i_cs, + const uchar *str, const uchar *end) +{ + int rc= json_scan_start(je, i_cs, str, end); + + json_uncount_scan(); + return rc; +} +#endif + /* Allocating memory and *also* using it (reading and writing from it) because some build instructions cause @@ -94,6 +194,114 @@ append_simple(String *s, const uchar *a, size_t a_len) } +static void report_bad_chr_note(const char *fname, int n_arg); + + +/* + Says whether a key can go between a pair of quotes without reaching + past them. A quote ends the string where it stands, a backslash begins + an escape that may swallow the quote meant to end it, and a character + below a space is not allowed inside a string unescaped at all. A key + holding none of the three is just so many characters of a name. + + Nothing is done about a key that holds one. Whether the document that + comes of splicing it can still be read is a property of the WHOLE + composition and not of the key: a key of `a":1,"b` closes one member + and opens another, and what comes out is a document that has always + been given back. Refusing it here would take that answer away. + + So the one thing this decides is whether the answer can be given + without being read back. A key it turns down is spliced exactly as it + always was, and the reading back settles it exactly as it always did. + + The key is read a character at a time rather than a byte at a time, + because a byte is not a character in every character set: in ucs2 the + letter 'b' is written 00 62, and a reading that went by bytes would + take the 00 for a control character and turn down every key there is. + A key holding something that is not a character of its own character + set at all is turned down, nothing being known about what it holds. +*/ +static bool json_key_span_is_inert(CHARSET_INFO *cs, const char *key, + size_t key_len) +{ + const uchar *p= (const uchar *) key, *end= p + key_len; + + while (p < end) + { + my_wc_t wc; + int c_len= cs->mb_wc(&wc, p, end); + + if (c_len <= 0 || wc == '"' || wc == '\\' || wc < 0x20) + return false; + + p+= c_len; + } + + return true; +} + + +/* + Appends the key of a path step to the document being built. + + A path is written in its own character set, which is not necessarily + the one of the document, so the key may have to be converted before it + can be spliced in. Only a conversion that lost nothing is taken: a + key holding a character the document's character set has no room for + goes in as it arrived, which is what was done with every key before + any key was converted at all, and a note says so. + + Whether the document that results can still be read is then decided + by the document's character set, exactly as it was decided before: + one that admits every byte keeps the document, and one that does not + refuses it on the way back out. Deciding it here instead would take + away documents that have always been given. + + Nothing else is done to the key: one holding a character that is not + allowed unescaped inside a JSON string stays invalid, and is caught + where it was caught before. + + 'inert' comes back saying whether the key went in as so many characters + of a name and nothing more, so that a caller which would rather not + read its answer back can tell when it has to. +*/ +static bool __attribute__((warn_unused_result)) +append_json_path_key(String *s, const json_path_step_t *step, + CHARSET_INFO *path_cs, String *tmp_key, + const char *fname, int n_arg, bool *inert) +{ + size_t key_len= (size_t) (step->key_end - step->key); + uint errors; + + if (!my_charset_same(path_cs, s->charset())) + { + if (tmp_key->copy((const char *) step->key, key_len, path_cs, + s->charset(), &errors) || + DBUG_IF("json_path_key_out_of_memory")) + return true; /* Out of memory. */ + + if (!errors) + { + *inert= json_key_span_is_inert(s->charset(), tmp_key->ptr(), + tmp_key->length()); + return append_simple(s, tmp_key->ptr(), tmp_key->length()); + } + + report_bad_chr_note(fname, n_arg); + /* + The key goes in written the way the path was, which is not the way + the document is. What those bytes come to mean once they are being + read as the document cannot be told from the key alone. + */ + *inert= false; + } + else + *inert= json_key_span_is_inert(path_cs, (const char *) step->key, key_len); + + return append_simple(s, step->key, key_len); +} + + /* Appends JSON string to the String object taking charsets in consideration. @@ -135,7 +343,7 @@ bool st_append_json(String *s, Appends arbitrary String to the JSON string taking charsets in consideration. */ -int st_append_escaped(String *s, const String *a) +json_append_result st_append_escaped(String *s, const String *a) { /* In the worst case one character from the 'a' string @@ -143,17 +351,19 @@ int st_append_escaped(String *s, const String *a) */ int str_len= a->length() * 12 * s->charset()->mbmaxlen / a->charset()->mbminlen; - if (!s->reserve(str_len, 1024) && - (str_len= - json_escape(a->charset(), (uchar *) a->ptr(), (uchar *)a->end(), - s->charset(), - (uchar *) s->end(), (uchar *)s->end() + str_len)) > 0) - { - s->length(s->length() + str_len); - return 0; - } - - return a->length(); + if (s->reserve(str_len, 1024) || + DBUG_IF("json_escape_reserve_out_of_memory")) + return JSON_APPEND_OOM; + + str_len= json_escape(a->charset(), (uchar *) a->ptr(), (uchar *)a->end(), + s->charset(), + (uchar *) s->end(), (uchar *)s->end() + str_len); + if (str_len < 0) + return str_len == JSON_ERROR_ILLEGAL_SYMBOL ? + JSON_APPEND_BAD_CHR : JSON_APPEND_OOM; + + s->length(s->length() + str_len); + return JSON_APPEND_OK; } @@ -303,11 +513,180 @@ int json_path_compare(const json_path_t *a, const json_path_t *b, } +/* + How a document is punctuated in the loose form - this much after a + comma, and this much after a key. + + Named here because two different pieces of code have to agree on it. + json_nice() writes it when it reads a document back, and the functions + that BUILD a document write it themselves and never read anything + back; if those two ever drifted apart, a document said to be in the + loose form would not be in it. The compact and detailed forms are + shorter and are taken as the first characters of the same two strings, + which is what the lengths below select. +*/ +static const char json_loose_comma[]= ", "; +static const char json_loose_colon[]= "\": "; + + +/* + Whether a document can be written in this character set at all. + + A document is punctuated with characters that most character sets + encode the way ASCII does, and a few do not: swe7 puts national letters + where the brackets, the braces and the backslash belong, so the bytes + that make an array anywhere else make a word there. A function that + writes an array out in such a character set has written something that + is not a document and cannot be read as one - which is what the server + has always done with it, and is not for this to change. What it is + for is to keep that result from being taken for a document later. + + Only the functions that write their own punctuation have to ask. One + that reads its whole result back afterwards reads it in the character + set it is written in, so a document it accepts is one that reads + there, and asking would tell it nothing new - which is why the seven + that always read back never asked, and why they ask now that they read + back only what their arguments left is_valid or is_nice false for. + + A set that cannot encode the punctuation can still hold a document, + but only a scalar one: writing a container in it would take the + brackets it cannot encode. So a function that has to write a + container to give its answer cannot give one there at all, and a + function given a document in such a set was given a scalar. + + What decides it is how narrow a character is, not whether the set is + ASCII-compatible. MY_CS_NONASCII marks two unrelated things: sets that + put other characters at the ASCII code points, and sets that are simply + too wide to hold ASCII a byte at a time. Only the first is a problem. + String::append(char) converts for a set whose characters are never one + byte, so ucs2 gets a real bracket, encoded as 005B; where a character + can be a single byte the byte goes in as it stands, and in swe7 that + byte is a letter. +*/ +static inline bool is_json_compatible_charset(CHARSET_INFO *cs) +{ + return cs->mbminlen > 1 || !(cs->state & MY_CS_NONASCII); +} + + +/* + Whether a container just composed has to be refused for the character + set it was composed in. + + A container is written with punctuation round it, and a set that does + not encode that punctuation as itself holds no container: what was + composed reads as something else, or as nothing. A released server + returned those bytes. The functions that read their whole answer back + have always returned NULL there instead, the reading being what fails, + and the ones that compose their answer say the same thing here, in the + same words and at the same position. + + Which is why it is read rather than reckoned. The question above is + not the question this is: a set stops being JSON-compatible as soon as + it puts ANY character somewhere other than ASCII does, and a set can do + that and still write brackets - sjis moves the backslash and leaves the + brackets where they were, so what it composes IS a document and is + returned. A compatible set is not read at all, so the reading is + confined to the sets that might have got it wrong. +*/ +static bool json_container_charset_refused(String *str, const char *fname) +{ + json_engine_t je; + + if (is_json_compatible_charset(str->charset())) + return false; + + json_scan_start(&je, str->charset(), (const uchar *) str->ptr(), + (const uchar *) str->ptr() + str->length()); + while (json_scan_next(&je) == 0) + {} + if (!je.s.error) + return false; + + report_json_error_ex(str->ptr(), &je, fname, 0, + Sql_condition::WARN_LEVEL_WARN); + return true; +} + + +/* + Whether the document argument leaves a function free to return what + it composes, instead of reading the whole answer again at the end to + find out what it is. + + Three things at once, and each of them is about the argument rather + than about anything composed from it. The value has to read as a + document, which is what the argument is asked. It has to be written + the way this writes, because what is composed keeps the argument's own + spacing between the pieces the function writes itself. And the set it + is written in has to be able to encode the punctuation that gets added + - nothing composed in a set that cannot is a document, however sound + the pieces were. + + Asked once, before anything is composed, because what is written early + is written before the later arguments have been looked at, so the + answer cannot wait for them. A later argument that turns out not to + be attested to does not come back here; it clears the splice marks + instead, and the answer goes through the reading back after all. +*/ + +static inline bool document_arg_composes_final(const Item *arg, + const String *js) +{ + return is_json_compatible_charset(js->charset()) && + arg->is_valid_json() && arg->is_nice_json(); +} + + +/* + Steps over the space standing at 'str', if one is standing there, and + returns where the next character begins. + + Composing writes some of its punctuation and copies the rest out of + the document it is working from, and where the two meet one side has + to give up the space between them. Which bytes that is depends on the + set the document is written in: a space is one byte in utf8, two in + ucs2 (0020), four in utf32 - and in the wide ones the FIRST of those + bytes is a zero. So a byte compared against ' ' answers no in exactly + the sets that can encode a document but do not write one character in + one byte, and the space is left where it stood, doubled. + + Reading one character and asking what it is answers the question the + way it was meant to be asked. +*/ +static const uchar *json_skip_space(CHARSET_INFO *cs, + const uchar *str, const uchar *end) +{ + my_wc_t wc; + int len; + + if (str >= end) + return str; + + len= cs->mb_wc(&wc, str, end); + return (len > 0 && wc == (my_wc_t) ' ') ? str + len : str; +} + + +/* Written out below, and named here because the loose form goes to it. */ +static int json_walk_nice_value(json_engine_t *je, String *to, + int &max_level); + + +/* + 'deepest', where a caller asks for it, comes back holding how many + structures the deepest part of what was written nests inside: 0 for a + scalar, 1 for an array of scalars, and so on. The count is free here - + the walk keeps it anyway, to know how far to indent - and it is the + only place a whole document is measured without being walked twice. +*/ static int json_nice(json_engine_t *je, String *nice_js, - Item_func_json_format::formats mode, int tab_size=4) + Item_func_json_format::formats mode, + uint *deepest= NULL, int tab_size=4) { int depth= 0; - static const char *comma= ", ", *colon= "\": "; + int reached= 0; + static const char *comma= json_loose_comma, *colon= json_loose_colon; uint comma_len, colon_len; int first_value= 1; int value_size = 0; @@ -320,26 +699,61 @@ static int json_nice(json_engine_t *je, String *nice_js, if (nice_js->alloc(je->s.str_end - je->s.c_str + 32)) goto error; - + DBUG_ASSERT(mode != Item_func_json_format::DETAILED || (tab_size >= 0 && tab_size <= TAB_SIZE_LIMIT)); if (mode == Item_func_json_format::LOOSE) { - comma_len= 2; - colon_len= 3; - } - else if (mode == Item_func_json_format::DETAILED) - { - comma_len= 1; - colon_len= 3; - } - else - { - comma_len= 1; - colon_len= 2; + /* + The loose form is written by the walk, which is this loop with the + other two modes taken out of it and stopped where the value ends + instead of where the document does. Written out twice they would + have to go on agreeing byte for byte, and what says whether they + still do is a check that measures a value by writing it through + HERE - so the day the two drifted apart, the drift is what would + be approved. + + The walk wants the value's head read, and returns where the + value ended. A document is a value with nothing after it, so what + is left is read through to find out whether anything is. + */ + if (json_read_value(je) || json_walk_nice_value(je, nice_js, reached)) + { + /* + Nothing was read wrong, so what went wrong was the writing: + there is no room for the answer and no answer to give. + */ + if (!je->s.error) + return 1; + } + else + { + while (json_scan_next(je) == 0) + {} + } + goto done; } + /* + The buffer above is a guess, and the indented form can outgrow it: + it spends a line and an indent where the compact form spends + nothing, so an input with more than sixteen separators in it needs + more room than was asked for. Everything written below therefore + may have to grow the buffer, and this is where a test arranges for + that growing to fail. Disarmed at both ways out, so that only the + writing done here is affected. + */ + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("+d,simulate_realloc_out_of_memory");); + + /* + What is left is the compact form and the indented one, which put the + same comma between values and differ over the space after a colon. + */ + comma_len= 1; + colon_len= (mode == Item_func_json_format::DETAILED) ? 3 : 2; + do { curr_state= je->state; @@ -358,25 +772,25 @@ static int json_nice(json_engine_t *je, String *nice_js, if (unlikely(je->s.error)) goto error; - if (!first_value) - nice_js->append(comma, comma_len); + if (!first_value && nice_js->append(comma, comma_len)) + goto error; if (mode == Item_func_json_format::DETAILED && append_tab(nice_js, depth, tab_size)) goto error; - nice_js->append('"'); - if (append_simple(nice_js, key_start, key_end - key_start)) + if (nice_js->append('"') || + append_simple(nice_js, key_start, key_end - key_start) || + nice_js->append(colon, colon_len)) goto error; - nice_js->append(colon, colon_len); } /* now we have key value to handle, so no 'break'. */ DBUG_ASSERT(je->state == JST_VALUE); goto handle_value; case JST_VALUE: - if (!first_value) - nice_js->append(comma, comma_len); + if (!first_value && nice_js->append(comma, comma_len)) + goto error; if (mode == Item_func_json_format::DETAILED && depth > 0 && @@ -392,8 +806,9 @@ static int json_nice(json_engine_t *je, String *nice_js, je->value_end - je->value_begin)) goto error; - curr_str.copy((const char *)je->value_begin, - je->value_end - je->value_begin, je->s.cs); + if (curr_str.copy((const char *)je->value_begin, + je->value_end - je->value_begin, je->s.cs)) + goto error; value_len= je->value_end - je->value_begin; first_value= 0; if (value_size != -1) @@ -405,10 +820,14 @@ static int json_nice(json_engine_t *je, String *nice_js, depth > 0 && !(curr_state != JST_KEY) && append_tab(nice_js, depth, tab_size)) goto error; - nice_js->append((je->value_type == JSON_VALUE_OBJECT) ? "{" : "[", 1); + if (nice_js->append((je->value_type == JSON_VALUE_OBJECT) ? + "{" : "[", 1)) + goto error; first_value= 1; value_size= (je->value_type == JSON_VALUE_OBJECT) ? -1: 0; depth++; + if (depth > reached) + reached= depth; } break; @@ -426,10 +845,12 @@ static int json_nice(json_engine_t *je, String *nice_js, nice_js->length(nice_js->length() - value_len); for (auto i = 0; i < (depth + 1) * tab_size + 1; i++) nice_js->chop(); - nice_js->append(curr_str); + if (nice_js->append(curr_str)) + goto error; } - - nice_js->append((je->state == JST_OBJ_END) ? "}": "]", 1); + + if (nice_js->append((je->state == JST_OBJ_END) ? "}": "]", 1)) + goto error; first_value= 0; value_size= -1; break; @@ -439,10 +860,424 @@ static int json_nice(json_engine_t *je, String *nice_js, }; } while (json_scan_next(je) == 0); - return je->s.error || *je->killed_ptr; + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("-d,simulate_realloc_out_of_memory");); + +done: + /* + Said only where the walk finished, a count off a walk that stopped + early being about the part of the document it got through rather + than about the document. + */ + if (deepest && !(je->s.error || *je->killed_ptr)) + *deepest= (uint) reached; + return je->s.error || *je->killed_ptr; + +error: + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("-d,simulate_realloc_out_of_memory");); + return 1; +} + + +/* + The room to write in, made to run out for as long as one of these is + standing. + + json_nice() arms the failure of its own appends around its own + writing, so that a test can see what happens when one of them fails. + The writing has moved out of it for the functions that no longer read + their result back, and the way of failing it had to move with it - + otherwise the checking of these appends would stop being tested by the + thing that was testing it, without anybody noticing that it did. + + Disarmed however the writing is left, so that only the writing done + while this stands is affected. +*/ +class Json_room_made_to_run_out +{ +public: + Json_room_made_to_run_out() + { + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("+d,simulate_realloc_out_of_memory");); + } + ~Json_room_made_to_run_out() + { + DBUG_EXECUTE_IF("json_nice_append_out_of_memory", + DBUG_SET("-d,simulate_realloc_out_of_memory");); + } +}; + + +/* + Writes the value the scanner is sitting on to the end of 'to', in the + loose form, walking the scanner over the value as it goes. + + This is json_nice() taken apart and put back together to be usable + during a reading instead of after one. json_nice() empties its output + and reads to the end of the document, which suits a function that has + finished and wants its whole result formatted again; it is no use to one + that is partway through a document and wants THIS value written into + what it has written so far. The formatting itself is the same, and has + to be: the two write the same punctuation from the same constants, so + a value put out by either is put out identically. + + The scanner must have read the value's head already - which is where + json_get_path_next() leaves it - and is left on the value's last + token, so the caller can carry on from there exactly as if it had + walked over the value itself. + + A scalar has no punctuation of its own and so is already written the + loose way wherever it came from; it is copied across as it stands, + which is what json_nice() does with one too. + + 'max_level' comes back holding the deepest the walk went, counted the + way a plain reading of the same value counts it - which is what a + caller that has to say how deep its answer ends up would otherwise + have to take a second reading to find out. It is raised and never + lowered, so a caller starting one walk where the last left off gets + the deepest of them all. + + Every writing of a value goes through here, which is what makes this + the one place the failure of these appends has to be arranged. + + Returns non-zero on a write that failed or a value that did not end. +*/ +static int json_walk_nice_value(json_engine_t *je, String *to, + int &max_level) +{ + Json_room_made_to_run_out room; + int depth; + int first_value= 1; + + if (je->stack_p > max_level) + max_level= je->stack_p; + + if (json_value_scalar(je)) + return append_simple(to, je->value_begin, + je->value_end - je->value_begin); + + if (to->append((je->value_type == JSON_VALUE_OBJECT) ? "{" : "[", 1)) + return 1; + depth= 1; + + while (json_scan_next(je) == 0) + { + if (je->stack_p > max_level) + max_level= je->stack_p; + + /* + The step above is what a killed query is stopped by here, and it + is the same step, taken the same number of times, that the reading + back used to be stopped by - so nothing is asked of this loop that + the reading it replaces was not already asked. + + A kill arriving before the writing begins is stopped by whichever + reading got here, which is the only kill the tests could reach + until now. This arms one that arrives partway through, so that + what stops it is the step above and nothing else. + */ + DBUG_EXECUTE_IF("json_kill_while_emitting", + { current_thd->set_killed(KILL_QUERY); }); + + switch (je->state) + { + case JST_KEY: + { + const uchar *key_start= je->s.c_str; + const uchar *key_end; + + do + { + key_end= je->s.c_str; + } while (json_read_keyname_chr(je) == 0); + + if (unlikely(je->s.error)) + return 1; + + if (!first_value && to->append(json_loose_comma, 2)) + return 1; + + if (to->append('"') || + append_simple(to, key_start, key_end - key_start) || + to->append(json_loose_colon, 3)) + return 1; + } + /* The key's value comes next, so there is no break here. */ + DBUG_ASSERT(je->state == JST_VALUE); + goto handle_value; + + case JST_VALUE: + if (!first_value && to->append(json_loose_comma, 2)) + return 1; + +handle_value: + if (json_read_value(je)) + return 1; + + if (je->stack_p > max_level) + max_level= je->stack_p; + + if (json_value_scalar(je)) + { + if (append_simple(to, je->value_begin, + je->value_end - je->value_begin)) + return 1; + first_value= 0; + } + else + { + if (to->append((je->value_type == JSON_VALUE_OBJECT) ? "{" : "[", 1)) + return 1; + first_value= 1; + depth++; + } + break; + + case JST_OBJ_END: + case JST_ARRAY_END: + if (to->append((je->state == JST_OBJ_END) ? "}" : "]", 1)) + return 1; + first_value= 0; + if (--depth == 0) + return je->s.error != 0; + break; + + default: + break; + } + } + + /* The document ended in the middle of the value. */ + return 1; +} + + +#ifdef DBUG_ASSERT_EXISTS +/* + Reads a value the way whoever receives it will read it: in the + character set the value says it is written in, from one end of it to + the other. +*/ +bool json_value_reads_as_document(const String *str) +{ + /* + Which is the question json_valid() is, so it is asked rather than + asked again here. The reading it does begins at json_scan_start() + like every other, so it is counted like every other and taken back + off afterwards - the same two steps json_scan_start_unbilled() makes, + in the order the borrowed reading leaves them in. + */ + bool valid= json_valid(str->ptr(), str->length(), str->charset()) != 0; + + json_uncount_scan(); + return valid; +} + + +/* + A kill is not simulated in a reading this file makes for itself. + + The arming that raises one partway through a walk is aimed at the + walks a statement makes on its way to an answer. A debug build makes + one walk MORE than a release build does - the one just below, which + writes a value out again to find out whether it was already written + that way - and a kill raised in THAT walk would be a debug build + ending a statement a release build finishes. What is checked here is + the code, and a check that changes the answer is not one. + + So it is held off for as long as the reading lasts, the way the + reading itself is held off the count of readings. +*/ +class Json_kill_unsimulated +{ + bool m_armed; +public: + Json_kill_unsimulated() : m_armed(DBUG_IF("json_kill_while_emitting")) + { + if (m_armed) + DBUG_SET("-d,json_kill_while_emitting"); + } + ~Json_kill_unsimulated() + { + if (m_armed) + DBUG_SET("+d,json_kill_while_emitting"); + } +}; + + +/* + Writes the value out again in the loose form and asks whether that + changed anything. A value already written that way comes back byte + for byte; one written any other way does not. +*/ +bool json_value_is_nice(const String *str) +{ + Json_kill_unsimulated unkilled; + json_engine_t je; + StringBuffer nice; + + json_scan_start_unbilled(&je, str->charset(), (const uchar *) str->ptr(), + (const uchar *) str->ptr() + str->length()); + if (json_nice(&je, &nice, Item_func_json_format::LOOSE)) + return false; + + return !stringcmp(&nice, str); +} + + +/* + How deep the value actually goes, read off the value itself. A + claimed depth is only ever compared against this one - a claim that + is larger than the truth is allowed, that being what an item saying + nothing amounts to, and one that is smaller is the failure this + exists to catch. +*/ +uint json_value_depth(const String *str) +{ + json_engine_t je; + uint deepest= 0; + + json_scan_start_unbilled(&je, str->charset(), (const uchar *) str->ptr(), + (const uchar *) str->ptr() + str->length()); + while (json_scan_next(&je) == 0) + { + if ((uint) je.stack_p > deepest) + deepest= (uint) je.stack_p; + } + + return deepest; +} +#endif + + +/* + A document being edited is walked once to find the place to edit, and + the pieces of it that go into the answer are copied from where that + walk left off - so the walk's pointers into it stay live across + everything that happens in between. What happens in between includes + working out a path and working out a value, and either can be any + expression a caller cares to write. + + Nothing here owns the document. It can be a table's row, a routine's + variable or another statement's user variable, and what an expression + writes is not this function's to know. That the bytes stay put is + true today because of the shapes an expression can take and not + because anything makes it so, which is exactly the kind of thing that + stops being true quietly. So a debug build holds on to a copy and + says whether it still matches. + + A release build keeps nothing and asks nothing, this being a check on + the code rather than on the data. +*/ +class Json_source_watch +{ +#ifndef DBUG_OFF + String m_held; + bool m_holding; +#endif +public: +#ifdef DBUG_OFF + void take(const String *) {} + bool unchanged(const String *) const { return true; } +#else + Json_source_watch() : m_holding(false) {} + void take(const String *js) + { + /* + A copy that cannot be made says nothing either way, and must not + turn into a complaint: the room to make it is made to run out on + purpose by the tests that check what happens when it does. + + A document that is not there says nothing either. Asking for one + is how a function finds out whether there is one, and the watch is + taken before anything else can run, so the two orders would + otherwise have to be told apart by every caller. + */ + m_holding= js && !m_held.copy(js->ptr(), js->length(), js->charset()); + } + bool unchanged(const String *js) const + { return !m_holding || !stringcmp(js, &m_held); } +#endif +}; + + +/* + A mark is a promise about a run of bytes, and the bytes are right + here, so a debug build keeps the promise by reading them. + + Nothing else ever will. The marks appear in no result and change no + output, so one of them being wrong shows up nowhere at all until + something starts acting on it - and by then the function that made the + promise is a long way from the code that believed it. +*/ +void Json_result_marks::set(const String *str, bool valid, bool nice, + uint depth) +{ + /* + The loose form is a formatting of a document, so there is no such + thing as a value formatted that way which is not one. Said before the + two below because it holds whatever the bytes turn out to be, and + because a caller that arrives here with the pair the wrong way round + has lost track of which question it was answering. + */ + DBUG_ASSERT(!nice || valid); + DBUG_ASSERT(!valid || json_value_reads_as_document(str)); + DBUG_ASSERT(!nice || json_value_is_nice(str)); + /* + A depth is only about a value that reads as a document, and is only + ever wrong in one direction - see json_value_depth(). + */ + DBUG_ASSERT(depth == JSON_DEPTH_UNKNOWN || !valid || + json_value_depth(str) <= depth); + m_valid= valid; + m_nice= nice; + m_depth= depth; +} + + +String *Json_value_arm::read(Item *arm, String *str) +{ + m_arm= arm; + String *res= m_read_as_json ? arm->val_json(str) : arm->val_str(str); + /* + An item that passes a value on passes the argument's answers on with + it, unread. This is the one moment both are in hand, so a debug + build holds the one against the other here, for the reason given at + Json_result_marks::set(): nothing else ever would. + + The reading costs nothing where there is no answer to hold, which is + every argument that is not a document. + */ + DBUG_ASSERT(!res || !arm->is_valid_json() || + json_value_reads_as_document(res)); + DBUG_ASSERT(!res || !arm->is_nice_json() || json_value_is_nice(res)); + return res; +} + + +/* + The three below are the argument's own answers, passed straight on. + An item that has read no argument yet says what an item with nothing + to say says. +*/ + +bool Json_value_arm::valid() const +{ + return m_arm && m_arm->is_valid_json(); +} + + +bool Json_value_arm::nice() const +{ + return m_arm && m_arm->is_nice_json(); +} -error: - return 1; + +uint Json_value_arm::depth() const +{ + return m_arm ? m_arm->last_depth() : JSON_DEPTH_UNKNOWN; } @@ -455,7 +1290,13 @@ void report_json_error_ex(const char *js, json_engine_t *je, Sql_condition::enum_warning_level lv) { THD *thd= current_thd; - int position= (int)((const char *) je->s.c_str - js); + /* + Where the scanner refused, which is not where it came to rest: it + goes on being asked for the next token by callers that are not + obliged to stop, so json_error() takes the reading at the refusal + and this reports that one. + */ + int position= (int)((const char *) je->s.error_pos - js); uint code; n_param++; @@ -508,6 +1349,58 @@ void report_json_error_ex(const char *js, json_engine_t *je, } +/* + Says that a character had nowhere to go, in the cases where that is + all that is being said: the bytes are passed on unchanged, as they + always were, and this note is the only sign that anything was amiss. + + A note and not a warning. A warning becomes an error inside a + statement running under strict mode, and every caller of this is + somewhere the released server said nothing at all, so a warning would + stop statements that used to finish. + + The argument is numbered the way the caller numbers it in its other + diagnostics, and the position is the beginning of it: what is + reported here is that the writing stopped, not where it stopped. +*/ +static void report_bad_chr_note(const char *fname, int n_arg) +{ + THD *thd= current_thd; + + if (thd) + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_JSON_BAD_CHR, + ER_THD(thd, ER_JSON_BAD_CHR), n_arg, fname, 0); +} + + +/* + The document argument of a call that has no other argument to work + out after it. + + Such a call reads the document and is finished with it before + anything else can run, so it can be answered with a view rather than + with bytes of its own - see Item::val_json_at_once(). A call that + does have another argument to work out cannot: working one out runs + whatever the caller wrote, and what the caller wrote can take the + document away. + + Asked of the argument count rather than written into each function, + so that a function which gains an optional argument later goes back + to being answered the careful way without anyone having to remember + to say so. + + Only for a function that returns nothing of the document: one + that passes it straight on, as JSON_UNQUOTE and JSON_COMPACT do when + there is nothing to unquote or to rewrite, gives its own caller a + view and has to read the document with val_json(). +*/ + +static inline String *val_json_arg0(Item **args, uint arg_count, String *str) +{ + return arg_count == 1 ? args[0]->val_json_at_once(str) + : args[0]->val_json(str); +} + #define NO_WILDCARD_ALLOWED 1 #define SHOULD_END_WITH_ARRAY 2 @@ -522,7 +1415,7 @@ void report_path_error_ex(const char *ps, json_path_t *p, Sql_condition::enum_warning_level lv) { THD *thd= current_thd; - int position= (int)((const char *) p->s.c_str - ps + 1); + int position= (int)((const char *) p->s.error_pos - ps + 1); uint code; n_param++; @@ -581,7 +1474,7 @@ static int path_setup_nwc(json_path_t *p, CHARSET_INFO *i_cs, if ((p->types_used & (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD | JSON_PATH_ARRAY_RANGE)) == 0) return 0; - p->s.error= NO_WILDCARD_ALLOWED; + json_error(&p->s, NO_WILDCARD_ALLOWED); } return 1; @@ -597,13 +1490,27 @@ CHARSET_INFO *def_path_charset(CHARSET_INFO *cs, CHARSET_INFO *alt) bool Item_func_json_valid::val_bool() { - String *js= args[0]->val_json(&tmp_value); + String *js= val_json_arg0(args, arg_count, &tmp_value); THD *thd; json_engine_t je; if ((null_value= args[0]->null_value)) return 0; + /* + What this function asks is what a value answers about itself, so + where the value has answered there is nothing left to find out. + + It is the same reading the check constraint leaves unrun at the + field boundary, left unrun here at the most direct site there is, + and the two cannot disagree: a mark says the characters read back + as a document, which is exactly what the walk below would go and + see. Nothing is said either way, the answer being true, and the + reading is the only thing that could have said anything. + */ + if (args[0]->is_valid_json()) + return true; + thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); je.killed_ptr= (uint32_t *) &thd->killed; @@ -632,10 +1539,12 @@ bool Item_func_json_equals::val_bool() String a_tmp, b_tmp; THD *thd; json_engine_t je; + Json_source_watch watch; String *a= args[0]->val_json(&a_tmp); if ((null_value= a == nullptr || args[0]->null_value)) return 1; + watch.take(a); String *b= args[1]->val_json(&b_tmp); if ((null_value= b == nullptr || args[1]->null_value)) return 1; @@ -659,6 +1568,7 @@ bool Item_func_json_equals::val_bool() JSON_DO_PAUSE_EXECUTION(thd, 0.0002); je.killed_ptr= (uint32_t *) &thd->killed; + DBUG_ASSERT(watch.unchanged(a)); if (json_normalize_engine(&je, &a_res, a->ptr(), a->length(), a->charset())) goto return_null; @@ -699,6 +1609,7 @@ bool Item_func_json_exists::fix_length_and_dec(THD *thd) bool Item_func_json_exists::val_bool() { json_engine_t je; + Json_source_watch watch; int array_counters[JSON_DEPTH_LIMIT]= {0}; THD *thd= current_thd; @@ -706,6 +1617,7 @@ bool Item_func_json_exists::val_bool() String *js= args[0]->val_json(&tmp_js); + watch.take(js); if (!path.parsed) { String *s_p= args[1]->val_str(&tmp_path); @@ -723,6 +1635,7 @@ bool Item_func_json_exists::val_bool() } null_value= 0; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -772,7 +1685,13 @@ bool Json_path_extractor::extract(String *str, Item *item_js, Item *item_jp, String *js= item_js->val_json(&tmp_js); int error= 0; int array_counters[JSON_DEPTH_LIMIT]= {0}; + Json_source_watch watch; + + /* Taken here rather than where they are used - see m_js_nice. */ + m_js_nice= item_js->is_nice_json(); + m_js_depth= item_js->last_depth(); + watch.take(js); if (!parsed) { String *s_p= item_jp->val_str(&tmp_path); @@ -806,6 +1725,7 @@ bool Json_path_extractor::extract(String *str, Item *item_js, Item *item_jp, if (item_js->null_value || item_jp->null_value) return true; + DBUG_ASSERT(watch.unchanged(js)); Json_engine_scan je(*js); str->length(0); str->set_charset(cs); @@ -895,10 +1815,23 @@ bool Item_func_json_quote::fix_length_and_dec(THD *thd) { collation.set(&my_charset_utf8mb4_bin); /* - Odd but realistic worst case is when all characters - of the argument turn into '\uXXXX\uXXXX', which is 12. + Odd but realistic worst case is when every character of the argument + has to be escaped, and six is what one of them costs. + + A character is written as the hex of its UTF-16 form, one unit for a + character of the first plane and two for any other, each unit taking + a `\uXXXX` of its own - so twelve where a pair is needed. A pair is + only ever needed for a character the set being written into cannot + carry, and the set here is utf8mb4, which carries every character + there is. So nothing is escaped for want of somewhere to put it and + what is left is the handful JSON will not have literally: a quote, a + backslash, and the control characters, all of them inside the first + plane and none of them costing more than six. + + The two for the quotes around the whole of it are this function's + own. */ - fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 12 + 2); + fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 6 + 2); return FALSE; } @@ -906,6 +1839,7 @@ bool Item_func_json_quote::fix_length_and_dec(THD *thd) String *Item_func_json_quote::val_str(String *str) { String *s= args[0]->val_str(&tmp_s); + json_append_result rc; if ((null_value= (args[0]->null_value || args[0]->result_type() != STRING_RESULT))) @@ -914,16 +1848,34 @@ String *Item_func_json_quote::val_str(String *str) str->length(0); str->set_charset(&my_charset_utf8mb4_bin); - if (str->append('"') || - st_append_escaped(str, s) || - str->append('"')) + if (str->append('"') || DBUG_IF("json_quote_open_out_of_memory")) + goto error; + + if ((rc= st_append_escaped(str, s))) { - /* Report an error. */ - null_value= 1; - return 0; + /* + A character no document can carry stops the writing, and this + function has always answered NULL for it. What it has never done + is SAY so: the three other JSON functions that write a value + through st_append_escaped() all report the note, and a value moved + from one of them to this one lost the only signal there is for the + condition. The optimizer trace calls it as well and reports + nothing, a trace not being an answer anybody asked for. The answer + is unchanged - only the silence is. + */ + if (rc == JSON_APPEND_BAD_CHR) + report_bad_chr_note(func_name(), 1); + goto error; } + if (str->append('"') || DBUG_IF("json_quote_close_out_of_memory")) + goto error; + return str; + +error: + null_value= 1; + return 0; } @@ -960,6 +1912,83 @@ String *Item_func_json_unquote::read_json(json_engine_t *je) } +/* + Returns the argument unchanged, but in the character set this item + declares. The bytes and the label have to agree: a value encoded one + way and labelled another is read wrongly by everything downstream of + it, and the character set is settled at fix time, so it is the bytes + that have to move. + + An argument that carries no character set at all is read one byte to + one character, which is how json_unescape() already reads it when the + value turns out to be a string. Both ways out of the function then + say the same thing about the same argument. +*/ +String *Item_func_json_unquote::return_as_is(String *str, String *js) +{ + CHARSET_INFO *from_cs= js->charset(); + uint errors; + + if (my_charset_same(from_cs, collation.collation)) + return js; + + if (from_cs == &my_charset_bin) + { + /* + An argument that carries no character set is read one byte to one + character, which is the reading json_unescape() gives it on the + other way out: my_charset_bin's mb_wc returns the byte itself. + Reading it any other way would have the two exits of this function + name the same byte as two different characters. latin1 in + particular is NOT that reading - MariaDB's latin1 is cp1252, which + differs from it at 27 of the 32 positions in 0x80-0x9F. + + Converted rather than copied, and not through String::copy(), + which would not convert: needs_conversion() answers false for a + binary source whose length divides the destination's mbminlen, and + utf8mb4's is 1, so the bytes would go across untouched and be + relabelled - 0x80 would leave here as a lone continuation byte, + which is not a character of the set the result says it is in. + copy_and_convert() runs the conversion that short-circuit skips. + */ + uint32 room= (uint32) (collation.collation->mbmaxlen * js->length()); + + if (str->alloc(room) || DBUG_IF("json_unquote_as_is_out_of_memory")) + { + null_value= 1; + return NULL; + } + str->length(copy_and_convert((char *) str->ptr(), room, + collation.collation, js->ptr(), + js->length(), from_cs, &errors)); + str->set_charset(collation.collation); + } + else if (str->copy(js->ptr(), js->length(), from_cs, + collation.collation, &errors) || + DBUG_IF("json_unquote_as_is_out_of_memory")) + { + null_value= 1; + return NULL; + } + + /* + Only a conversion that lost nothing is worth having. A character + with nowhere to go comes out of the conversion as a substitute for + itself, and returning that would answer differently than this + call has always answered - the argument would come back altered + rather than merely mislabelled. The label is the lesser of the two + wrongs, so the bytes stay as they arrived and the note says why. + */ + if (errors) + { + report_bad_chr_note("unquote", 0); + return js; + } + + return str; +} + + String *Item_func_json_unquote::val_str(String *str) { json_engine_t je; @@ -970,7 +1999,7 @@ String *Item_func_json_unquote::val_str(String *str) return NULL; if (unlikely(je.s.error) || je.value_type != JSON_VALUE_STRING) - return js; + return return_as_is(str, js); int buf_len= je.value_len; if (js->charset()->cset != my_charset_utf8mb4_bin.cset) @@ -1006,7 +2035,7 @@ String *Item_func_json_unquote::val_str(String *str) ER_JSON_BAD_CHR, ER_THD(current_thd, ER_JSON_BAD_CHR), 0, "unquote", 0); } - return js; + return return_as_is(str, js); } @@ -1149,13 +2178,20 @@ String *Item_func_json_extract::read_json(String *str, json_engine_t je, sav_je; json_path_t p; const uchar *value; + uint32 copy_start, copy_len; int not_first_value= 0, count_path= 0; uint n_arg; - size_t v_len; int possible_multiple_values; + /* How far down the deepest of the values written out goes. */ + int deepest_value= 0; + /* The same, where the answer is read back instead of counted. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; int array_size_counter[JSON_DEPTH_LIMIT]; uint has_negative_path= 0; THD *thd; + Json_source_watch watch; + + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; @@ -1164,6 +2200,7 @@ String *Item_func_json_extract::read_json(String *str, JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); for (n_arg=1; n_arg < arg_count; n_arg++) { json_path_with_flags *c_path= paths + n_arg - 1; @@ -1203,8 +2240,16 @@ String *Item_func_json_extract::read_json(String *str, goto error; } + DBUG_ASSERT(watch.unchanged(js)); json_get_path_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length(), &p); + /* + This walk is now the only one there is, so it is the one that has to + notice being killed. It used to be worth noticing only over the + result, that being read separately afterwards; the reading of the + document itself went uninterrupted however long the document was. + */ + je.killed_ptr= (uint32_t *) &thd->killed; while (json_get_path_next(&je, &p) == 0) { @@ -1231,39 +2276,90 @@ String *Item_func_json_extract::read_json(String *str, goto return_ok; } + if ((not_first_value && str->append(STRING_WITH_LEN(json_loose_comma))) || + (not_first_value && DBUG_IF("json_extract_comma_out_of_memory"))) + goto error; + + copy_start= str->length(); + if (json_value_scalar(&je)) - v_len= je.value_end - value; + { + /* + A scalar is punctuated with nothing, so however it is written in + the document is how it is written in the loose form as well, and + it is copied across as it stands - which is the copy json_nice() + used to make of it, made here instead. Copied and not written, + so it must not be converted on the way: it came out of the + document, and the document is in the character set the result is + being built in. + */ + if (append_simple(str, value, je.value_end - value) || + DBUG_IF("json_extract_scalar_out_of_memory")) + goto error; + } else { + /* + A container is written out rather than copied. How it looks in + the document is not how the loose form looks, and finding that + out by writing the whole result and reading it back again is the + reading being done away with here. + + The walk over the value is the walk this loop was going to make + anyway - json_skip_level() went over exactly these characters + without writing anything down. + */ if (possible_multiple_values) sav_je= je; - if (json_skip_level(&je)) - goto error; - v_len= je.s.c_str - value; + /* + How far down this value goes, counted from itself rather than + from the document it was cut out of. Reading a container has + already put it on the stack, so the level here is the value's + own outermost one, and what the walk reaches above that is what + it holds. The bracket written round the whole answer sits one + further out again - see where it is closed. + */ + { + int base= je.stack_p, reached= je.stack_p; + + if (json_walk_nice_value(&je, str, reached)) + goto error; + if (reached - base + 1 > deepest_value) + deepest_value= reached - base + 1; + } if (possible_multiple_values) je= sav_je; } - if ((not_first_value && str->append(", ", 2))) - goto error; - while(count_path) + /* + A value that matched more than one of the paths asked for is + written once and then repeated from where it was written, rather + than being formatted again or held anywhere of its own. The room is + taken first, so that the read below is from a buffer that has + already finished moving. + */ + copy_len= str->length() - copy_start; + while (--count_path) { - if (str->append((const char *) value, v_len)) + if (str->append(STRING_WITH_LEN(json_loose_comma)) || + str->reserve(copy_len)) goto error; - count_path--; - if (count_path) - { - if (str->append(", ", 2)) - goto error; - } + str->q_append(str->ptr() + copy_start, copy_len); } not_first_value= 1; if (!possible_multiple_values) { - /* Loop to the end of the JSON just to make sure it's valid. */ - while (json_scan_next(&je) == 0) {} + /* + The rest of the document is parsed only to check that it is one. + Not done at all where the item has already attested that the + value is_valid. + */ + if (!args[0]->is_valid_json()) + { + while (json_scan_next(&je) == 0) {} + } break; } } @@ -1280,15 +2376,82 @@ String *Item_func_json_extract::read_json(String *str, if (possible_multiple_values && str->append(']')) goto error; /* Out of memory. */ - js= str; - json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), - (const uchar *) js->ptr() + js->length()); - je.killed_ptr= (uint32_t *) &thd->killed; + /* + In a character set that cannot encode the punctuation this function + writes - see is_json_compatible_charset() - what has been composed is + not a document and no reasoning about it can make it one, the + brackets holding the values having come out as national letters. + Such a set can still hold a document, so the walk above succeeded + and there is nothing else to notice it by. Reading the result back + is what has always noticed it, and where it cannot encode that + reading stays exactly where it was. + */ + /* + And where the bracket written round the values takes the answer + deeper than a document is allowed to go. That bracket is a level + this function adds: every value in it was measured against the + document it came out of, where the bracket did not exist. A + released server found this out by reading the answer back and being + refused, so being refused by the same reading is what it is owed - + working it out here and complaining directly would put the + complaint somewhere else in the text. + */ + if (!is_json_compatible_charset(str->charset()) || + (possible_multiple_values && deepest_value + 1 >= JSON_DEPTH_LIMIT)) + { + js= str; + json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), + (const uchar *) js->ptr() + js->length()); + je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, &tmp_js, Item_func_json_format::LOOSE)) - goto error; + if (json_nice(&je, &tmp_js, Item_func_json_format::LOOSE, &read_back_depth)) + goto error; + + /* + Both marks come from the reading back, for the reason given where + Item_func_json_insert::val_str() marks the same answer, and so + does the depth. + */ + m_marks.set(&tmp_js, true, true, read_back_depth); + return &tmp_js; + } + + /* + Nothing is read back here. Every value in the result was written + out in the loose form while the document was being walked, and the + brackets that hold them are written the same way, so the result is + already written as reading it back would have written it. And it is + a document because each part of it was read as one on the way in. + + Unlike the six that EDIT a document, this asks nothing of the + argument: nothing of it is kept. What comes back is written here + out of values this function itself walked to and read, so the only + thing that can be wrong with it is what holds those values + together - which is why the one condition above is whether the + brackets can be written at all, and why there is no trust predicate + here to ask. m_marks.set() reads it back in a debug build all the + same. + + This is also why the result is returned in the caller's own + buffer now. It used to be returned in tmp_js, which is the + scratch the argument may have been read into - so the answer was + being written over the document it was made from, and only got away + with it because the document had been finished with. + + How deep it goes was counted as it was written: each value put in + was walked, and the deepest of them is the deepest the answer goes, + plus the bracket written round them where there is one. + */ + m_marks.set(str, true, true, + (uint) deepest_value + (possible_multiple_values ? 1 : 0)); + return str; return_ok: + /* + A caller that passed no buffer wanted the value picked out, not + written out, and reads it through out_val. What comes back only has + to say that something was found. + */ return &tmp_js; error: @@ -1581,6 +2744,7 @@ bool Item_func_json_contains::val_bool() json_engine_t je, ve; int result; THD *thd; + Json_source_watch watch; if ((null_value= args[0]->null_value)) return 0; @@ -1588,6 +2752,7 @@ bool Item_func_json_contains::val_bool() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); if (!a2_parsed) { val= args[1]->val_json(&tmp_val); @@ -1600,6 +2765,7 @@ bool Item_func_json_contains::val_bool() return 0; } + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -1626,6 +2792,7 @@ bool Item_func_json_contains::val_bool() goto return_null; path.cur_step= path.p.steps; + DBUG_ASSERT(watch.unchanged(js)); if (json_find_path(&je, &path.p, &path.cur_step, array_counters)) { if (je.s.error) @@ -1822,7 +2989,10 @@ bool Item_func_json_contains_path::val_bool() int UNINIT_VAR(n_found); int array_sizes[JSON_DEPTH_LIMIT]; uint has_negative_path= 0; + /* Asked once: the walk below reaches it on every step. */ + bool js_attested; THD *thd; + Json_source_watch watch; if ((null_value= args[0]->null_value)) return 0; @@ -1830,6 +3000,7 @@ bool Item_func_json_contains_path::val_bool() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); if (parse_one_or_all(this, args[1], &ooa_parsed, ooa_constant, &mode_one)) goto null_return;; @@ -1856,6 +3027,7 @@ bool Item_func_json_contains_path::val_bool() goto null_return; } + DBUG_ASSERT(watch.unchanged(js)); json_get_path_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length(), &p); je.killed_ptr= (uint32_t *) &thd->killed; @@ -1867,6 +3039,7 @@ bool Item_func_json_contains_path::val_bool() } result= 0; + js_attested= args[0]->is_valid_json(); while (json_get_path_next(&je, &p) == 0) { int n_path= arg_count - 2; @@ -1898,6 +3071,15 @@ bool Item_func_json_contains_path::val_bool() p_found[n_path-1]= TRUE; } } + + /* + Nothing left that could unsettle the answer. The walk goes on past + a settled one only so that a fault later in the document can take + it back, and a value the item has attested is_valid has no such + fault to be found. + */ + if (result && js_attested) + break; } if (likely(je.s.error == 0)) @@ -1927,17 +3109,531 @@ bool is_json_type(const Item *item) { if (Type_handler_json_common::is_json_type_handler(item->type_handler())) return true; - const Item_func_conv_charset *func; - if (!(func= dynamic_cast(item->real_item()))) + /* + The wrapper is asked what it is put round rather than made to + prove what it is, which is one virtual call where a dynamic_cast + walks the base classes of whatever it was handed before it can + say no. Every argument that is not a document reaches here and + is told no, and on the arms this is called from those are most of + them. + */ + Item *arg= item->real_item()->conv_charset_arg(); + if (!arg) return false; - item= func->arguments()[0]; + item= arg; + } + return false; +} + + +/* + What splicing values into a document has left the document able to say + about itself. Starts out saying both and is only ever cleared: one + value that cannot be attested to is enough, however many others went + in cleanly. + + Named for what holds rather than for what went wrong, and named after + the questions an item answers, so that a value's own answer and what + became of it here read the same way round. A document is what the + the loose form is a form OF a document, so nothing clears the second + without clearing the first. + + Every caller keeps a set, so every value put in is accounted for + whether or not the caller ends up believing what they say. A caller + that reads its whole answer back afterwards has a stronger answer than + anything collected on the way and takes that instead; it still hands + one of these over, there being nowhere else for a value's own answer + to go. +*/ +struct Json_splice_marks +{ + bool is_valid; /* everything that went in read as a document */ + bool is_nice; /* everything that went in was written the loose way */ + /* + Set when the two above were cleared because a value goes past the + depth limit only once it is in place. Kept apart from the other + reasons, so that a caller which still reads its whole answer back + can check the depth it reckoned at the splice against what that + reading found: a reading that reached the end is an answer inside + the limit, and nothing should have said otherwise. + + Only that check ever reads it, so it is only there where it can be + read. That is not the same set of builds as the debug one: an + assertion survives into a build with DBUG_OFF set where it was asked + to print rather than to stop, and the expression it is given is + compiled there. + */ +#ifdef DBUG_ASSERT_EXISTS + bool is_deep; +#endif + /* + The deepest any value put in reaches, counted from the outside of + the document being composed rather than from the value itself. A + caller starts it at the depth of whatever it is writing the values + inside, so that it holds for a document with no values in it too, + and it is then the depth of the whole composed answer - every other + part of that answer being punctuation the caller wrote itself. + + Wrong only ever upwards, like the answers it is made of: a value + taken on trust contributes the bound the trusting was done against + rather than a measurement, and a bound is not smaller than the + truth. + */ + uint deepest; + + Json_splice_marks(uint depth) + : is_valid(true), is_nice(true), deepest(depth) + { +#ifdef DBUG_ASSERT_EXISTS + is_deep= false; +#endif + } +}; + + +/* + Appends a value whose type is JSON, which is spliced into the result + as it stands instead of being quoted as a string. Any complete JSON + value counts, a bare number or string as much as an object: what the + type buys the value is that its own punctuation is kept rather than + escaped away. + + Being typed as JSON is not the same as being JSON. The type comes + from a check constraint, the constraint can be switched off for the + duration of a statement, and the bytes it would have rejected stay in + the column afterwards. Nothing on the way from the column to here + reads them. + + So the value is parsed as it is copied, and a value that does not + parse is copied ALL THE SAME, with a complaint saying so. Undoing the + copy is not this function's to do: the bytes have gone in where the + caller asked for them, and how much of a result there is around them + is the caller's to know. What is owed here is to say so, which is + what the marks are for, and each caller answers it the way its own + result works. The four that compose a document and return what they + composed return NULL rather than bytes no reader will take; the + ones that read their whole answer back find out there, exactly as + before. + + A value written in another character set is converted first, because + the result is read in the character set it is being built in, not in + the one the value arrived in. The quoting arm converts the same way, + through json_escape(); a value spliced as JSON cannot go through + json_escape() without its punctuation being escaped along with + everything else, so it is converted whole and then parsed as whatever + it has become. A character set that cannot hold the value leaves the + bytes where they were, again because that is what was done before. + + 'depth' is how many structures the value will sit inside once it has + been spliced. Its own nesting adds to that, and it is the total that + the scanner's limit applies to. + + 'marks' is where the parse goes, the bytes having gone in whatever it + found. See Json_splice_marks. + + 'is_valid' and 'is_nice' are what the item handing the value over + says about it. is_valid true is the one case where the parse can be + left undone; is_nice false is what the writing out further down is + for. 'need_nice' is the caller saying that what it composes is what + it returns, so a value that is not is_nice has to be made so. + + 'value_depth' is how deep the item says the value goes, or + JSON_DEPTH_UNKNOWN where it does not say. It is only ever read + alongside is_valid. + + 'lv' is the level to complain at. A caller that has another + complaint to make about the same value - which every caller that + reads its whole answer back does, from the reading - leaves this a + note, so that the two do not say the same thing twice at the same + volume. The four that compose their answer and return it have no + second complaint and no answer either, and pass a warning: it is the + only word the statement gets about why it was given NULL, and under + strict mode it stops the statement where the value went wrong rather + than letting it finish with nothing. +*/ +static int append_json_typed_value(String *str, const String *sv, uint depth, + const char *fname, int n_param, + Json_splice_marks &marks, + bool caller_reads_back, bool is_valid, + bool is_nice, bool need_nice, + uint value_depth, + Sql_condition::enum_warning_level lv) +{ + StringBuffer cnv; + const char *ptr= sv->ptr(); + size_t length= sv->length(); + json_engine_t je; + int max_level= 0; + bool reformat; + uint32 sav_len; + /* + Whether the bytes at 'ptr' are written in the character set the + result is being built in. What an item answers about a value is + about the characters it holds, so this is what says whether the + answer is about the bytes that are going in. + */ + bool in_result_charset= my_charset_same(sv->charset(), str->charset()); + + /* + Bytes that carry no character set are left where they are, in both + directions and for opposite reasons. Going into a result that is + bytes, there is no character set to convert to. Coming out of a + value that is bytes, String::copy() cannot convert at all - + needs_conversion() answers false across that boundary, so the copy + would relabel the bytes and change nothing else, which is the one + thing this must not do. The scan below still reads whatever gets + appended in the character set of the result, so bytes that do not + encode a JSON value there are still refused. + */ + if (!in_result_charset && + sv->charset() != &my_charset_bin && str->charset() != &my_charset_bin) + { + uint errors; + + if (cnv.copy(sv->ptr(), sv->length(), sv->charset(), str->charset(), + &errors) || + DBUG_IF("json_splice_convert_out_of_memory")) + return 1; /* Out of memory. */ + + /* + Only a conversion that lost nothing is worth having. One that + substituted a character has changed the value, and putting a + changed value in would be worse than leaving the bytes as they + arrived, which is what was always done with them. The parse + below reads them as the result will be read and says so. + + One that lost nothing kept the characters it was given, and being + a document is a property of characters, so a value that arrived as + one is still one after it - and still formatted the way it was, the + loose form's spacing being characters like any other. So what was + answered about it before the conversion is answered about it + after, and the shortcut below can have it. + */ + if (!errors) + { + ptr= cnv.ptr(); + length= cnv.length(); + in_result_charset= true; + } + } + + /* + A value somebody has already attested to is copied in without being + read, which is the whole of what attesting buys. Three things + have to hold besides. + + It must be written the way the result is going to be written, or the + caller must not care. A value copied in as it stands brings its own + spacing with it, so a result that has to come out in the loose form + cannot take one that is not in it without reading it - which is what + the writing out further down is for. + + It must be going in in the character set the result is read in - + either because it arrived in it, or because it was converted into it + just above without losing anything. A lossy conversion, and equally + the untouched copy made when there is no conversion to be had, leave + bytes that nothing has attested to and fall through to the parse. + + And it must not make the result too deep to read back. The value's + own nesting is what the parse below measures, and the shortcut is not + doing that - so it takes the smallest thing it has that the nesting + cannot be more than. + + There are two such things and either alone would do. The item may + have worked the depth out while it was writing the value, in which + case it says so. And a value can only nest as deeply as it is long, + whatever it holds: every level takes a character to open and one to + close, so a value of n characters reaches at most n/2 levels, and + there are at most length/mbminlen characters in it. That second one + is exact for the values where being wrong would matter most - a run + of nothing but brackets is all opening and closing - and hopeless for + a long shallow document, which is what the first one is for. + + Where the smaller of them still leaves the result inside the limit, + the parse could not have found a breach to report and skipping it + takes nothing away. Where it does not, the value is read as before: + giving up costs a reading and nothing else. + */ + if (is_valid && (is_nice || !need_nice) && in_result_charset) + { + /* The most levels the value can turn out to have - see above. */ + uint bound= MY_MIN(value_depth, + (uint) (length / (2 * str->charset()->mbminlen))); + + if (depth + bound < JSON_DEPTH_LIMIT) + { + if (!is_nice) + marks.is_nice= false; + /* + What the value was let in on, rather than what it turned out to + be - nothing measured it. A bound is not smaller than the + truth, which is all a caller composing from these needs of it. + */ + if (depth + bound > marks.deepest) + marks.deepest= depth + bound; + return append_simple(str, ptr, length); + } + } + + /* + The deepest the value goes is read off the scanner as it runs rather + than charged to it in advance. Starting the scan with stack_p + already raised looks like the shorter way to ask the same question, + but json_scan_start() marks stack[0] as the end of the document and + the scanner stops when it pops back to that mark, so raising the + pointer past it leaves unwritten slots underneath and the scan never + finishes. + */ + /* + A value that is not already written the loose way is written out + again as it is read, where the caller has said that what it composes + is what it returns. The reading has to happen either way, and a + reading that writes as it goes costs nothing more than one that does + not - so the value arrives in the form the result needs instead of + the caller reading its whole answer afterwards to put it in that + form. + + Where the caller does NOT say that, the value is left exactly as it + arrived. Two quite different callers say nothing here. One builds + an array or an object and returns what it built, so a value put + in as it stands is what a released server returns too and writing + it out afresh would change an answer. The other reads its whole + result back before answering, and that reading is where the spacing + gets settled - and it complains, when it has to, about positions in + the text it read, so a value that arrived any longer or shorter than + it used to would move them. + + A character set that cannot encode the punctuation cannot be written + into at all, and is left alone for the same reason it is everywhere + else here. + */ + reformat= need_nice && !is_nice && + is_json_compatible_charset(str->charset()); + sav_len= str->length(); + + json_scan_start(&je, str->charset(), (const uchar *) ptr, + (const uchar *) ptr + length); + /* + Every reading here has to let go of a killed query, and this one is + no different for being over a value rather than over a document. + json_scan_start() leaves the scanner deaf to a kill - it points at a + word that is always zero - so a reading that is meant to hear one + says so on the line after, as every other reading in this file does. + + It matters more here than the size of a value suggests. A released + server does no reading at all at this point: it copies the bytes in + and goes on, so a kill arriving while a value is going in is heard + at the very next step. The reading is ours, and an unheard kill + would be ours too - a stretch of work, as long as the value, that + nothing could interrupt and that nobody had before. + */ + je.killed_ptr= (uint32_t *) ¤t_thd->killed; + + if (reformat) + { + if (json_read_value(&je) || json_walk_nice_value(&je, str, max_level)) + { + /* + Nothing was read wrong, so what went wrong was the writing: + there is no room for the answer and no answer to give. + */ + if (!je.s.error) + return 1; + } + else + { + /* + The walk stops where the value ends, and a value is only a + document when nothing follows it. + */ + while (json_scan_next(&je) == 0) + {} + } + + /* + Written out again only if it came out whole and fits where it is + going. Either way the caller is owed what a released server + returns, so anything else is taken back off and the bytes go in + as they arrived - which is also what leaves the complaint below + about the same text it has always been about. + */ + if (je.s.error || max_level + (int) depth >= JSON_DEPTH_LIMIT) + { + str->length(sav_len); + reformat= false; + } + } + else + { + while (json_scan_next(&je) == 0) + { + if (je.stack_p > max_level) + max_level= je.stack_p; + } + } + + /* + Spliced as it stands, so the document being built is written the + loose way only if this value was. Nothing is measured to find that + out - the item was asked, and one that says nothing is taken at its + word. + */ + if (!reformat && !is_nice) + marks.is_nice= false; + + /* + Too deep only once it is in place. There is nothing wrong with the + value itself, so a caller that reads its whole result back says it + about the answer rather than about this one value - which is what it + has always said, and where it has always said it. + + A caller that does not read back has nobody else to hear it from, so + for that one the scanner is put into error and the complaint goes out + below at whatever level that caller asked for. Clearing the mark is + what carries it further: a composed answer that reaches past the + limit cannot be read back by this server at all, so the caller + composing it has nothing to return, and the mark is where it finds + that out. + */ + if (je.s.error == 0 && max_level + (int) depth >= JSON_DEPTH_LIMIT) + { + marks.is_valid= marks.is_nice= false; +#ifdef DBUG_ASSERT_EXISTS + marks.is_deep= true; +#endif + if (!caller_reads_back) + json_error(&je.s, JE_DEPTH); + } + /* + Measured rather than reckoned, this being the reading the shortcut + above was for going without. Only where the reading finished: what + a walk that stopped early reached is about the part of the value it + got through, and there is nothing to compose out of a value that + did not read as one anyway. + */ + else if (je.s.error == 0 && depth + (uint) max_level > marks.deepest) + marks.deepest= depth + (uint) max_level; + + /* + Said here rather than left to the caller: the callers report through + the engine they scanned the DOCUMENT with, which knows nothing about + what went wrong with a value, and reports nothing at all when that + engine is not itself in error. + + At the level the caller asked for, and see there for what picks it. + */ + if (je.s.error) + { + marks.is_valid= marks.is_nice= false; + report_json_error_ex(ptr, &je, fname, n_param, lv); + } + + return reformat ? 0 : append_simple(str, ptr, length); +} + + +/* + Writes a value that is not itself a document, as a JSON string when + the value is one and bare otherwise. + + A character that cannot be written into a document at all leaves the + value part written and stops there, which is what has always happened + to it. The caller is told which of the two failures it was, because + only one of them is a reason to give up on the whole result. +*/ +static json_append_result __attribute__((warn_unused_result)) +append_escaped_value(String *str, const String *sv, bool quoted, + const char *fname, int n_param) +{ + json_append_result rc; + + if ((quoted && str->append('"')) || + DBUG_IF("json_value_open_quote_out_of_memory")) + return JSON_APPEND_OOM; + + if ((rc= st_append_escaped(str, sv))) + { + if (rc == JSON_APPEND_BAD_CHR) + report_bad_chr_note(fname, n_param + 1); + return rc; + } + + if ((quoted && str->append('"')) || + DBUG_IF("json_value_close_quote_out_of_memory")) + return JSON_APPEND_OOM; + + return JSON_APPEND_OK; +} + + +__attribute__((warn_unused_result)) +/* + Appends one value to the document being built. + + 'depth' is how many structures the value will end up inside, and it is + consulted for EVERY value rather than only for one whose type is JSON: + it is recorded before the type is looked at, because a value goes in + where the caller says whatever it turns out to be. The functions that + BUILD a document pass 1, the value going directly inside the array or + object they are making. The functions that EDIT one pass the depth + the path reached, taken off the scanner as je.stack_p and raised by + whatever wrap they are adding - none of them passes 0. + + 'fname' and 'n_param' name the function and the argument a complaint + is about, and they reach BOTH arms: a value spliced in as a document + complains through the trailing-text and depth notes, and one written + out as a string complains through the bad-character note. +*/ +static int append_json_value(String *str, Item *item, String *tmp_val, + uint depth, const char *fname, int n_param, + Json_splice_marks &marks, + bool caller_reads_back, bool need_nice, + Sql_condition::enum_warning_level lv= + Sql_condition::WARN_LEVEL_NOTE) +{ + /* + Said before anything is looked at, because it holds for whatever the + value turns out to be: it is going in where the caller says, so the + answer reaches at least that far down. Only a value that is a + document of its own reaches further, and that is the one arm below + that has anything to add. + + It is not the caller's own depth restated. A caller that wraps - + putting a new array where a scalar was and the new value beside it - + passes a depth one further down than anything it has written, + and a value QUOTED at that depth would otherwise be counted nowhere + at all. + */ + if (depth > marks.deepest) + marks.deepest= depth; + + /* + And too deep before anything is looked at either, for the same + reason. A value put where the limit has already been reached is + past it whatever it turns out to be: the arms below add whatever + nesting the value has of its own and none of them takes any away. + Only the arm that READS the value can say how much further down it + reaches, so this is the whole of the question for the three that + read nothing - a bare word, a null, and a value written out as a + string. + + Saying so is all that is owed here. A value only arrives this deep + through a function that EDITS a document, by putting a new array + where a value already sat; every one of those returns what it + composed only while these marks stand, so taking them away puts it + back on the reading that a released server always did, and that + reading complains about the answer in the words it has always used. + The functions that BUILD a document put their values one level down + and cannot get here. + */ + if (depth >= JSON_DEPTH_LIMIT) + { + marks.is_valid= marks.is_nice= false; +#ifdef DBUG_ASSERT_EXISTS + marks.is_deep= true; +#endif } - return false; -} - -static int append_json_value(String *str, Item *item, String *tmp_val) -{ if (item->type_handler()->is_bool_type()) { longlong v_int= item->val_int(); @@ -1962,18 +3658,41 @@ static int append_json_value(String *str, Item *item, String *tmp_val) } { String *sv= item->val_json(tmp_val); + int rc; + if (item->null_value) goto append_null; if (is_json_type(item)) - return str->append(sv->ptr(), sv->length()); + return append_json_typed_value(str, sv, depth, fname, n_param, marks, + caller_reads_back, item->is_valid_json(), + item->is_nice_json(), need_nice, + item->last_depth(), lv); - if (item->result_type() == STRING_RESULT) - { - return str->append('"') || - st_append_escaped(str, sv) || - str->append('"'); - } - return st_append_escaped(str, sv); + /* + Being a document is what gets a value spliced rather than quoted, + and an item can attest to a value that is still quoted. A stored + program's variable is the case: it is never typed as a document, + however it was declared, and it attests to its value all the same + - see Item_sp_variable::is_valid_json(), which is answered so that + a function handed the value as its DOCUMENT can stop reading what + it already knows, and not so that this one splices it. + + Nothing here reads the answer, and quoting a value that is a + document is what quoting any other value is: the characters are + written out escaped and the quotes go round them. + */ + + rc= append_escaped_value(str, sv, item->result_type() == STRING_RESULT, + fname, n_param); + /* + A character that could not be written leaves the value half + written and the quote around it unclosed. A caller that gives up + here does not care; one that carries on is building something that + no longer reads as a document. + */ + if (rc == JSON_APPEND_BAD_CHR) + marks.is_valid= marks.is_nice= false; + return rc; } append_null: @@ -1981,9 +3700,20 @@ static int append_json_value(String *str, Item *item, String *tmp_val) } +/* + The same, reading the value out of a row rather than off an Item. + Only JSON_ARRAYAGG reads a row back this way, so the level it + complains at is the one that composer asks for everywhere else. +*/ +__attribute__((warn_unused_result)) static int append_json_value_from_field(String *str, - Item *i, Field *f, const uchar *key, size_t offset, String *tmp_val) + Item *i, Field *f, const uchar *key, size_t offset, String *tmp_val, + uint depth, const char *fname, int n_param, Json_splice_marks &marks) { + /* Said up front for the reason given in append_json_value(). */ + if (depth > marks.deepest) + marks.deepest= depth; + if (i->type_handler()->is_bool_type()) { longlong v_int= f->val_int(key + offset); @@ -2008,18 +3738,47 @@ static int append_json_value_from_field(String *str, } { String *sv= f->val_str(tmp_val, key + offset); + int rc; + if (f->is_null_in_record(key)) goto append_null; if (is_json_type(i)) - return str->append(sv->ptr(), sv->length()); - - if (i->result_type() == STRING_RESULT) - { - return str->append('"') || - st_append_escaped(str, sv) || - str->append('"'); - } - return st_append_escaped(str, sv); + /* + The bytes come out of a record rather than off an item, so there + is nobody here to ask about them - but the column they came out + of was asked, once for all its rows. A column of a table the + server built for itself can say that its values read as + documents, one producer having filled every row of it and every + store into it having kept what it was given, and it can say that + none of them arrived written any way but the loose one. A column + of any other table says neither, having nowhere to keep a yes. + + Nothing is written out again here. The only callers that read a + value out of a record are building a document and returning + what they built, and a value put in as it stands is what a + released server returns. + + It can say how deep they go as well, and that answer is a figure + rather than a yes: the deepest any value written into the column + has gone, which is at least as deep as this one. What it saves + is the only thing left to read a trusted value for - the reading + that is skipped elsewhere is a validating one, and this one + counts brackets and nothing else. Without it the caller falls + back on how long the value is, nothing nesting deeper than half + its length, which is exact for a run of brackets and says + nothing whatever about a long shallow document. + */ + return append_json_typed_value(str, sv, depth, fname, n_param, marks, + false, f->is_valid_json_static(), + f->is_nice_json_static(), false, + f->json_static_depth(), + Sql_condition::WARN_LEVEL_WARN); + + rc= append_escaped_value(str, sv, i->result_type() == STRING_RESULT, + fname, n_param); + if (rc == JSON_APPEND_BAD_CHR) + marks.is_valid= marks.is_nice= false; + return rc; } append_null: @@ -2027,18 +3786,107 @@ static int append_json_value_from_field(String *str, } -static int append_json_keyname(String *str, Item *item, String *tmp_val) +/* + Writes a key, which is a JSON string and nothing else. + + A character that cannot be written into a document at all leaves the + key part written and stops there, the same as it does in a value - + and the caller is told which of the two failures it was for the same + reason. What the caller then does with a key it could not write is + its own business: a constructor gives up on the whole document and an + aggregate finishes the pair around what went in, but neither of them + can say so without being told, and a key that could not be written is + worth as much saying as a value that could not. +*/ +static json_append_result append_json_keyname(String *str, Item *item, + String *tmp_val, + const char *fname, int n_param) { + json_append_result rc; String *sv= item->val_str(tmp_val); if (item->null_value) goto append_null; - return str->append('"') || - st_append_escaped(str, sv) || - str->append("\": ", 3); + if (str->append('"') || DBUG_IF("json_keyname_quote_out_of_memory")) + return JSON_APPEND_OOM; + + if ((rc= st_append_escaped(str, sv))) + { + if (rc == JSON_APPEND_BAD_CHR) + report_bad_chr_note(fname, n_param + 1); + return rc; + } + + return (str->append(STRING_WITH_LEN(json_loose_colon)) || + DBUG_IF("json_keyname_colon_out_of_memory")) ? + JSON_APPEND_OOM : JSON_APPEND_OK; append_null: - return str->append("\"\": ", 4); + return (str->append('"') || + str->append(STRING_WITH_LEN(json_loose_colon))) ? + JSON_APPEND_OOM : JSON_APPEND_OK; +} + + +/* + How much room a value needs in a document being built, before any + value has been seen and so out of what the argument says about itself. + + A value that is already a document goes in as it stands, but it is + written out again with the rest and a space arrives after every + separator it brought with it, so the room for it covers the writing + the same way JSON_REMOVE asks for it to. One written as text and not + a document is written as a JSON string, and twice its characters is + what is asked for that. + + Twice does not cover an escaping: a character written out as the hex + of its UTF-16 form costs six, and twelve where the character needs a + pair of them, so a value escaped throughout is asked for too little + and is cut. What it costs turns on the character set being written + into as well as the value's own, neither of them alone, and pricing + it properly widens what these functions declare enough to move a + result from a memory temporary table onto disk. That is left to be + done together with the work that stops a declared width deciding + where a temporary table lives. + + A string carries no separators of its own, so the spacing has nothing + to add to it. A boolean is written as one of the two words 'true' and + 'false', the longer of which is five characters. What is left is a + number, which is written as itself. A value that is not there at all + is written as 'null', so nothing is ever shorter than four characters. + + The punctuation that goes around it is the caller's to ask for. + + Whether it IS a document has to be the question the writing asks, and + not a shorter one that happens to agree most of the time. Aggregating + the arguments into one character set wraps whichever of them has to + move, and the wrapper attests to itself when asked for a type handler + - so a document that arrived wrapped would be priced as a string and + charged for quotes that the writing, which looks through the wrapper, + is never going to put round it. is_json_type() is that same look. +*/ +static ulonglong json_value_reserve(Item *arg) +{ + /* + A document is a string: every type handler in the JSON collection is + built over a string one. An argument of any other result type is + not one and its own type has said so, so the look through the + wrappers round it is only made where it can come back yes. + */ + const bool is_string= arg->result_type() == STRING_RESULT; + const bool is_document= is_string && is_json_type(arg); + ulonglong length; + + if (is_string && !is_document) + length= static_cast(arg->max_char_length()) * 2 + 2; + else if (arg->type_handler()->is_bool_type()) + length= 5; + else if (is_document) + length= static_cast(arg->max_char_length()) * 2; + else + length= arg->max_char_length(); + + return length < 4 ? 4 : length; } @@ -2063,26 +3911,18 @@ bool Item_func_json_array::fix_length_and_dec(THD *thd) return TRUE; for (n_arg=0 ; n_arg < arg_count ; n_arg++) - { - ulonglong arg_length; - Item *arg= args[n_arg]; - - if (arg->result_type() == STRING_RESULT && - !Type_handler_json_common::is_json_type_handler(arg->type_handler())) - arg_length= arg->max_char_length() * 2; /*escaping possible */ - else if (arg->type_handler()->is_bool_type()) - arg_length= 5; - else - arg_length= arg->max_char_length(); - - if (arg_length < 4) - arg_length= 4; /* can be 'null' */ - - char_length+= arg_length + 4; - } + char_length+= json_value_reserve(args[n_arg]) + 4; fix_char_length_ulonglong(char_length); tmp_val.set_charset(collation.collation); + /* + An argument can leave this with no document to return - see + val_str() - and that is decided by what the argument holds rather + than by whether it was NULL, so an argument that is never NULL does + not settle it. The constructor with no arguments at all is the one + that cannot happen to, and it is answered above. + */ + set_maybe_null(); return FALSE; } @@ -2091,18 +3931,47 @@ String *Item_func_json_array::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; + /* + The array written here is one structure of its own, which is where + the values go inside - so that is where the reckoning starts, and an + array with nothing in it is one deep for the brackets alone. + */ + Json_splice_marks marks(1); + + m_marks.clear(); + /* + Said for this evaluation rather than left where the last one put it: + a row refused below would otherwise leave it set for the row after, + which composes a document and would be answered NULL for it. + */ + null_value= 0; str->length(0); str->set_charset(collation.collation); - if (str->append('[') || - ((arg_count > 0) && append_json_value(str, args[0], &tmp_val))) + if (str->append('[')) goto err_return; - for (n_arg=1; n_arg < arg_count; n_arg++) + for (n_arg=0; n_arg < arg_count; n_arg++) { - if (str->append(", ", 2) || - append_json_value(str, args[n_arg], &tmp_val)) + if ((n_arg && str->append(STRING_WITH_LEN(json_loose_comma))) || + append_json_value(str, args[n_arg], &tmp_val, 1, func_name(), + (int) n_arg, marks, false, false, + Sql_condition::WARN_LEVEL_WARN)) + goto err_return; + + /* + A value that is not a document is the whole array not being one, + and there is no other answer to compose, so the arguments after it + are not evaluated. The marks ARE the reading a closing pass would + have done, taken a value at a time as the values go in, so this is + where the question is answered. + + A released server spliced such a value as it stood and returned + bytes no reader will take. The warning naming the argument is + raised where it always was, just above. + */ + if (!marks.is_valid) goto err_return; } @@ -2113,7 +3982,28 @@ String *Item_func_json_array::val_str(String *str) result_limit= current_thd->variables.max_allowed_packet; if (str->length() <= result_limit) + { + /* + Nothing was read back, so what can be said about the array is only + what was learned putting it together: the brackets and separators + are written here in the loose form, the values that were + quoted came out of json_escape() and are documents on their own, + and the values that went in as they stand were either read as they + went or passed by something that had already read them. + Which of the two it was does not matter here - marks.is_valid and + marks.is_nice are cleared by whichever of them did not hold, and + that is the whole of what this has to go on. + + is_valid is settled above, a value at a time, so only the brackets + are left to ask about. + */ + if (json_container_charset_refused(str, func_name())) + goto err_return; + + DBUG_ASSERT(marks.is_valid); + m_marks.set(str, true, marks.is_nice, marks.deepest); return str; + } push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_ALLOWED_PACKET_OVERFLOWED, @@ -2135,13 +4025,18 @@ bool Item_func_json_array_append::fix_length_and_dec(THD *thd) ulonglong char_length; collation.set(args[0]->collation); - char_length= args[0]->max_char_length(); + /* + The document is written out again around what is added to it, a + space arriving after every separator that is copied, so the room for + it has to cover the writing and not only the reading - the same + allowance JSON_REMOVE asks for, adding the same spacing. + */ + char_length= static_cast(args[0]->max_char_length()) * 2; for (n_arg= 1; n_arg < arg_count; n_arg+= 2) { paths[n_arg/2].set_constant_flag(args[n_arg]->const_item()); - char_length+= - static_cast(args[n_arg+1]->max_char_length()) + 4; + char_length+= json_value_reserve(args[n_arg+1]) + 4; } fix_char_length_ulonglong(char_length); @@ -2150,6 +4045,37 @@ bool Item_func_json_array_append::fix_length_and_dec(THD *thd) } +/* + Returns the document in 'from' as this function's answer instead of + reading it again to find out what it is. + + A caller gets here only where everything that went into the answer is + attested to, so what it returns is a document written the loose + way and nothing has to look at it to say so. Which of the two + the answer is already sitting in is the one thing the callers differ + in, so the copy is made only where it is owed. + + The depth is the caller's to work out - it is the only part of this + that depends on what the function did to the document - and 'to' is + what comes back, or NULL where the buffer would not grow. Reporting + that is the caller's as well: they do not all report it in the same + place, some of them having a document engine to complain through and + some not. +*/ + +String *Item_json_func::return_json(String *to, const String *from, + uint depth) +{ + if ((from != to && to->copy(from->ptr(), from->length(), from->charset())) || + DBUG_IF("json_return_out_of_memory")) + return NULL; + + null_value= 0; + m_marks.set(to, true, true, depth); + return to; +} + + String *Item_func_json_array_append::val_str(String *str) { json_engine_t je; @@ -2157,9 +4083,26 @@ String *Item_func_json_array_append::val_str(String *str) uint n_arg, n_path; size_t str_rest_len; const uchar *ar_end; + const char *js_end; THD *thd; + String *const to= str; + bool compose_final; + uint depth; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; + Json_source_watch watch; + /* + Cleared by anything spliced in that leaves is_valid or is_nice + false for the answer, the depth a path reaches among it. See + Item_func_json_insert::val_str() for what the fourth of them + starts at and why. + */ + Json_splice_marks splice(0); DBUG_ASSERT(fixed()); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; @@ -2167,10 +4110,27 @@ String *Item_func_json_array_append::val_str(String *str) thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, for the reason given where + Item_func_json_insert::val_str() asks it. + */ + compose_final= document_arg_composes_final(args[0], js); + /* Taken here for the reason given at the same place there. */ + js_depth= args[0]->last_depth(); + for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { int array_counters[JSON_DEPTH_LIMIT]= {0}; json_path_with_flags *c_path= paths + n_path; + + /* + Taken before the path is worked out, and afresh every time round. + See Item_func_json_insert::val_str(). + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -2189,10 +4149,18 @@ String *Item_func_json_array_append::val_str(String *str) if (args[n_arg]->null_value) goto return_null; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; + /* + Where the document ends, read where the walk over it starts rather + than after the value has been worked out, so that the end copied + from is the end that was walked to. See Json_source_watch. + */ + js_end= js->end(); + c_path->cur_step= c_path->p.steps; if (json_find_path(&je, &c_path->p, &c_path->cur_step, array_counters)) @@ -2206,6 +4174,20 @@ String *Item_func_json_array_append::val_str(String *str) if (json_read_value(&je)) goto js_error; + /* + How many structures the value will sit inside once it is in + place. Read off here because walking to the end of the array + below pops the scanner back out of it. + + Reading a container puts it on the stack; reading a scalar does + not. So for an array, and for an object about to be wrapped in a + new one, the scanner is already counting the structure the value + is going into, and counting it again would refuse a document that + fits. Only a scalar being wrapped needs the one added, the new + array being a structure the scanner has never seen. + */ + depth= (uint) je.stack_p + (json_value_scalar(&je) ? 1 : 0); + str->length(0); str->set_charset(js->charset()); if (str->reserve(js->length() + 8, 1024)) @@ -2222,7 +4204,9 @@ String *Item_func_json_array_append::val_str(String *str) str->q_append(js->ptr(), ar_end-(const uchar *) js->ptr()); if (n_items) str->append(", ", 2); - if (append_json_value(str, args[n_arg+1], &tmp_val)) + if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, splice, + !compose_final, compose_final)) goto return_null; /* Out of memory. */ if (str->reserve(str_rest_len, 1024)) @@ -2239,6 +4223,13 @@ String *Item_func_json_array_append::val_str(String *str) if (je.value_type == JSON_VALUE_OBJECT) { + /* + Wrapping puts what was already there inside a new array, so + the KEPT value goes down a level too - and how far down it + already reached was never measured, it having been copied + rather than read. Say so and let the answer be read back. + */ + splice.is_valid= splice.is_nice= false; if (json_skip_level(&je)) goto js_error; c_to= je.s.c_str; @@ -2246,15 +4237,27 @@ String *Item_func_json_array_append::val_str(String *str) else c_to= je.value_end; + /* + The brackets and the comma are written, and go through a String + that converts them; the value and the rest of the document are + copied, and must not. They are already in the character set + being built in - they came out of the document, which is what + the result is being made from - so converting them writes them + a second time. The arm above copies its two pieces of document + the same way, with q_append(). + */ if (str->append('[') || - str->append((const char *) c_from, c_to - c_from) || + append_simple(str, c_from, c_to - c_from) || str->append(", ", 2) || - append_json_value(str, args[n_arg+1], &tmp_val) || + append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, splice, + !compose_final, compose_final) || str->append(']') || - str->append((const char *) je.s.c_str, - js->end() - (const char *) je.s.c_str)) + append_simple(str, je.s.c_str, + js_end - (const char *) je.s.c_str)) goto return_null; /* Out of memory. */ } + DBUG_ASSERT(watch.unchanged(js)); { /* Swap str and js. */ if (str == &tmp_js) @@ -2270,12 +4273,36 @@ String *Item_func_json_array_append::val_str(String *str) } } + /* + A document that was already written the loose way, with a value + written that way put inside it, is written that way already. See + Item_func_json_insert::val_str() for why the answer is in js, and + for what makes this safe to believe - and for how deep it goes, + which is worked out the same way here. Wrapping puts a new array + where the value it holds already sat, and the value going in beside + it is counted from inside that array, so the new level is counted + with it. + */ + if (compose_final && splice.is_valid && splice.is_nice) + { + if (!return_json(to, js, MY_MAX(js_depth, splice.deepest))) + goto return_null; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; + /* + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. + */ + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -2292,9 +4319,26 @@ String *Item_func_json_array_insert::val_str(String *str) json_engine_t je; String *js= args[0]->val_json(&tmp_js); uint n_arg, n_path; + const char *js_end; THD *thd; + String *const to= str; + bool compose_final; + uint depth; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; + Json_source_watch watch; + /* + Cleared by anything spliced in that leaves is_valid or is_nice + false for the answer, the depth a path reaches among it. See + Item_func_json_insert::val_str() for what the fourth of them + starts at and why. + */ + Json_splice_marks splice(0); DBUG_ASSERT(fixed()); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; @@ -2302,6 +4346,16 @@ String *Item_func_json_array_insert::val_str(String *str) thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, for the reason given where + Item_func_json_insert::val_str() asks it. + */ + compose_final= document_arg_composes_final(args[0], js); + /* Taken here for the reason given at the same place there. */ + js_depth= args[0]->last_depth(); + for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { int array_counters[JSON_DEPTH_LIMIT]= {0}; @@ -2309,6 +4363,12 @@ String *Item_func_json_array_insert::val_str(String *str) const char *item_pos; int n_item, corrected_n_item; + /* + Taken before the path is worked out, and afresh every time round. + See Item_func_json_insert::val_str(). + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -2325,7 +4385,7 @@ String *Item_func_json_array_insert::val_str(String *str) c_path->p.last_step->type != JSON_PATH_ARRAY) { if (c_path->p.s.error == 0) - c_path->p.s.error= SHOULD_END_WITH_ARRAY; + json_error(&c_path->p.s, SHOULD_END_WITH_ARRAY); path_err: report_path_error(s_p, &c_path->p, n_arg); @@ -2337,10 +4397,17 @@ String *Item_func_json_array_insert::val_str(String *str) if (args[n_arg]->null_value) goto return_null; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; + /* + Where the document ends, read where the walk over it starts. See + Json_source_watch. + */ + js_end= js->end(); + c_path->cur_step= c_path->p.steps; if (json_find_path(&je, &c_path->p, &c_path->cur_step, array_counters)) @@ -2361,6 +4428,15 @@ String *Item_func_json_array_insert::val_str(String *str) continue; } + /* + How many structures the value will sit inside once it is in + place: the ones holding the array, and the array itself. Reading + the array has already put it on the stack, so the scanner is + counting it; read off here, before the walk below moves the + scanner about inside it. + */ + depth= (uint) je.stack_p; + item_pos= 0; n_item= 0; corrected_n_item= c_path->p.last_step[1].n_item; @@ -2406,7 +4482,9 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 1); goto return_null; /* Out of memory. */ } - if (append_json_value(str, args[n_arg+1], &tmp_val)) + if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, splice, + !compose_final, compose_final)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ @@ -2421,7 +4499,7 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 1); goto return_null; /* Out of memory. */ } - size= js->end() - item_pos; + size= js_end - item_pos; if (append_simple(str, item_pos, size)) { my_error(ER_OUTOFMEMORY, MYF(0), (int) size); @@ -2445,12 +4523,14 @@ String *Item_func_json_array_insert::val_str(String *str) my_error(ER_OUTOFMEMORY, MYF(0), 2); goto return_null; /* Out of memory. */ } - if (append_json_value(str, args[n_arg+1], &tmp_val)) + if (append_json_value(str, args[n_arg+1], &tmp_val, depth, func_name(), + (int) n_arg + 1, splice, + !compose_final, compose_final)) { my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); goto return_null; /* Out of memory. */ } - size= js->end() - item_pos; + size= js_end - item_pos; if (append_simple(str, item_pos, size)) { my_error(ER_OUTOFMEMORY, MYF(0), (int) size); @@ -2458,6 +4538,7 @@ String *Item_func_json_array_insert::val_str(String *str) } } + DBUG_ASSERT(watch.unchanged(js)); { /* Swap str and js. */ if (str == &tmp_js) @@ -2473,12 +4554,36 @@ String *Item_func_json_array_insert::val_str(String *str) } } + /* + A document that was already written the loose way, with a value + written that way put inside it, is written that way already. See + Item_func_json_insert::val_str() for why the answer is in js, and + for what makes this safe to believe - and for how deep it goes, + which is worked out the same way here. Wrapping puts a new array + where the value it holds already sat, and the value going in beside + it is counted from inside that array, so the new level is counted + with it. + */ + if (compose_final && splice.is_valid && splice.is_nice) + { + if (!return_json(to, js, MY_MAX(js_depth, splice.deepest))) + goto return_null; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; + /* + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. + */ + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -2494,21 +4599,40 @@ String *Item_func_json_object::val_str(String *str) { DBUG_ASSERT(fixed()); uint n_arg; + /* One deep for the braces alone - see Item_func_json_array::val_str(). */ + Json_splice_marks marks(1); + + m_marks.clear(); + /* Said per evaluation, as in the sister constructor. */ + null_value= 0; str->length(0); str->set_charset(collation.collation); - if (str->append('{') || - (arg_count > 0 && - (append_json_keyname(str, args[0], &tmp_val) || - append_json_value(str, args[1], &tmp_val)))) + if (str->append('{')) goto err_return; - for (n_arg=2; n_arg < arg_count; n_arg+=2) + /* + The first pair used to be written outside this loop, the separator + being what set it apart. It is written here instead so that the + refusal below is written once for every pair rather than twice. + */ + for (n_arg=0; n_arg < arg_count; n_arg+=2) { - if (str->append(", ", 2) || - append_json_keyname(str, args[n_arg], &tmp_val) || - append_json_value(str, args[n_arg+1], &tmp_val)) + if ((n_arg && str->append(STRING_WITH_LEN(json_loose_comma))) || + append_json_keyname(str, args[n_arg], &tmp_val, func_name(), + (int) n_arg) || + append_json_value(str, args[n_arg+1], &tmp_val, 1, func_name(), + (int) n_arg + 1, marks, false, false, + Sql_condition::WARN_LEVEL_WARN)) + goto err_return; + + /* + As in the sister constructor: one value that is not a document is + the whole object not being one, so the arguments after it are not + evaluated. + */ + if (!marks.is_valid) goto err_return; } @@ -2519,7 +4643,15 @@ String *Item_func_json_object::val_str(String *str) result_limit= current_thd->variables.max_allowed_packet; if (str->length() <= result_limit) + { + /* As in the sister constructor above, and so is the refusal. */ + if (json_container_charset_refused(str, func_name())) + goto err_return; + + DBUG_ASSERT(marks.is_valid); + m_marks.set(str, true, marks.is_nice, marks.deepest); return str; + } push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_ALLOWED_PACKET_OVERFLOWED, @@ -2533,7 +4665,15 @@ String *Item_func_json_object::val_str(String *str) } -static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) +/* + 'wrapped' is raised where the merging puts what it was given inside a + new array. That array is a level of its own, and how deep either + document already went was never measured - they were copied, not read. + Whoever is composing an answer out of this therefore cannot say how + deep the answer is, and has to have it read back rather than guess. +*/ +static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2, + bool *wrapped, uint colon_len) { DBUG_EXECUTE_IF("json_check_min_stack_requirement", return dbug_json_check_min_stack_requirement();); @@ -2580,6 +4720,22 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) *je2= sav_je2; } + /* + Written the compact way on purpose, but only half of what + follows earns it. A key only this document holds is copied out + of it below from where the space after the colon stands, so + writing one here would write it twice. A key both documents + hold is composed instead, and a composed value begins at the + value itself - so that arm writes the space it needs before it + starts, and it is the only one that has to. + + It writes one only where the answer is going out as it is + composed, which is what colon_len says. Where the answer is to + be read back, the reading writes the whole of it anyway, and + writing a space here would only move every offset a released + server reports - including the ones it reports about documents + it then refuses. + */ if (str->append('"') || append_simple(str, key_start, key_end - key_start) || str->append("\":", 2)) @@ -2599,7 +4755,9 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) } /* Json_2 has same key as Json_1. Merge them. */ - if ((ires= do_merge(str, je1, je2))) + if (colon_len == 3 && str->append(' ')) + return 3; + if ((ires= do_merge(str, je1, je2, wrapped, colon_len))) return ires; goto merged_j1; } @@ -2617,6 +4775,26 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) continue; } + /* + The loop above ends on a refusal exactly as it ends on the object's + end, its condition asking only whether the scanner moved. An + argument that broke before its first value was complete would + otherwise be merged from as far as it got, and what came out would + be an object no argument held: composed, well formed, and missing + whatever stood past the break. Nothing downstream has any reason + to reject such a value, which is what makes it the quiet kind of + wrong. + + Refusing it is what these functions already do for the same + breakage met one loop further in, and what every other JSON + function does with the same characters. Every loop of this shape + was letting it through, here and in the two functions' other + argument, and once more where an argument is copied whole instead + of being merged with anything. + */ + if (unlikely(je1->s.error)) + return 1; + *je2= sav_je2; /* Now loop through the Json_2 keys. @@ -2672,6 +4850,10 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) continue; } + /* The other loop, ending the same way - see above. */ + if (unlikely(je2->s.error)) + return 1; + if (str->append('}')) return 3; } @@ -2692,6 +4874,8 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) } else { + /* A level of its own - see where this is declared. */ + *wrapped= true; if (str->append('[')) return 3; if (je1->value_type == JSON_VALUE_OBJECT) @@ -2704,7 +4888,16 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) end1= je1->value_end; } - if (str->append((const char*) beg1, end1 - beg1)) + /* + Copied, not written: these bytes came out of a document and are + already in the character set the answer is being built in. The + appends that convert are for punctuation written here, which + arrives as plain ASCII and has to be turned into the set; putting + a copied span through one of those writes it a second time, and in + a set that spends more than one byte on a character what comes out + is not a document at all. + */ + if (append_simple(str, beg1, end1 - beg1)) return 3; if (json_value_scalar(je2)) @@ -2729,8 +4922,9 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) end2= je2->s.c_str; } + /* The comma is written and converts; the span is copied and must not. */ if ((n_items1 && n_items2 && str->append(", ", 2)) || - str->append((const char*) beg2, end2 - beg2)) + append_simple(str, beg2, end2 - beg2)) return 3; if (je2->value_type != JSON_VALUE_ARRAY && @@ -2742,6 +4936,76 @@ static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2) } +/* + How deep the deepest of the document arguments goes, for the two + functions whose answer is composed out of nothing else. Taken one + argument at a time, beside where that argument is evaluated. + + What an argument answers is about the value it has just passed, + and by the end of the loop it need not be about that value any more. + An argument that reads a variable answers out of what is in the + variable now, so a later argument that assigns to the same variable + moves the first one's answer out from under the bytes that went into + the merge - and a document written over by a shallower one then + answers a depth smaller than the answer really goes, which is the one + direction a depth must never be wrong in. The other two marks are + already taken this way; this is the third of them, and was the only + one left to the end. + + One argument that will not say makes the answer not say either: what + is being asked is how deep the deepest part of the result is, and a + part nothing measured could be anywhere. +*/ +static uint deepest_document_argument(uint deepest, Item *arg) +{ + uint arg_depth= arg->last_depth(); + + if (deepest == JSON_DEPTH_UNKNOWN || arg_depth == JSON_DEPTH_UNKNOWN) + return JSON_DEPTH_UNKNOWN; + return MY_MAX(deepest, arg_depth); +} + + +/* + Says what is wrong with a document argument that was read only as far + as its first value, when anything is. + + These two functions read a document argument no further than that. + What they compose comes out of that value and nothing else, so neither + text standing after it nor a break the reading never got as far as + reaches the answer, and neither has ever stopped an answer being + given. What is there is still not a document, and every other JSON + function says so about the very same characters. + + Saying so here takes nothing away, which is why it is a note. A + warning becomes an error under a strict mode, and that would take away + an answer that has always been given back. An argument the merging + broke off inside arrives here already refused, and is reported from + where it was refused: an engine that has been refused does not move + again, so carrying the reading on cannot relocate the complaint. + + The reading is carried on from where the merging left off rather than + started again, so an argument with nothing after its value pays one + step and no more. It is carried on over a copy, because a reading + done to find out whether to say something must not be the reason + anything else is said: the caller reports its own failures out of its + own engine, and this must leave that engine where it found it. +*/ +static void report_json_trailing_note(const json_engine_t *je, + const String *js, + const char *fname, int n_param) +{ + json_engine_t tail= *je; + + while (json_scan_next(&tail) == 0) + /* There is nothing left to compose from, only to read. */; + + if (tail.s.error) + report_json_error_ex(js->ptr(), &tail, fname, n_param, + Sql_condition::WARN_LEVEL_NOTE); +} + + String *Item_func_json_merge::val_str(String *str) { DBUG_ASSERT(fixed()); @@ -2749,6 +5013,22 @@ String *Item_func_json_merge::val_str(String *str) String *js1= args[0]->val_json(&tmp_js1), *js2=NULL; uint n_arg; THD *thd; + String *const to= str; + bool compose_final; + /* + The formatting of the punctuation written here, settled from the first + document alone and before anything is composed - see below. + */ + uint colon_len; + /* Raised where the merging puts what it was given inside a new array. */ + bool wrapped= false; + /* How deep the deepest document argument goes, taken as each is read. */ + uint deepest= 0; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; + Json_source_watch watch; + + m_marks.clear(); if (args[0]->null_value) goto null_return; @@ -2756,15 +5036,46 @@ String *Item_func_json_merge::val_str(String *str) thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + /* + Whether what is composed here is what will be returned. Every + document merged in has a say, and they are only read one at a time, + so the answer is settled over the loop. The FORMATTING cannot wait + for that - what is written early is written before the later + arguments have been looked at - so it follows the first document + alone, which is known here. A later argument answering is_valid or + is_nice false then sends the answer through the reading back after + all, and the + positions it reports are offsets into text this function composed + rather than into anything a caller wrote. + */ + compose_final= document_arg_composes_final(args[0], js1); + deepest= deepest_document_argument(deepest, args[0]); + colon_len= compose_final ? 3 : 2; + for (n_arg=1; n_arg < arg_count; n_arg++) { str->set_charset(js1->charset()); str->length(0); + /* + The document on the left is not read until below, and working out + the one on the right can be any expression at all. See + Json_source_watch. + */ + watch.take(js1); js2= args[n_arg]->val_json(&tmp_js2); + DBUG_ASSERT(watch.unchanged(js1)); if (args[n_arg]->null_value) goto null_return; + /* + Asked after the argument has been evaluated, what it answers being + about the value it has just passed. + */ + if (!args[n_arg]->is_valid_json() || !args[n_arg]->is_nice_json()) + compose_final= false; + deepest= deepest_document_argument(deepest, args[n_arg]); + json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; @@ -2773,9 +5084,29 @@ String *Item_func_json_merge::val_str(String *str) (const uchar *) js2->ptr() + js2->length()); je2.killed_ptr= (uint32_t *) &thd->killed; - if (do_merge(str, &je1, &je2)) + if (do_merge(str, &je1, &je2, &wrapped, colon_len)) goto error_return; + /* + Putting the two inside a new array makes the answer a level deeper + than either of them, and how deep either of them went was never + measured. Have it read back, the same reading a released server + always did, which is also what puts the complaint in the place + that server put it. + */ + if (wrapped) + compose_final= false; + + /* + Both were read as far as their first value and no further. Only + the first turn of the loop has a document argument on the left: + after it, the left is what this function itself composed, and + nothing stands after that. + */ + if (n_arg == 1) + report_json_trailing_note(&je1, js1, func_name(), 0); + report_json_trailing_note(&je2, js2, func_name(), (int) n_arg); + { /* Swap str and js1. */ if (str == &tmp_js1) @@ -2791,14 +5122,53 @@ String *Item_func_json_merge::val_str(String *str) } } + /* + Documents that were all written the loose way, joined with + punctuation written the same way, make a document written that way. + See Item_func_json_insert::val_str() for why the answer is in js1, + and for what makes this safe to believe. + + Nothing is spliced in here - every piece of the answer is a document + argument, and each of them has already had its say in compose_final + as it came round the loop - so there is no set of splice marks to + ask, and the one condition is the whole of it. + + Which is also what says how deep the answer goes, ONE LEVEL OUT. + Merging two arrays lays their members side by side and merging two + objects puts their keys together, and neither of those moves + anything further inside than it was. But merging an array with + anything else makes that other thing a MEMBER of the array, and a + member sits one level inside - so the deepest of the arguments is + not enough on its own, which the reading back said the first time + this was written without the level. + + Which of the two happened is not worth telling apart here: a level + more than the answer needs turns down a splice that could have been + taken, and costs a reading nobody is owed an answer about. + */ + if (compose_final) + { + if (!return_json(to, js1, + deepest == JSON_DEPTH_UNKNOWN ? deepest : deepest + 1)) + goto error_return; + + return to; + } + json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je1, str, Item_func_json_format::LOOSE)) + if (json_nice(&je1, str, Item_func_json_format::LOOSE, &read_back_depth)) goto error_return; null_value= 0; + /* + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. + */ + m_marks.set(str, true, true, read_back_depth); return str; error_return: @@ -2866,6 +5236,17 @@ static int copy_value_patch(String *str, json_engine_t *je) copy_value_patch(str, je)) return 1; } + + /* + The same loop end again, in the arm taken when there is nothing to + merge with and the argument is copied whole. Everything said over + the walk that merges two objects together holds here word for word, + and this loop is reached for every object nested inside the value as + well as for the value itself. + */ + if (unlikely(je->s.error)) + return 1; + if (str->append('}')) return 1; @@ -2874,7 +5255,7 @@ static int copy_value_patch(String *str, json_engine_t *je) static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, - bool *empty_result) + bool *empty_result, uint colon_len) { DBUG_EXECUTE_IF("json_check_min_stack_requirement", return dbug_json_check_min_stack_requirement();); @@ -2927,7 +5308,7 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, if (str->append('"') || append_simple(str, key_start, key_end - key_start) || - str->append("\":", 2)) + str->append(json_loose_colon, colon_len)) return 3; while (json_scan_next(je2) == 0 && @@ -2944,7 +5325,7 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, } /* Json_2 has same key as Json_1. Merge them. */ - if ((ires= do_merge_patch(str, je1, je2, &mrg_empty))) + if ((ires= do_merge_patch(str, je1, je2, &mrg_empty, colon_len))) return ires; if (mrg_empty) @@ -2962,6 +5343,15 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, /* Just append the Json_1 key value. */ if (json_skip_key(je1)) return 1; + /* + This span begins where the colon left off, and in a document + written the loose way that is the space after it - the space + just written above. The other way out of this loop has the + value written afresh, with no space of its own, which is why + the colon writes one at all. + */ + if (colon_len == 3) + key_start= json_skip_space(je1->s.cs, key_start, je1->s.c_str); if (append_simple(str, key_start, je1->s.c_str - key_start)) return 3; first_key= 0; @@ -2970,6 +5360,13 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, continue; } + /* + Ends on a refusal as well as on the object's end, exactly as the + loops in do_merge() do, and refused here for the same reason. + */ + if (unlikely(je1->s.error)) + return 1; + *je2= sav_je2; /* Now loop through the Json_2 keys. @@ -3018,7 +5415,7 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, if (str->append('"') || append_simple(str, key_start, key_end - key_start) || - str->append("\":", 2)) + str->append(json_loose_colon, colon_len)) return 3; if (json_read_value(je2)) @@ -3037,6 +5434,10 @@ static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2, continue; } + /* The other loop, ending the same way - see above. */ + if (unlikely(je2->s.error)) + return 1; + if (str->append('}')) return 3; } @@ -3062,6 +5463,25 @@ String *Item_func_json_merge_patch::val_str(String *str) uint n_arg; bool empty_result, merge_to_null; THD *thd= current_thd; + String *const to= str; + bool compose_final; + uint colon_len; + Json_source_watch watch; + /* How deep the deepest document argument goes, taken as each is read. */ + uint deepest= 0; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; + /* + Cleared by anything that goes into the answer that the answer cannot + be said to hold: a document taken over without having been read on + the way in, and a document whose item attests is_nice false. Nothing is + spliced into a patched document - every piece of the answer is a + document argument - so the fourth of them is never moved off nothing + here. + */ + Json_splice_marks splice(0); + + m_marks.clear(); JSON_DO_PAUSE_EXECUTION(thd, 0.0002); @@ -3069,15 +5489,55 @@ String *Item_func_json_merge_patch::val_str(String *str) je1.s.error= je2.s.error= 0; merge_to_null= args[0]->null_value; + /* + The formatting follows the first document alone, for the reason given + where Item_func_json_merge::val_str() picks its own. A first + argument that is SQL NULL contributes nothing to the answer and is + not asked, and leaves nothing to read the character set off either. + */ + compose_final= !merge_to_null && document_arg_composes_final(args[0], js1); + colon_len= compose_final ? 3 : 2; + deepest= deepest_document_argument(deepest, args[0]); + if (!compose_final) + splice.is_nice= false; + for (n_arg=1; n_arg < arg_count; n_arg++) { + /* + The document on the left is not read until below, and working out + the one on the right can be any expression at all. See + Json_source_watch. A left-hand side that is SQL NULL has no bytes + to keep an eye on. + */ + if (!merge_to_null) + watch.take(js1); js2= args[n_arg]->val_json(&tmp_js2); + DBUG_ASSERT(merge_to_null || watch.unchanged(js1)); + /* + Taken before the arms below part, an argument that is dropped + being asked here just as one that goes in is. A depth folded in + from an argument the answer does not hold is a depth larger than + the answer needs, which is the side a depth is allowed to be + wrong on; asking only the ones that stay would make the reading + turn on which arm was taken, and the arm is settled by later + arguments. + */ + deepest= deepest_document_argument(deepest, args[n_arg]); if (args[n_arg]->null_value) { merge_to_null= true; goto cont_point; } + /* + Asked after the argument has been evaluated, what it answers being + about the value it has just passed. A document merged in is + read as it is copied, so nothing here is about whether it is a + document - only about how it is written. + */ + if (!args[n_arg]->is_nice_json()) + splice.is_nice= false; + json_scan_start(&je2, js2->charset(),(const uchar *) js2->ptr(), (const uchar *) js2->ptr() + js2->length()); je2.killed_ptr= (uint32_t *) &thd->killed; @@ -3086,11 +5546,65 @@ String *Item_func_json_merge_patch::val_str(String *str) { if (json_read_value(&je2)) goto error_return; + + /* + Said before it is settled whether the argument goes into the + answer at all. An object merged onto SQL NULL contributes + nothing and is dropped just below, and being dropped makes what + stands after it no more a document than it was; the very same + characters are spoken for in every other argument position. + + Only the first value was read, so getting to the end of it comes + first here, where the merging below does it for itself. A value + that cannot be got to the end of is one the merging will report + on in its own words; there is nothing to add to that. + + Not asked at all of a value already attested to. + Being a document is a statement about the whole of it - a value + with anything standing after it is not one - so the walk could + only ever come back saying what is already known, and it is the + walk this whole exercise is about not making. It is the one the + released server did not make either: it took the first value and + copied the rest in unread. + */ + if (!args[n_arg]->is_valid_json()) + { + json_engine_t adopted= je2; + + if (json_value_scalar(&adopted) || !json_skip_level(&adopted)) + report_json_trailing_note(&adopted, js2, func_name(), (int) n_arg); + } + if (je2.value_type == JSON_VALUE_OBJECT) goto cont_point; merge_to_null= false; - str->set(js2->ptr(), js2->length(), js2->charset()); + /* + Taken over by copying it rather than by pointing at it. The next + argument is read into the very buffer this one came in, and a + buffer that has to grow to hold it is not the buffer it was: what + was pointed at is gone by the time it comes to be read. + */ + if (str->copy(js2->ptr(), js2->length(), js2->charset())) + goto error_return; + + /* + The whole of it goes in and only its first value was read, so + whatever stands after that value goes in unread. Text standing + there is no reason to refuse the document: what comes of it is + a matter for whatever the answer turns out to be, and a later + argument merging over this one leaves it out of the answer + altogether, which is an answer that has always been given back. + + A document whose item attests is_valid has nothing against it + to begin with, and goes in the way it was written. Anything + else has to be read back, so say here that it has to be. + */ + if (!args[n_arg]->is_valid_json()) + splice.is_valid= splice.is_nice= false; + else if (!args[n_arg]->is_nice_json()) + splice.is_nice= false; + goto cont_point; } @@ -3102,9 +5616,19 @@ String *Item_func_json_merge_patch::val_str(String *str) (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; - if (do_merge_patch(str, &je1, &je2, &empty_result)) + if (do_merge_patch(str, &je1, &je2, &empty_result, colon_len)) goto error_return; + /* + Both were read as far as their first value and no further. Only + the first turn of the loop has a document argument on the left: + after it, the left is what this function itself composed, or a + document taken over, which was spoken for where it was taken. + */ + if (n_arg == 1) + report_json_trailing_note(&je1, js1, func_name(), 0); + report_json_trailing_note(&je2, js2, func_name(), (int) n_arg); + if (empty_result) str->append(STRING_WITH_LEN("null")); @@ -3127,13 +5651,49 @@ String *Item_func_json_merge_patch::val_str(String *str) if (merge_to_null) goto null_return; + /* + Documents that were all written the loose way, patched with + punctuation written the same way, make a document written that way. + See Item_func_json_insert::val_str() for why the answer is in js1, + and for what makes this safe to believe. + + Asked of the splice marks alone, where the others ask compose_final + too, and it comes to the same thing: what the others keep in that + variable this one keeps in colon_len, which is the wider formatting + only where the first document answers is_valid and is_nice and can + be written - + and anything else clears is_nice on the spot. A document taken over + whole clears both marks where it was taken. + + How deep it goes is read off the arguments, and WITHOUT the extra + level Item_func_json_merge::val_str() takes. Patching is not + merging: the two documents are walked in step, a key of one going + among the keys of the other at the level both were at, and a value + replacing a value where that value stood. Nothing is ever made a + member of anything, which is the case that costs the level there, + and a key the patch drops only takes levels away. + */ + if (splice.is_valid && splice.is_nice) + { + if (!return_json(to, js1, deepest)) + goto error_return; + + return to; + } + json_scan_start(&je1, js1->charset(),(const uchar *) js1->ptr(), (const uchar *) js1->ptr() + js1->length()); je1.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je1, str, Item_func_json_format::LOOSE)) + if (json_nice(&je1, str, Item_func_json_format::LOOSE, &read_back_depth)) goto error_return; null_value= 0; + /* + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. + */ + m_marks.set(str, true, true, read_back_depth); return str; error_return: @@ -3159,8 +5719,9 @@ bool Item_func_json_length::fix_length_and_dec(THD *thd) longlong Item_func_json_length::val_int() { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; + Json_source_watch watch; uint length= 0; int array_counters[JSON_DEPTH_LIMIT]= {0}; int err; @@ -3172,6 +5733,7 @@ longlong Item_func_json_length::val_int() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -3198,6 +5760,7 @@ longlong Item_func_json_length::val_int() goto null_return; path.cur_step= path.p.steps; + DBUG_ASSERT(watch.unchanged(js)); if (json_find_path(&je, &path.p, &path.cur_step, array_counters)) { if (je.s.error) @@ -3232,9 +5795,13 @@ longlong Item_func_json_length::val_int() }; } - if (!err) + /* + The rest of the document is parsed only to check that it is one. + Not done at all where the item has already attested that the value + is_valid: the parse could only report what is already known. + */ + if (!err && !args[0]->is_valid_json()) { - /* Parse to the end of the JSON just to check it's valid. */ while (json_scan_next(&je) == 0) {} } @@ -3251,7 +5818,7 @@ longlong Item_func_json_length::val_int() longlong Item_func_json_depth::val_int() { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; uint depth= 0, c_depth= 0; bool inc_depth= TRUE; @@ -3316,17 +5883,38 @@ bool Item_func_json_type::fix_length_and_dec(THD *thd) String *Item_func_json_type::val_str(String *str) { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; const char *type; + THD *thd; if ((null_value= args[0]->null_value)) return 0; + thd= current_thd; + je.killed_ptr= (uint32_t *) &thd->killed; + + /* + Returns the type of the document's first value, or NULL if the + document does not parse all the way through. Parsing it through + comes first, then: a type worked out for text that turns out not to + be a document is a type that is never returned. + + Not done at all where the item has already attested that the value + is_valid. The parse could only report what is already known, and it + is the parse this whole change is about not making. + + What follows it is not that parse repeated. json_read_value() stops + at the bracket that opens a container - see read_obj() - so for any + document but a bare scalar it is one step and not a second walk. + */ + if (!args[0]->is_valid_json() && + !json_valid_engine(&je, js->ptr(), js->length(), js->charset())) + goto error; json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); - je.killed_ptr= (uint32_t *) ¤t_thd->killed; + je.killed_ptr= (uint32_t *) &thd->killed; if (json_read_value(&je)) goto error; @@ -3354,12 +5942,6 @@ String *Item_func_json_type::val_str(String *str) break; } - /* ensure the json is at least valid. */ - while(json_scan_next(&je) == 0) {} - - if (je.s.error) - goto error; - str->set(type, strlen(type), &my_charset_utf8mb3_general_ci); return str; @@ -3378,7 +5960,13 @@ bool Item_func_json_insert::fix_length_and_dec(THD *thd) JSON_DO_PAUSE_EXECUTION(thd, 0.0002); collation.set(args[0]->collation); - char_length= args[0]->max_char_length(); + /* + The document is written out again around what is put into it, a + space arriving after every separator that is copied, so the room for + it has to cover the writing and not only the reading - the same + allowance JSON_REMOVE asks for, adding the same spacing. + */ + char_length= static_cast(args[0]->max_char_length()) * 2; for (n_arg= 1; n_arg < arg_count; n_arg+= 2) { @@ -3387,8 +5975,8 @@ bool Item_func_json_insert::fix_length_and_dec(THD *thd) In the resulting JSON we can insert the property name from the path, and the value itself. */ - char_length+= args[n_arg/2]->max_char_length() + 6; - char_length+= args[n_arg/2+1]->max_char_length() + 4; + char_length+= static_cast(args[n_arg]->max_char_length()) + 6; + char_length+= json_value_reserve(args[n_arg+1]) + 4; } fix_char_length_ulonglong(char_length); @@ -3403,19 +5991,71 @@ String *Item_func_json_insert::val_str(String *str) String *js= args[0]->val_json(&tmp_js); uint n_arg, n_path; json_string_t key_name; + StringBuffer tmp_key; + const char *js_end; THD *thd; + String *const to= str; + bool compose_final; + uint colon_len; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; + Json_source_watch watch; + /* + Cleared by anything spliced in that leaves is_valid or is_nice false + for the answer, the depth a path reaches among it. The deepest a + spliced value ends up starts at nothing: this writes no structure of + its own, so where the answer is deepest is either somewhere a value + went in or somewhere the document already was. + */ + Json_splice_marks splice(0); DBUG_ASSERT(fixed()); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, because the composing has to know it: + an answer that is going to be written out again must be composed + exactly as it always was, positions reported against it being + offsets into it. A document whose item attests neither is_valid nor + is_nice is such an answer, and it is the one a caller wrote out by + hand. + + A character set that cannot encode the punctuation written here is + the other one - see is_json_compatible_charset(). Nothing composed in + such a set is a document, however sound the pieces were, and until + the reading back went away it was the only thing that ever noticed. + So it is kept exactly where it was. + */ + compose_final= document_arg_composes_final(args[0], js); + colon_len= compose_final ? 3 : 2; + /* + And how deep the document goes, taken here rather than where it is + used, which is after every other argument has been worked out. + + THIS IS THE CANONICAL SITE for this too. The three answers are + about one value and have to be taken over one moment: an argument + can be any expression a caller cares to write, a stored function + among them, and one of those can assign the very thing args[0] reads + - a stored program's variable, say - between the two readings. The + document in hand is still the one this composed from, so a depth + read afterwards can be the depth of a value this answer is not + about, and too small is the one direction a depth must never be + wrong in. + */ + js_depth= args[0]->last_depth(); + thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); str->set_charset(collation.collation); tmp_js.set_charset(collation.collation); - json_string_set_cs(&key_name, collation.collation); for (n_arg=1, n_path=0; n_arg < arg_count; n_arg+=2, n_path++) { @@ -3425,6 +6065,18 @@ String *Item_func_json_insert::val_str(String *str) json_path_step_t *lp; int corrected_n_item; + /* + Taken before the path is worked out. A path is any expression a + caller cares to write, and it is the first thing here that can + reach the document; once the walk below has started, its pointers + into the document are live and there is nothing left to catch. + + Taken afresh every time round, too: the end of this loop swaps + what was composed here into js, so what the document IS changes + between one path and the next, on purpose. + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -3449,10 +6101,17 @@ String *Item_func_json_insert::val_str(String *str) if (args[n_arg]->null_value) goto return_null; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; + /* + Where the document ends, read where the walk over it starts. See + Json_source_watch. + */ + js_end= js->end(); + if (c_path->p.last_step < c_path->p.steps) goto v_found; @@ -3506,6 +6165,16 @@ String *Item_func_json_insert::val_str(String *str) if (je.value_type == JSON_VALUE_OBJECT) { + /* + Wrapping puts what was already there inside a new array, so + the KEPT value goes down a level as well as the one being + put in - and how far down the kept one already reached was + never measured, it having been copied rather than read. Say + so and let the answer be read back. A scalar has nothing + inside it and so needs no saying. + */ + if (do_array_autowrap) + splice.is_valid= splice.is_nice= false; if (json_skip_level(&je)) goto js_error; } @@ -3513,9 +6182,12 @@ String *Item_func_json_insert::val_str(String *str) if ((do_array_autowrap && (append_simple(str, v_from, je.s.c_str - v_from) || str->append(", ", 2))) || - append_json_value(str, args[n_arg+1], &tmp_val) || + append_json_value(str, args[n_arg+1], &tmp_val, + (uint) je.stack_p + (do_array_autowrap ? 1 : 0), + func_name(), (int) n_arg + 1, splice, + !compose_final, compose_final) || (do_array_autowrap && str->append(']')) || - append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) + append_simple(str, je.s.c_str, js_end - (const char *) je.s.c_str)) goto js_error; /* Out of memory. */ goto continue_point; @@ -3553,19 +6225,40 @@ String *Item_func_json_insert::val_str(String *str) v_to= (const char *) (je.s.c_str - je.sav_c_len); str->length(0); + /* + One deeper than the scanner says. Getting here means the walk + ran off the end of the array, and an array is taken off the + stack BEFORE the state that says it ended is set - so by now the + array the value is going into is no longer counted, and the + value goes inside it. The arm that finds the place it was + looking for reads the level after skipping back over the value, + with the array still counted, and so needs no such correction. + */ if (append_simple(str, js->ptr(), v_to - js->ptr()) || (n_item > 0 && str->append(", ", 2)) || - append_json_value(str, args[n_arg+1], &tmp_val) || - append_simple(str, v_to, js->end() - v_to)) + append_json_value(str, args[n_arg+1], &tmp_val, + (uint) je.stack_p + 1, + func_name(), (int) n_arg + 1, splice, + !compose_final, compose_final) || + append_simple(str, v_to, js_end - v_to)) goto js_error; /* Out of memory. */ } else /*JSON_PATH_KEY*/ { uint n_key= 0; + bool key_inert= false; if (je.value_type != JSON_VALUE_OBJECT) continue; + /* + The key of the step is written in the character set of the path, + which is not necessarily the one the document is written in. + Comparing it against a key of the document compares characters, + so it has to be read in its own character set. + */ + json_string_set_cs(&key_name, c_path->p.s.cs); + while (json_scan_next(&je) == 0 && je.state != JST_OBJ_END) { switch (je.state) @@ -3594,10 +6287,40 @@ String *Item_func_json_insert::val_str(String *str) if (append_simple(str, js->ptr(), v_to - js->ptr()) || (n_key > 0 && str->append(", ", 2)) || str->append('"') || - append_simple(str, lp->key, lp->key_end - lp->key) || - str->append("\":", 2) || - append_json_value(str, args[n_arg+1], &tmp_val) || - append_simple(str, v_to, js->end() - v_to)) + DBUG_IF("json_insert_key_out_of_memory")) + goto js_error; /* Out of memory. */ + + if (append_json_path_key(str, lp, c_path->p.s.cs, &tmp_key, + func_name(), (int) n_arg + 1, &key_inert) || + DBUG_IF("json_insert_path_key_out_of_memory")) + goto js_error; /* Out of memory. */ + + /* + A key that reaches past the quotes around it can make the answer + anything at all - unreadable, or a different document that reads + perfectly well. Which of the two it is can only be told by + reading the whole of it, so say here that it has to be. + */ + if (!key_inert) + splice.is_valid= splice.is_nice= false; + + /* + Written the loose way when that is how the answer is going out, + like the comma above it and the value after it. A key put in + here is the only punctuation the editing adds of its own, so it + is the only place the two formats can differ. + */ + /* + One deeper than the scanner says, for the reason given at the + end of the array above: the object is taken off the stack before + the state that says it ended, and the new key goes inside it. + */ + if (str->append(json_loose_colon, colon_len) || + append_json_value(str, args[n_arg+1], &tmp_val, + (uint) je.stack_p + 1, + func_name(), (int) n_arg + 1, splice, + !compose_final, compose_final) || + append_simple(str, v_to, js_end - v_to)) goto js_error; /* Out of memory. */ } @@ -3620,10 +6343,13 @@ String *Item_func_json_insert::val_str(String *str) } if (append_simple(str, js->ptr(), v_to - js->ptr()) || - append_json_value(str, args[n_arg+1], &tmp_val) || - append_simple(str, je.s.c_str, js->end()-(const char *) je.s.c_str)) + append_json_value(str, args[n_arg+1], &tmp_val, (uint) je.stack_p, + func_name(), (int) n_arg + 1, splice, + !compose_final, compose_final) || + append_simple(str, je.s.c_str, js_end - (const char *) je.s.c_str)) goto js_error; /* Out of memory. */ continue_point: + DBUG_ASSERT(watch.unchanged(js)); { /* Swap str and js. */ if (str == &tmp_js) @@ -3639,12 +6365,93 @@ String *Item_func_json_insert::val_str(String *str) } } + /* + A document that was already written the loose way, edited with + pieces that were themselves written that way, is written that way + already: reading it back would write it exactly as it stands. So it + is passed as it stands, and the reading below is the reading + this whole exercise is about doing away with. + + THIS IS THE CANONICAL SITE. The other five that edit a document + point here rather than repeat it. + + What makes the answer A DOCUMENT. What was kept of the document was + read as one on the way in - that is compose_final's + args[0]->is_valid_json() - and is copied out of it whole, at offsets + the walk above computed rather than guessed. What was put in was + either read here as it went in, or attested to by whoever produced + it, and either way splice.is_valid says so. What was written here + is punctuation and a key, and a key that could reach past its own + quotes clears the same mark rather than being refused. + + What makes it WRITTEN THE LOOSE WAY. The kept parts were, by + args[0]->is_nice_json(); the values put in were, by splice.is_nice; + and the punctuation written here was, colon_len having been picked + from compose_final before anything was composed. + + And none of that means anything where the punctuation cannot be + written at all, which is compose_final's third part - see + is_json_compatible_charset(). A document in such a character set is a + scalar, because a container would take brackets it does not have. + + Nothing here takes any of it on faith. m_marks.set() reads the + answer back in a debug build and stops the server if these + conditions ever hold over something that is not a document written + that way - which is the same reading, kept exactly where it is worth + its cost and nowhere else. + + The answer is in js rather than str: each pass writes into str and + then swaps the two, so after an odd number of passes str is the + scratch. It is returned in the buffer the caller supplied, which + every path through this function has always done. + + HOW DEEP IT GOES, when nothing read it back to find out. Editing a + document puts a value somewhere inside it and copies the rest of it + across untouched, so the answer is deepest either where a value went + in - which splice.deepest holds, counted from the outside - or + somewhere the document already went, which is what the document was + able to say about itself. Neither of those can be short: the first + is a bound the splice was let through on, and the second is + whatever args[0] attested, which is nothing at all unless something + measured it. An item that attests to nothing leaves the answer + attesting to nothing, which is where this stood before there was + anything to ask. + */ + if (compose_final && splice.is_valid && splice.is_nice) + { + if (!return_json(to, js, MY_MAX(js_depth, splice.deepest))) + goto js_error; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; + /* + The reading back just above is what sets both marks. It also says + whether the depth reckoned at each splice was reckoned right: a + reading that got to the end is an answer inside the limit, so + nothing should have been marked as taking it past one. The other + way round is allowed - a value can be marked for reasons that leave + a readable answer behind, an empty one among them. + */ + DBUG_ASSERT(!splice.is_deep); + + /* + Both marks come from the reading back and from it alone: json_nice() + got to the end of what was composed, which is what makes it a + document, and what is marked is what json_nice() WROTE rather than + what it read, which is what makes it written the loose way. This is + the reading that stays wherever the document handed in was nobody's + word - it is not a leftover, and taking it out would take 10.11's + answers with it. The depth comes from there too, and is the one + measurement rather than a bound. + */ + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -3658,7 +6465,15 @@ String *Item_func_json_insert::val_str(String *str) bool Item_func_json_remove::fix_length_and_dec(THD *thd) { collation.set(args[0]->collation); - max_length= args[0]->max_length; + /* + What is left of the document is written out again, with a space after + every separator that is copied, so what comes back can be longer than + what went in even though something was taken out of it. A separator + has a value on either side of it and the shortest value is one + character, so at worst every second character gains one - which is + the allowance JSON_LOOSE asks for, adding the same spacing. + */ + fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * 2); mark_constant_paths(paths, args+1, arg_count-1); set_maybe_null(); @@ -3673,8 +6488,17 @@ String *Item_func_json_remove::val_str(String *str) uint n_arg, n_path; json_string_t key_name; THD *thd; + String *const to= str; + bool compose_final; + uint comma_len; + Json_source_watch watch; + /* How deep the argument item attested its document to go. */ + uint js_depth; + /* How deep the reading back below found the answer to go. */ + uint read_back_depth= JSON_DEPTH_UNKNOWN; DBUG_ASSERT(fixed()); + m_marks.clear(); if (args[0]->null_value) goto null_return; @@ -3683,7 +6507,17 @@ String *Item_func_json_remove::val_str(String *str) JSON_DO_PAUSE_EXECUTION(thd, 0.0002); str->set_charset(js->charset()); - json_string_set_cs(&key_name, js->charset()); + + /* + Whether what is composed here is what will be returned, or only + what a reading back at the end will be made from. Asked once, + before anything is composed, for the reason given where + Item_func_json_insert::val_str() asks it. + */ + compose_final= document_arg_composes_final(args[0], js); + comma_len= compose_final ? 2 : 1; + /* Taken here for the reason given at the same place there. */ + js_depth= args[0]->last_depth(); for (n_arg=1, n_path=0; n_arg < arg_count; n_arg++, n_path++) { @@ -3693,6 +6527,13 @@ String *Item_func_json_remove::val_str(String *str) json_path_step_t *lp; int n_item= 0; + /* + Taken afresh every time round: the end of this loop swaps what + was composed here into js, so what the document IS changes + between one path and the next, on purpose. + */ + watch.take(js); + if (!c_path->parsed) { String *s_p= args[n_arg]->val_str(tmp_paths+n_path); @@ -3711,7 +6552,7 @@ String *Item_func_json_remove::val_str(String *str) c_path->p.last_step--; if (c_path->p.last_step < c_path->p.steps) { - c_path->p.s.error= TRIVIAL_PATH_NOT_ALLOWED; + json_error(&c_path->p.s, TRIVIAL_PATH_NOT_ALLOWED); report_path_error(s_p, &c_path->p, n_arg); goto null_return; } @@ -3723,6 +6564,7 @@ String *Item_func_json_remove::val_str(String *str) if (args[n_arg]->null_value) goto null_return; + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -3786,6 +6628,13 @@ String *Item_func_json_remove::val_str(String *str) if (je.value_type != JSON_VALUE_OBJECT) continue; + /* + Set here and not once before the loop: each path is written in + its own character set, and the key has to be read in the one the + path it came from was written in. + */ + json_string_set_cs(&key_name, c_path->p.s.cs); + while (json_scan_next(&je) == 0 && je.state != JST_OBJ_END) { switch (je.state) @@ -3819,13 +6668,42 @@ String *Item_func_json_remove::val_str(String *str) if (json_skip_key(&je) || json_scan_next(&je)) goto js_error; - rem_end= (je.state == JST_VALUE && n_item == 0) ? + rem_end= (je.state == JST_VALUE && n_item == 0) ? (const char *) je.s.c_str : (const char *) (je.s.c_str - je.sav_c_len); + /* + Taking out the first piece of an array takes the comma after it + with it, and in a document written the loose way a space stands + behind that comma. That space belonged to the piece that has just + gone; leaving it puts it behind the bracket instead, in front of a + piece that already has all the spacing it needs. Every other + piece ends AT its comma and so leaves nothing over. + + Only where what is composed here is the answer. Where it is not, + the reading back at the end settles the spacing and drops that + space itself, so skipping it changes no answer - but the reading + complains, when it has to, about a position in the text it was + given, and a text composed a character shorter than a released + server composed it moves every such position by one. This is the + same reason the comma below is written the width a released + server writes it. + */ + if (compose_final && je.state == JST_VALUE && n_item == 0) + rem_end= (const char *) json_skip_space(je.s.cs, (const uchar *) rem_end, + (const uchar *) js->end()); + str->length(0); + /* + What is removed reaches from just before the piece to just before + the piece after it, so the two ends join without punctuation + everywhere but between two keys, where one comma has to be put + back. It is the only punctuation this function writes, and so + the only place the two formats can differ. + */ if (append_simple(str, js->ptr(), rem_start - js->ptr()) || - (je.state == JST_KEY && n_item > 0 && str->append(",", 1)) || + (je.state == JST_KEY && n_item > 0 && + str->append(json_loose_comma, comma_len)) || append_simple(str, rem_end, js->end() - rem_end)) goto js_error; /* Out of memory. */ @@ -3844,13 +6722,39 @@ String *Item_func_json_remove::val_str(String *str) } } + /* + Nothing is spliced in here - what is written is what was read, + less the piece that was asked for - so a document that was written + the loose way is still written that way with the piece gone, and + reading it back would write it exactly as it stands. See + Item_func_json_insert::val_str() for why the answer is in js, and + for what makes this safe to believe. + + Nothing goes in, so nothing can go deeper: whatever the document + said about its own depth is still true of it with a piece taken + out, and taking one out is the only thing that happens here. + */ + if (compose_final) + { + if (!return_json(to, js, js_depth)) + goto js_error; /* Out of memory. */ + + return to; + } + json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, Item_func_json_format::LOOSE)) + if (json_nice(&je, str, Item_func_json_format::LOOSE, &read_back_depth)) goto js_error; null_value= 0; + /* + Both marks come from the reading back, and so does the depth, for + the reason given where Item_func_json_insert::val_str() marks the + same answer. + */ + m_marks.set(str, true, true, read_back_depth); return str; js_error: @@ -3910,17 +6814,21 @@ static int check_key_in_list(String *res, String *Item_func_json_keys::val_str(String *str) { json_engine_t je; - String *js= args[0]->val_json(&tmp_js); + Json_source_watch watch; + String *js= val_json_arg0(args, arg_count, &tmp_js); uint n_keys= 0; int array_counters[JSON_DEPTH_LIMIT]= {0}; THD *thd; + m_marks.clear(); + if ((args[0]->null_value)) goto null_return; thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); json_scan_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; @@ -3949,6 +6857,7 @@ String *Item_func_json_keys::val_str(String *str) path.cur_step= path.p.steps; + DBUG_ASSERT(watch.unchanged(js)); if (json_find_path(&je, &path.p, &path.cur_step, array_counters)) { if (je.s.error) @@ -3988,7 +6897,7 @@ String *Item_func_json_keys::val_str(String *str) if (!check_key_in_list(str, key_start, key_len)) { - if ((n_keys > 0 && str->append(", ", 2)) || + if ((n_keys > 0 && str->append(STRING_WITH_LEN(json_loose_comma))) || str->append('"') || append_simple(str, key_start, key_len) || str->append('"')) @@ -4009,6 +6918,39 @@ String *Item_func_json_keys::val_str(String *str) if (unlikely(je.s.error || str->append(']'))) goto err_return; + /* + What was just written is an array of the object's key names, and + all three things there are to say about it are settled by how it + was written rather than by anything that would have to be measured. + + It READS AS A DOCUMENT. A key goes in between quotes with its bytes + copied across as they stood, and they stood inside a document that + parsed - so whatever needed escaping in them is already escaped, and + a key that would not go into an array would not have come out of an + object. Nothing else can get in: every key written was copied out + of an object this walk read to its end, and a break the walk reaches + refuses the answer whole at the test just above. What it does not + reach it does not read - text standing after the object, or a second + document behind it - so the argument need not be a document all the + way through for this to hold, and it is not asked to be. + + It is FORMATTED THE PLAIN WAY. The one thing that stands between two + of anything here is the pair of characters the plain formatting puts + there, and nothing goes after the opening bracket or before the + closing one. + + It is ONE LEVEL DEEP. An array of strings, and a key name is a + string whatever it holds. + + All of it in the character set the argument was read in - the copy + is same-set, so the bytes mean here what they meant there - which + leaves only whether a document can be written in that set at all. + */ + { + bool is_json_compatible= is_json_compatible_charset(str->charset()); + + m_marks.set(str, is_json_compatible, is_json_compatible, 1); + } null_value= 0; return str; @@ -4106,7 +7048,8 @@ int Item_func_json_search::compare_json_value_wild(json_engine_t *je, } -static int append_json_path(String *str, const json_path_t *p) +static int append_json_path(String *str, const json_path_t *p, + bool *path_bytes_well_formed) { const json_path_step_t *c; @@ -4123,6 +7066,17 @@ static int append_json_path(String *str, const json_path_t *p) } else /*JSON_PATH_ARRAY*/ { + /* + The brackets go in through String::append(), which converts them, + so a set that writes no character in one byte gets a bracket of + its own width. The number between them does not go that way: the + digits are written as themselves, whatever the set. It is the + one part of a path not formatted the way the rest of it is, and one + stray byte is enough to leave every character after it reading + from the wrong place. + */ + if (str->charset()->mbminlen > 1) + *path_bytes_well_formed= false; if (str->append('[') || str->append_ulonglong(c->n_item) || @@ -4137,13 +7091,18 @@ static int append_json_path(String *str, const json_path_t *p) String *Item_func_json_search::val_str(String *str) { + Json_source_watch watch; String *js= args[0]->val_json(&tmp_js); + watch.take(js); String *s_str= args[2]->val_str(&tmp_path); json_engine_t je; json_path_t p, sav_path; uint n_arg; int array_sizes[JSON_DEPTH_LIMIT]; uint has_negative_path= 0; + bool path_bytes_well_formed= true; + + m_marks.clear(); if (args[0]->null_value || args[2]->null_value) goto null_return; @@ -4178,6 +7137,7 @@ String *Item_func_json_search::val_str(String *str) goto null_return; } + DBUG_ASSERT(watch.unchanged(js)); json_get_path_start(&je, js->charset(),(const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length(), &p); @@ -4204,10 +7164,11 @@ String *Item_func_json_search::val_str(String *str) if (n_path_found == 2) { if (str->append('[') || - append_json_path(str, &sav_path)) + append_json_path(str, &sav_path, &path_bytes_well_formed)) goto js_error; } - if (str->append(", ", 2) || append_json_path(str, &p)) + if (str->append(STRING_WITH_LEN(json_loose_comma)) || + append_json_path(str, &p, &path_bytes_well_formed)) goto js_error; } if (mode_one) @@ -4224,7 +7185,7 @@ String *Item_func_json_search::val_str(String *str) goto null_return; if (n_path_found == 1) { - if (append_json_path(str, &sav_path)) + if (append_json_path(str, &sav_path, &path_bytes_well_formed)) goto js_error; } else @@ -4233,6 +7194,35 @@ String *Item_func_json_search::val_str(String *str) goto js_error; } + /* + A path is written out as a JSON string, and the keys that go into it + are copied from the document with whatever escaping they were + written with there - a key holding a quote was already written with + that quote escaped, or the document would not have read. Several + paths are put in an array with the loose formatting between them. + + How deep it goes is settled by which of those two was written and by + nothing else: one path is a string and nests nothing, several are an + array of strings and nest one. That is a figure this function has + in hand rather than one it would have to go and measure, and without + it a caller splicing the answer falls back to guessing the depth + from the length - which for paths of any size says more levels than + a document is allowed and sends the whole thing to be read again. + + None of that is worth anything if the bytes do not read at all, and + a path carrying an array index in a set that writes no character in + one byte does not - see the number written into it. A set that can + write answers only for what went through it, so the path says for + itself whether it was written, and an unencoded one is passed on as + the bytes it is rather than as a document. + */ + { + bool result_is_document= is_json_compatible_charset(str->charset()) && + path_bytes_well_formed; + + m_marks.set(str, result_is_document, result_is_document, + n_path_found == 1 ? 0 : 1); + } null_value= 0; return str; @@ -4288,11 +7278,16 @@ bool Item_func_json_format::fix_length_and_dec(THD *thd) String *Item_func_json_format::val_str(String *str) { - String *js= args[0]->val_json(&tmp_js); + String *js= val_json_arg0(args, arg_count, &tmp_js); json_engine_t je; + Json_source_watch watch; int tab_size= 4; + uint deepest= JSON_DEPTH_UNKNOWN; THD *thd; + m_marks.clear(); + + watch.take(js); if ((null_value= args[0]->null_value)) return 0; @@ -4316,29 +7311,65 @@ String *Item_func_json_format::val_str(String *str) tab_size= TAB_SIZE_LIMIT; } + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), (const uchar *) js->ptr()+js->length()); je.killed_ptr= (uint32_t *) &thd->killed; - if (json_nice(&je, str, fmt, tab_size)) + if (json_nice(&je, str, fmt, &deepest, tab_size)) { null_value= 1; report_json_error(js, &je, 0); return 0; } + /* + Read back and written out again, so is_valid holds whatever the + argument was - but only json_loose() writes the loose form, the + other two formats being the whole point of the other two names. + The depth is what that reading measured, and no formatting changes + it: all three write the same structures and differ only in what + they put between them. + */ + m_marks.set(str, true, fmt == LOOSE, deepest); return str; } -String *Item_func_json_format::val_json(String *str) +/* + Nothing is read and nothing is written: the argument's value is passed + straight on. So whatever the argument was able to say about it is + still true of it here, and this function says neither more nor less. +*/ +String *Item_func_json_format::forward_json(String *js) { - String *js= args[0]->val_json(&tmp_js); + m_marks.clear(); if ((null_value= args[0]->null_value)) return 0; + m_marks.set(js, args[0]->is_valid_json(), args[0]->is_nice_json(), + args[0]->last_depth()); return js; } + +String *Item_func_json_format::val_json(String *str) +{ + return forward_json(args[0]->val_json(&tmp_js)); +} + + +/* + What is returned IS the argument's value, so a caller that is + finished with it before anything else can run leaves the argument free + to answer with a view of what it already holds - see + Item::val_json_at_once(). The promise is passed along with the value + rather than stopping here. +*/ +String *Item_func_json_format::val_json_at_once(String *str) +{ + return forward_json(args[0]->val_json_at_once(&tmp_js)); +} + int Arg_comparator::compare_json_str_basic(Item *j, Item *s) { String *js,*str; @@ -4455,17 +7486,94 @@ bool Item_func_json_arrayagg::fix_fields(THD *thd, Item **ref) { bool res= Item_func_group_concat::fix_fields(thd, ref); m_tmp_json.set_charset(collation.collation); - /* account for opening and closing brackets */ - max_length= MY_MIN(max_length + 2*collation.collation->mbminlen, UINT_MAX32); + /* + Account for the opening and closing brackets, asking for the room + they take rather than working it out again: the width the item + declares and the room the limit keeps back for them are the same + figure, and two formats of it are two chances for one to move. + + Added in a type wide enough to hold the sum. A cap of a gigabyte in + a set of four bytes to the character fills the width on its own, and + an addition made in that width would carry the brackets round to + nothing. + */ + max_length= (uint32) MY_MIN((ulonglong) max_length + + reserved_result_length(), UINT_MAX32); return res; } +/* + func_name() ends with '(' so that "Row %lu was cut by %s)" reads + properly. A diagnostic that names the function on its own wants the + name without it. +*/ +static const char json_arrayagg_name[]= "json_arrayagg"; + + +/* + Cleared where a group begins rather than where its result is asked + for. Without ORDER BY or DISTINCT the rows are written out as they + arrive, long before anything asks for the result, so clearing the mark + at that point would throw away what the rows had already reported. +*/ +void Item_func_json_arrayagg::clear() +{ + m_bad_element= false; + m_closed= false; + m_elements_valid= true; + m_elements_depth= 1; + Item_func_group_concat::clear(); +} + + +/* + Returning nothing for a row is not the same as the row not being + there. The separator between one element and the next is written + before the element is asked for (dump_leaf_key(), item_sum.cc), and it + stays written whether an element follows it or not, so a row declined + here leaves a separator with nothing on one side of it. + + That is what has always been written for a value holding a character + no document can carry, and it still is - the writing is not this + function's to undo, and how much of a group there is around it is not + its to know either. A row lost to a buffer that would not grow is a + different matter: nothing about the data asked for it and the + statement is over anyway, so the group is marked and refused whole + when it is asked for. + + A row holding something that does not parse as JSON reaches neither + of these. It is written out as it stands, the same as it always was. + + What becomes of the group in the two cases that leave it not a + document is settled where it is asked for, both of them through the + same mark - see val_str(). +*/ String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) { + int rc; + /* + One deep for the brackets the group is put inside, which is where + every element written here sits - see Item_func_json_array::val_str(). + Carried across the elements rather than kept per element: the group + is as deep as its deepest element, and the elements are written out + one at a time long before the brackets go on. + */ + Json_splice_marks marks(1); + m_tmp_json.length(0); - if (append_json_value(&m_tmp_json, i, tmp)) + rc= append_json_value(&m_tmp_json, i, tmp, 1, json_arrayagg_name, 0, marks, + false, false, Sql_condition::WARN_LEVEL_WARN); + if (!marks.is_valid) + m_elements_valid= false; + m_elements_depth= MY_MAX(m_elements_depth, marks.deepest); + + if (rc) + { + if (rc == JSON_APPEND_OOM) + m_bad_element= true; return NULL; + } return &m_tmp_json; } @@ -4473,30 +7581,48 @@ String *Item_func_json_arrayagg::get_str_from_item(Item *i, String *tmp) String *Item_func_json_arrayagg::get_str_from_field(Item *i,Field *f, String *tmp, const uchar *key, size_t offset) { + int rc; + /* One deep for the brackets - see get_str_from_item() just above. */ + Json_splice_marks marks(1); + m_tmp_json.length(0); - if (append_json_value_from_field(&m_tmp_json, i, f, key, offset, tmp)) + rc= append_json_value_from_field(&m_tmp_json, i, f, key, offset, tmp, + 1, json_arrayagg_name, 0, marks); + if (!marks.is_valid) + m_elements_valid= false; + m_elements_depth= MY_MAX(m_elements_depth, marks.deepest); + + if (rc) + { + if (rc == JSON_APPEND_OOM) + m_bad_element= true; return NULL; + } return &m_tmp_json; } +/* + Cut back to where the element that overran began, rather than to the + byte the limit falls on. What GROUP_CONCAT passes as old_length + is the length the result had before the separator and this element + were written, so it is the end of the last whole element - the one + place a cut leaves something the brackets can go round. + + A cut anywhere else leaves half an element behind, and the three ways + it can land are each their own kind of wrong. On a quote it leaves a + string opened twice; on a separator it leaves one with nothing after + it; and just past an opening quote it leaves an empty string that no + row of the group ever held - which reads back as an element and is + not one, so nothing downstream has any way to know it is not data. +*/ void Item_func_json_arrayagg::cut_max_length(String *result, - uint old_length, uint max_length) const + uint old_length, uint max_length __attribute__((unused))) const { - if (result->length() == 0) - return; - - if (result->end()[-1] != '"' || old_length == max_length) - { - Item_func_group_concat::cut_max_length(result, old_length, max_length); - return; - } - - Item_func_group_concat::cut_max_length(result, old_length, max_length-1); - result->append('"'); + result->length(old_length); } @@ -4508,25 +7634,122 @@ Item *Item_func_json_arrayagg::copy_or_same(THD* thd) String* Item_func_json_arrayagg::val_str(String *str) { + m_marks.clear(); + if ((str= Item_func_group_concat::val_str(str))) { String s; - s.append('['); - s.swap(*str); - str->append(s); - str->append(']'); + + /* + The brackets are built up in a buffer of their own which is then + exchanged with the result, and the exchange carries the character + set across along with the bytes. Left at its default the buffer + would say that what it holds is bytes rather than text, which + costs twice over: a bracket is then written one byte wide even + where a character of this result is two or four, leaving + everything between the two of them a byte out of step; and a + client asking for a character set other than the one the result + was computed in is handed the bytes as they stand, where every + other function here converts them first. + */ + s.set_charset(collation.collation); + + /* + A row that could not be written out left the elements it sits + between joined by nothing but their separator, so there is no + array here to return. Reachable only through a failure to + allocate, which has already raised an error of its own. + */ + if (m_bad_element) + { + null_value= 1; + return NULL; + } + + /* + A group that is not a document is not returned either, for the + reason the constructors give where they refuse the same thing: the + elements were read as they went in, that reading is the only one + there is, and what it found has nowhere else to be acted on. + + The row that made it so has already been reported - a value that + did not parse through the warning naming the argument, one carrying + a character no document can hold through the note saying which - so + what is added here is the refusal and not the complaint. + + A group cut back to fit the length limit is NOT this. It is cut to + the last whole element and the brackets go round what is left, so + it is a shorter document and still a document; the mark below + declines it all the same, that being about the answer this group + had rather than about the one it gives. + */ + if (!m_elements_valid) + { + null_value= 1; + return NULL; + } + + /* + Asked for a second time, the group is already inside its brackets + and there is nothing left to do to it. What can be said about it + is said either way: the answer is about the value, and the value + is the same one. + */ + if (!m_closed) + { + /* + The brackets are put on last, so a buffer that will not take them + leaves behind something that reads as a value of its own rather + than as the array it is meant to be. + */ + if (s.append('[')) + goto bad_result; + + s.swap(*str); + if (str->append(s) || str->append(']')) + goto bad_result; + + m_closed= true; + } + + /* + Asked once the brackets are on, for the reason given there. The + bare name, func_name() here carrying the open bracket that the cut + warning writes its own closing one after. + */ + if (json_container_charset_refused(str, json_arrayagg_name)) + goto bad_result; + + /* + A group that was cut to fit the length limit is cut back to the + last whole element, so what stands between the brackets reads - + but it is a shorter answer than the group has, with the row it + stopped at reported, so this declines it. + + Never said to be nicely written: what goes between the elements is + the separator this function inherits from GROUP_CONCAT, a comma + with nothing after it, where a nicely written document puts a + space there as well. There is no asking for another one - the + grammar takes no SEPARATOR here and writes that comma itself. + */ + m_marks.set(str, !warning_for_row, false, m_elements_depth); } return str; + +bad_result: + null_value= 1; + return NULL; } Item_func_json_objectagg:: Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item) - :Item_sum(thd, item) + :Item_sum(thd, item), m_bad_pair(false), m_closed(false), + m_pairs_valid(true), m_pairs_depth(1), m_cut(false), m_row_count(0), + m_thd(NULL) { quick_group= FALSE; result.set_charset(collation.collation); - result.append('{'); } @@ -4561,9 +7784,19 @@ Item_func_json_objectagg::fix_fields(THD *thd, Item **ref) result.set_charset(collation.collation); result_field= 0; null_value= 1; + /* + The cap, and on top of it the room the two braces take, for the + reason the sister aggregate carries the same allowance: the shortest + answer this function gives is the braces with nothing between them, + and a cap smaller than that is answered over the cap with nothing + left to cut. A width worked out from the cap alone would then be a + column too narrow to hold the answer, which is cut into it without a + word said. + */ max_length= (uint32) MY_MIN((ulonglong) thd->gconcat_max_len() / collation.collation->mbminlen - * collation.collation->mbmaxlen, UINT_MAX32); + * collation.collation->mbmaxlen + + 2 * brace_length(), UINT_MAX32); if (check_sum_func(thd, ref)) @@ -4579,7 +7812,7 @@ void Item_func_json_objectagg::cleanup() DBUG_ENTER("Item_func_json_objectagg::cleanup"); Item_sum::cleanup(); - result.length(1); + result.length(0); DBUG_VOID_RETURN; } @@ -4590,33 +7823,168 @@ Item *Item_func_json_objectagg::copy_or_same(THD* thd) } +/* + The opening brace is written here rather than where the object is + built up, because here the result has the character set it will be + read in and the brace can be written at the width that asks for. The + closing one is written when the result is asked for, by which time the + same is true of it, so the two ends match. +*/ void Item_func_json_objectagg::clear() { - result.length(1); + result.length(0); null_value= 1; + /* + The brace is a write into the object like any other, and a group + that lost one is refused whole rather than returned short. + */ + m_bad_pair= false; + m_closed= false; + m_pairs_valid= true; + m_pairs_depth= 1; + m_cut= false; + m_row_count= 0; + m_thd= current_thd; + if (result.append('{')) + m_bad_pair= true; } +/* + The mirror of json_arrayagg_name above, and the other way round: the + message the cut warning goes into supplies its own ')', so the name + written into it is the one with the opening bracket already on it. + func_name() is the bare name here, being written into notes about the + data as well. +*/ +static const char json_objectagg_cut_name[]= "json_objectagg("; + + bool Item_func_json_objectagg::add() { StringBuffer buf; + /* + One deep for the braces the group is put inside, and carried across + the pairs - see Item_func_json_arrayagg::get_str_from_item(). + */ + Json_splice_marks marks(1); String *key; + int rc; + /* + Where this pair begins, so that a pair taking the object past the + length limit can be taken back off whole - see the cut at the end + of this function. + */ + uint32 old_length= result.length(); + + /* + A cut group takes nothing further, so neither argument is evaluated + for the rows after the cut. The rows are still read; see the cut at + the end of this function. + + An argument with side effects - a stored function that writes, an + assignment, a warning raised per row - therefore runs for the rows + up to the cut and not for those after it. Nothing documents how + many times an aggregate evaluates its arguments. + */ + if (m_cut) + return 0; key= args[0]->val_str(&buf); if (args[0]->is_null()) return 0; + m_row_count++; + + /* + Whether this pair needs a separator in front of it is whether a pair + has been written already, which is what null_value says until this + row changes it. Comparing the buffer against the opening brace would + be comparing it against a width that is not always one byte. + */ + if ((!null_value && result.append(STRING_WITH_LEN(", "))) || + (!null_value && DBUG_IF("json_objectagg_separator_out_of_memory"))) + goto bad_pair; null_value= 0; - if (result.length() > 1) - result.append(STRING_WITH_LEN(", ")); - result.append('"'); - st_append_escaped(&result,key); - result.append(STRING_WITH_LEN("\":")); + if (result.append('"') || DBUG_IF("json_objectagg_key_out_of_memory")) + goto bad_pair; + + /* + A key with a character no document can carry has always gone in as + however much of it could be written, and the pair has always been + finished around it. Only a buffer that would not grow gives up. + */ + if ((rc= st_append_escaped(&result, key))) + { + if (rc == JSON_APPEND_OOM) + goto bad_pair; + /* + The escaping writes into the buffer and says afterwards how much of + what it wrote counts, and a key it refused counts for none - so + the key that goes in is the empty one, with the quotes and the + colon written round it here as they are round any other. The + object that comes of it is a document, and its keys are not the + ones that were asked for; the note is what says so, and nothing + about being a document is dropped for it. + */ + report_bad_chr_note(func_name(), 1); + } + + if (result.append(STRING_WITH_LEN("\":")) || + DBUG_IF("json_objectagg_colon_out_of_memory")) + goto bad_pair; buf.length(0); - append_json_value(&result, args[1], &buf); + rc= append_json_value(&result, args[1], &buf, 1, func_name(), 1, marks, + false, false, Sql_condition::WARN_LEVEL_WARN); + if (!marks.is_valid) + m_pairs_valid= false; + m_pairs_depth= MY_MAX(m_pairs_depth, marks.deepest); + if (rc == JSON_APPEND_OOM) + goto bad_pair; + + /* + The same length limit the sister aggregate is held to, read where it + is read there - the session's setting as it stands while the group is + being built, rather than the width this item was fixed for. + + Cut back to where this pair began rather than to the byte the limit + falls on, for the reason given at + Item_func_json_arrayagg::cut_max_length(): pairs are what the braces + can go round, and half of one is not. + + The limit is on the bytes returned, and the brace that closes the + object is one of them; it is written when the group is asked for, + which is after every pair has been through here. So the room for it + is kept back from the limit rather than left to be discovered once + there is nothing to be done about it. The opening brace is in the + buffer already and is counted where it stands. Neither of them is + one byte wide in a set that writes no character in one byte, which + is why the width is asked for rather than assumed. + */ + { + uint32 close_len= brace_length(); + + DBUG_ASSERT(m_thd); + if (result.length() + close_len > m_thd->gconcat_max_len()) + { + result.length(old_length); + m_cut= true; + report_cut_value_error(m_thd, m_row_count, json_objectagg_cut_name); + } + } + + return 0; +bad_pair: + /* + The pair has been written out in part, and the buffer will not take + the rest of it. Whatever else the group goes on to add, the object + it is building is already broken, so it is refused as a whole + rather than returned half written. + */ + m_bad_pair= true; return 0; } @@ -4624,10 +7992,38 @@ bool Item_func_json_objectagg::add() String* Item_func_json_objectagg::val_str(String* str) { DBUG_ASSERT(fixed()); + m_marks.clear(); + if (null_value) return 0; - result.append('}'); + if (m_bad_pair || (!m_closed && result.append('}'))) + { + null_value= 1; + return 0; + } + m_closed= true; + + /* + And an object that is not a document is not returned, for the reason + the sister aggregate gives where it refuses the same thing. Only a + VALUE gets an object here: a key that could not be written leaves + the pair whole around an empty one. + + The braces are on by now, which is what the second of these reads. + */ + if (!m_pairs_valid || json_container_charset_refused(&result, func_name())) + { + null_value= 1; + return 0; + } + + /* + Never said to be nicely written: a key is followed here by a colon + and the value straight after it, where the loose form puts a space + between the two. + */ + m_marks.set(&result, true, false, m_pairs_depth); return &result; } @@ -4637,7 +8033,9 @@ String *Item_func_json_normalize::val_str(String *buf) String tmp; json_engine_t je; THD *thd; - String *raw_json= args[0]->val_str(&tmp); + String *raw_json= val_json_arg0(args, arg_count, &tmp); + + m_marks.clear(); DYNAMIC_STRING normalized_json; if (init_dynamic_string(&normalized_json, NULL, 0, 0)) @@ -4664,6 +8062,20 @@ String *Item_func_json_normalize::val_str(String *buf) if (buf->append(normalized_json.str, normalized_json.length)) goto null_return; + /* + Written out afresh from a document that was read through in full, + and written in the character set this function declares, whatever + the argument arrived in. Not written the loose way, though: the + normal form puts nothing after a comma or a colon. + + As deep as what it was made from. Normalising sorts the keys of an + object and takes the spacing out; it puts nothing inside anything + and takes nothing out of anything, so every structure that was there + is there still and no other is. So whatever the argument could say + about its own depth is said about this, and where it said nothing + this says nothing, which is where it stood. + */ + m_marks.set(buf, true, false, args[0]->last_depth()); goto end; null_return: @@ -4700,6 +8112,27 @@ void json_skip_current_level(json_engine_t *js, json_engine_t *value) } +/* + Put an engine back where it was remembered from, so the next candidate + can be tried against it. + + A refusal has to survive that. The snapshot was taken before the + engine failed, so restoring over the failure returns an engine that + reads as healthy, standing somewhere the scanner never reached; the + walk then goes on questioning a document it can no longer read, and + the complaint that is finally made - if one is made at all - names + wherever the walk ran out rather than what was wrong with the + document. Nonzero says the walk is over. +*/ +static int json_resume_scan(json_engine_t *je, const json_engine_t *saved) +{ + if (je->s.error) + return 1; + *je= *saved; + return 0; +} + + /* At least one of the two arguments is a scalar. */ bool json_find_overlap_with_scalar(json_engine_t *js, json_engine_t *value) { @@ -4765,7 +8198,8 @@ bool json_compare_arr_and_obj(json_engine_t *js, json_engine_t *value) int res1= json_find_overlap_with_object(js, value, true); if (res1) return TRUE; - *value= loc_val; + if (json_resume_scan(value, &loc_val)) + return FALSE; } if (js->value_type == JSON_VALUE_ARRAY) json_skip_level(js); @@ -4831,9 +8265,11 @@ int json_find_overlap_with_array(json_engine_t *js, json_engine_t *value, if (!json_value_scalar(value)) json_skip_level(value); } - *js= current_js; + if (json_resume_scan(js, ¤t_js)) + return FALSE; } - *value= loc_value; + if (json_resume_scan(value, &loc_value)) + return FALSE; if (!json_value_scalar(js)) json_skip_level(js); } @@ -4858,8 +8294,16 @@ int compare_nested_object(json_engine_t *js, json_engine_t *value) int result= 0; const char *value_begin= (const char*)value->s.c_str-1; const char *js_begin= (const char*)js->s.c_str-1; - json_skip_level(value); - json_skip_level(js); + /* + A refusal here means the fragment was not read to its end, so what + lies between begin and end is a piece of a document rather than one. + Normalizing that piece finds it unterminated and says so, which + replaces the reason the scanner gave with a complaint about where it + was made to stop. The engine already carries the reason; leave it + alone and let the caller report it. + */ + if (json_skip_level(value) || json_skip_level(js)) + return 0; const char *value_end= (const char*)value->s.c_str; const char *js_end= (const char*)js->s.c_str; json_engine_t je; @@ -4874,14 +8318,20 @@ int compare_nested_object(json_engine_t *js, json_engine_t *value) { goto error; } + /* + Only the code is carried over. The normalizing engine read a copy of + the value, so where it stopped is a position in that copy and says + nothing about this document; json_error() takes the reading from the + engine being refused, which is the one the caller reports through. + */ if (json_normalize_engine(&je, &a_res, a.ptr(), a.length(), value->s.cs)) { - value->s.error= je.s.error; + json_error(&value->s, je.s.error); goto error; } if (json_normalize_engine(&je, &b_res, b.ptr(), b.length(), value->s.cs)) { - js->s.error= je.s.error; + json_error(&js->s, je.s.error); goto error; } @@ -4960,7 +8410,8 @@ int json_find_overlap_with_object(json_engine_t *js, json_engine_t *value, only js (first argument i.e json document) and continue. */ - *js= loc_js; + if (json_resume_scan(js, &loc_js)) + return FALSE; continue; } } @@ -4977,7 +8428,8 @@ int json_find_overlap_with_object(json_engine_t *js, json_engine_t *value, return FALSE; if (!json_value_scalar(value)) json_skip_level(value); - *js= loc_js; + if (json_resume_scan(js, &loc_js)) + return FALSE; } } /* @@ -5075,6 +8527,7 @@ bool Item_func_json_overlaps::val_bool() json_engine_t je, ve; int result; THD *thd; + Json_source_watch watch; if ((null_value= (js == nullptr) || args[0]->null_value)) return 0; @@ -5082,6 +8535,7 @@ bool Item_func_json_overlaps::val_bool() thd= current_thd; JSON_DO_PAUSE_EXECUTION(thd, 0.0002); + watch.take(js); if (!a2_parsed) { val= args[1]->val_json(&tmp_val); @@ -5094,6 +8548,7 @@ bool Item_func_json_overlaps::val_bool() return 0; } + DBUG_ASSERT(watch.unchanged(js)); json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), (const uchar *) js->ptr() + js->length()); je.killed_ptr= (uint32_t *) &thd->killed; diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index cfa6a15077fe3..dd3056c7ed69f 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -26,6 +26,60 @@ #include "item_sum.h" #include "sql_type_json.h" +/* + The three below are read by DBUG_ASSERT and by nothing else, so they + are here wherever a DBUG_ASSERT is compiled rather than wherever a + debug build is. Those are not the same set: DBUG_ASSERT_AS_PRINTF + turns the assertion into a printed complaint and leaves its expression + standing, while setting DBUG_OFF, so a guard written the other way + round leaves those expressions with nothing to call and that build + stops compiling. +*/ +#ifdef DBUG_ASSERT_EXISTS +/* + Reads a value the way whoever receives it will read it, and says + whether it got to the end. For the debug checks that hold the marks to + what they say; the reading is not counted, being the debug build's work + and not the server's. +*/ +bool json_value_reads_as_document(const String *str); +/* + Writes the value out again in the loose form and says whether that + changed anything. For the same debug checks, and uncounted for the + same reason. +*/ +bool json_value_is_nice(const String *str); +/* + Reads the value and says how deep it actually goes. For the checks + that hold a claimed depth to the truth - a claim larger than this is + what saying nothing amounts to, and one smaller is the failure worth + catching. Uncounted for the same reason as the two above. +*/ +uint json_value_depth(const String *str); +#endif + +#ifndef DBUG_OFF +/* + Holds the reading count still across a reading the released server does + not do. + + The count is of the work a server does, and a debug build's reading back + of a value to check what was claimed about it is not that work: counting + it would move a number kept to watch queries for reasons no query has. + Where the reading is one call the count is taken straight back off it; + where it is a whole expression, whose readings the caller cannot count, + the figure is put back as it stood. +*/ +class Json_scans_unbilled +{ + THD *m_thd; + ulong m_scans; +public: + Json_scans_unbilled(THD *thd); + ~Json_scans_unbilled(); +}; +#endif + class json_path_with_flags { public: @@ -48,7 +102,23 @@ void report_json_error_ex(const char *js, json_engine_t *je, const char *fname, int n_param, Sql_condition::enum_warning_level lv); bool check_overlaps(json_engine_t *js, json_engine_t *value, bool compare_whole); -int st_append_escaped(String *s, const String *a); + +/* + What an append of escaped text to a document came to. The two ways + of failing are kept apart because they are not answered alike: a + buffer that would not grow has already raised an error of its own and + the statement is over, while a character that cannot be written into + a document is an ordinary property of the data, which callers have + always carried on past. +*/ +enum json_append_result +{ + JSON_APPEND_OK= 0, + JSON_APPEND_OOM= 1, + JSON_APPEND_BAD_CHR= 2 +}; + +json_append_result st_append_escaped(String *s, const String *a); int json_find_overlap_with_object(json_engine_t *js, json_engine_t *value, bool compare_whole); @@ -82,6 +152,15 @@ class Json_path_extractor: public json_path_with_flags { protected: String tmp_js, tmp_path; + /* + What the document said about itself, taken while extract() had just + evaluated it and before it worked the path out. A path is any + expression a caller cares to write, so the two readings would + otherwise be a stored function apart, and what the document answers + can change over that distance while the document in hand does not. + */ + bool m_js_nice; + uint m_js_depth; virtual ~Json_path_extractor() { } virtual bool check_and_get_value(Json_engine_scan *je, String *to, int *error)=0; @@ -110,11 +189,10 @@ class Item_func_json_valid: public Item_bool_func set_maybe_null(); return FALSE; } - bool set_format_by_check_constraint(Send_field_extended_metadata *to) const - override + bool json_valid_of_column_processor(void *arg) override { - static const Lex_cstring fmt(STRING_WITH_LEN("json")); - return to->set_format_name(fmt); + return Type_handler_json_common:: + is_json_valid_of_name(this, *(const LEX_CSTRING *) arg); } enum Functype functype() const override { return JSON_VALID_FUNC; } @@ -166,6 +244,14 @@ class Item_func_json_exists: public Item_bool_func class Item_json_func: public Item_str_func { +protected: + Json_result_marks m_marks; + /* + Return a document as this function's answer rather than reading it + again to find out what it is - out of line, where the rule it stands + on is written. + */ + String *return_json(String *to, const String *from, uint depth); public: Item_json_func(THD *thd) :Item_str_func(thd) { } @@ -179,6 +265,9 @@ class Item_json_func: public Item_str_func { return Type_handler_json_common::json_type_handler(max_length); } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } }; @@ -227,15 +316,42 @@ class Item_func_json_query: public Item_json_func, bool fix_length_and_dec(THD *thd) override; String *val_str(String *to) override { + m_marks.clear(); null_value= Json_path_extractor::extract(to, args[0], args[1], collation.collation, func_name_cstring(), true); - return null_value ? NULL : to; + if (null_value) + return NULL; + /* + The piece is copied out with whatever spacing the document it came + from was written with, so it is formatted the loose way exactly when + that document was. Cutting it out cannot change that: the loose + form writes the same punctuation wherever a value sits, so a value + inside a document is written there the way it would be written on + its own, and the two ends of the cut are the two ends of the value + with no spacing of the document's left on either side. + + And a piece of a document does not nest deeper than the document + it was cut out of, wherever inside it the cut was made - so what + the document could say about its own depth is said about the + piece as well. + + Both of them as the document answered them when it was read, not + as it answers them now - see Json_path_extractor::m_js_nice. + */ + m_marks.set(to, true, m_js_nice, m_js_depth); + return to; } bool check_and_get_value(Json_engine_scan *je, String *res, int *error) override { return je->check_and_get_value_complex(res, error); } + /* + Returns a slice of the searched document, delimited by the two ends + of a value the scanner has just parsed. A fully parsed value is a + document in its own right. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -269,6 +385,7 @@ class Item_func_json_unquote: public Item_str_func protected: String tmp_s; String *read_json(json_engine_t *je); + String *return_as_is(String *str, String *js); public: Item_func_json_unquote(THD *thd, Item *s): Item_str_func(thd, s) {} LEX_CSTRING func_name_cstring() const override @@ -335,6 +452,16 @@ class Item_func_json_extract: public Item_json_str_multipath double val_real() override; my_decimal *val_decimal(my_decimal *) override; uint get_n_paths() const override { return arg_count - 1; } + /* + The values found are put together into a result of their own. Each + of them was read as a document on the way in and is written back out + as one, so what holds them is all that can be wrong with the result + - and that is the brackets, written only when several values can + match. In a character set that cannot encode those, the result is + read back through json_nice() as it always was, and anything that + will not read comes back as NULL. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -413,6 +540,23 @@ class Item_func_json_array: public Item_json_func static LEX_CSTRING name= {STRING_WITH_LEN("json_array") }; return name; } + /* + A document or nothing. Every value is read as it goes in - the ones + quoted here come out of json_escape() and are documents of their + own, and the ones spliced as they stand were either read on the way + or passed by something that had already read them - and a value that + did not read as one takes the whole result down to NULL rather than + into the answer. What goes between them is written here. + + Which leaves the brackets and the separators. In a character set + that might not write those as themselves the result is read back + before it is returned, and one that does not read comes back NULL, + exactly as it does for the functions that always read back. + + Said for Item_func_json_object as well, which composes the same way + through the same routines. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -436,6 +580,19 @@ class Item_func_json_array_append: public Item_json_str_multipath static LEX_CSTRING name= {STRING_WITH_LEN("json_array_append") }; return name; } + /* + Each of the six functions that EDIT a document returns a document + or NULL, and for the same two reasons in all of them. + + Where the document it was given is_valid, and the result character + set can represent the punctuation being written, the result is a + document by construction: the parts retained were parsed as one on + input, and what goes between them is written here. Where either + does not hold, the whole result is re-parsed through json_nice() + before being returned, and anything that fails to parse gives + NULL. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -494,6 +651,11 @@ class Item_func_json_merge: public Item_func_json_array static LEX_CSTRING name= {STRING_WITH_LEN("json_merge_preserve") }; return name; } + /* + A document or nothing, for the reason given where + Item_func_json_array_append declares the same. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -518,6 +680,11 @@ class Item_func_json_merge_patch: public Item_func_json_merge }; +/* + Reads its argument through in full and writes the document out again + in a normal form, in utf8mb4 whatever the argument arrived in. What + it writes is the compact formatting, so it is never in the loose form. +*/ class Item_func_json_normalize: public Item_json_func { public: @@ -530,6 +697,12 @@ class Item_func_json_normalize: public Item_json_func return name; } bool fix_length_and_dec(THD *thd) override; + /* + Written out afresh from a document read through in full, and always + in utf8mb4, which can encode everything written into it whatever the + argument arrived in. + */ + bool is_valid_json_static() const override { return true; } Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } }; @@ -624,6 +797,11 @@ class Item_func_json_insert: public Item_json_str_multipath bool fix_length_and_dec(THD *thd) override; String *val_str(String *) override; uint get_n_paths() const override { return arg_count/2; } + /* + A document or nothing, for the reason given where + Item_func_json_array_append declares the same. + */ + bool is_valid_json_static() const override { return true; } LEX_CSTRING func_name_cstring() const override { static LEX_CSTRING json_set= {STRING_WITH_LEN("json_set") }; @@ -654,6 +832,11 @@ class Item_func_json_remove: public Item_json_str_multipath static LEX_CSTRING name= {STRING_WITH_LEN("json_remove") }; return name; } + /* + A document or nothing, for the reason given where + Item_func_json_array_append declares the same. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -661,7 +844,17 @@ class Item_func_json_remove: public Item_json_str_multipath }; -class Item_func_json_keys: public Item_str_func +/* + Typed as returning a document like the rest of the family: the keys + are returned as an array, and a function given that array in value + position is meant to embed an array. + + The array is attested as well. It is written one key at a time out of + a document that parsed, and being an array of strings fixes its + validity, its formatting and its depth without measuring any of the + three - see where the result is returned. +*/ +class Item_func_json_keys: public Item_json_func { protected: json_path_with_flags path; @@ -669,7 +862,7 @@ class Item_func_json_keys: public Item_str_func public: Item_func_json_keys(THD *thd, List &list): - Item_str_func(thd, list) {} + Item_json_func(thd, list) {} LEX_CSTRING func_name_cstring() const override { static LEX_CSTRING name= {STRING_WITH_LEN("json_keys") }; @@ -677,6 +870,12 @@ class Item_func_json_keys: public Item_str_func } bool fix_length_and_dec(THD *thd) override; String *val_str(String *) override; + /* + A document or nothing, whichever row it is asked about: an object + gives back an array of its key names and anything else gives back + NULL, so a column of these can be built saying so. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -708,6 +907,15 @@ class Item_func_json_search: public Item_json_str_multipath bool fix_length_and_dec(THD *thd) override; String *val_str(String *) override; uint get_n_paths() const override { return arg_count > 4 ? arg_count - 4 : 0; } + /* + A path is a JSON string, and every path returned here is built from + pieces of a document that has just been parsed. Several of them go + inside brackets, which needs a character set that can represent a + bracket - and a document in a character set that cannot is a + scalar, there being no way to write a container in it, so it holds + one value and yields one path. + */ + bool is_valid_json_static() const override { return true; } protected: Item *shallow_copy(THD *thd) const override @@ -728,6 +936,7 @@ class Item_func_json_format: public Item_json_func protected: formats fmt; String tmp_js; + String *forward_json(String *js); public: Item_func_json_format(THD *thd, Item *js, formats format): Item_json_func(thd, js), fmt(format) {} @@ -738,6 +947,7 @@ class Item_func_json_format: public Item_json_func bool fix_length_and_dec(THD *thd) override; String *val_str(String *str) override; String *val_json(String *str) override; + String *val_json_at_once(String *str) override; protected: Item *shallow_copy(THD *thd) const override @@ -758,6 +968,51 @@ class Item_func_json_arrayagg : public Item_func_group_concat const uchar *key, size_t offset) override; void cut_max_length(String *result, uint old_length, uint max_length) const override; + /* + The two brackets, which val_str() writes round the group once it is + asked for. They go in through String::append(), which writes them + in the set of the result, so each of them is as wide as the + narrowest character that set has. + */ + uint32 reserved_result_length() const override + { return 2 * collation.collation->mbminlen; } + /* + A row of this group could not be written out at all, the buffer + having failed to grow. Neither a row whose value does not parse as + JSON nor one holding a character no document can carry is this: + the first is written out as it stands and the second is dropped + where it always was, both with a note. Those two are answered by + the mark below instead. Reset for each group by clear(). + */ + bool m_bad_element; + /* + The brackets have been put on already. Nothing says how often the + result of a group is asked for, and the buffer they go into belongs + to this item and outlives the asking, so putting them on once per + call would put on one pair per call. Reset for each group by + clear(). + */ + bool m_closed; + /* + Whether everything that went into this group leaves the array around + it answerable for. Cleared by a value that did not read as JSON, or + one holding a character that could not be written, which is dropped + and leaves the separator either side of it with nothing between + them. Reset for each group by clear(), the same as above, and kept + apart from it because they say different things - that one means the + group is missing a row, this one that the group is complete and + still not a document. Either way there is no array to return, + so a group this is cleared for is refused when it is asked for. + */ + bool m_elements_valid; + /* + How deep the deepest element written so far reaches, counted from + outside the brackets that go round the group. Accumulated over the + elements for the same reason as the mark above: the group is written + out a row at a time, and the answer is the deepest of them. + */ + uint m_elements_depth; + Json_result_marks m_marks; public: String m_tmp_json; /* Used in get_str_from_*.. */ Item_func_json_arrayagg(THD *thd, Name_resolution_context *context_arg, @@ -765,15 +1020,49 @@ class Item_func_json_arrayagg : public Item_func_group_concat const SQL_I_List &is_order, String *is_separator, bool limit_clause, Item *row_limit, Item *offset_limit): Item_func_group_concat(thd, context_arg, is_distinct, is_select, is_order, - is_separator, limit_clause, row_limit, offset_limit) + is_separator, limit_clause, row_limit, offset_limit), + m_bad_element(false), m_closed(false), m_elements_valid(true), + m_elements_depth(1) { } + /* + A copy is not fixed again, so anything fix_fields() settled has to + be settled here too. The buffer's character set is one of those: + left at its default the copy would read the values it is given as + bytes rather than as the characters they are, and say they were + not JSON. + */ Item_func_json_arrayagg(THD *thd, Item_func_json_arrayagg *item) : - Item_func_group_concat(thd, item) {} + Item_func_group_concat(thd, item), m_bad_element(false), + m_closed(false), m_elements_valid(true), m_elements_depth(1) + { + m_tmp_json.set_charset(collation.collation); + } const Type_handler *type_handler() const override { return Type_handler_json_common::json_type_handler_sum(this); } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } + /* + A document or nothing, for the reason given where + Item_func_json_array declares the same: the elements are read as + they arrive, a group carrying one that did not read as a document + comes back NULL, and the brackets are written here. A group cut to + fit its length limit is cut back to the last whole element, so the + brackets still go round a document. + */ + bool is_valid_json_static() const override { return true; } + /* + Asked for rather than read off. The aggregate this inherits from + settles a NULL before any of the group is written, that being what + it says about a group with no rows in it and the only time it says + it, so Item_sum::is_null() reads what is already there. A group + refused here is refused as it is written out, and what is already + there is then the answer to a question nothing has asked yet. + */ + bool is_null() override { update_null_value(); return null_value; } LEX_CSTRING func_name_cstring() const override { @@ -783,6 +1072,7 @@ class Item_func_json_arrayagg : public Item_func_group_concat bool fix_fields(THD *thd, Item **ref) override; enum Sumfunctype sum_func() const override { return JSON_ARRAYAGG_FUNC; } + void clear() override; String* val_str(String *str) override; Item *copy_or_same(THD* thd) override; @@ -796,12 +1086,92 @@ class Item_func_json_arrayagg : public Item_func_group_concat class Item_func_json_objectagg : public Item_sum { String result; + /* + A pair of this group could not be written out in full, the buffer + having failed to grow part way through it. Reset for each group by + clear(). + */ + bool m_bad_pair; + /* + The closing brace has been written already. Nothing says how often + the result of a group is asked for, and the buffer it goes into is + this item's own and outlives the asking, so writing it once per call + would write one brace per call. Reset for each group by clear(). + */ + bool m_closed; + /* + Whether everything that went into this group leaves the object + around it answerable for. Cleared by a value that did not read as + JSON, or one holding a character that could not be written, which + leaves the string it was being written into unclosed. Reset for + each group by clear(), and kept apart from the flag above for the + same reason as in the sister aggregate - one means a pair is + missing, this one that the pairs are all there and still do not make + a document. Either way there is no object to return, so a group + this is cleared for is refused when it is asked for. + + A KEY holding such a character is neither: it goes in as the empty + key with its pair finished round it, so the object is a document + whose keys are not the ones that were asked for, and the note raised + where it happens is the whole of what is said about it. + */ + bool m_pairs_valid; + /* + How deep the deepest value written so far reaches, counted from + outside the braces that go round the group - the sister aggregate's + m_elements_depth, for the same reason. + */ + uint m_pairs_depth; + /* + The group has been cut back to fit group_concat_max_len and nothing + further goes into it. A later pair small enough to fit in what is + left would go in behind the pair that did not fit, putting the + object out of the order the group was read in. Once this is set, + add() evaluates neither argument. Reset for each group by clear(). + */ + bool m_cut; + /* + Which row of this group is being added, counted over the rows that + have a key to make a pair of - a row whose key is NULL is not one of + them and is not counted. Neither is a row read after the cut, whose + key is not evaluated. Only the cut warning reads it, to say where + the group stopped, and it has been raised by then. + + Counted per group, where the sister aggregate's row_count runs on + across the groups of a statement. Nothing here needs it to run on, + and a number naming a row of the group is what the warning reads as. + */ + uint m_row_count; + /* + The session this group is being built for, taken once by clear() + rather than looked up again for every row of it. A group runs on + one connection, so the lookup answers the same thing every time it + is made; what is read THROUGH it is read per row still, the length + limit being the session's setting as it stands. + */ + THD *m_thd; + Json_result_marks m_marks; + /* + How wide a brace is. Both of them go in through String::append(), + which writes them in the set of the result, so each is as wide as + the narrowest character that set has. + */ + uint32 brace_length() const + { return collation.collation->mbminlen; } public: + /* + The opening brace is not written here. This runs while the + expression is being parsed, before there is a character set to write + it in, and a brace put down now would be one byte wide however wide a + character of the result turns out to be. clear() writes it instead, + once per group and once the width is known. + */ Item_func_json_objectagg(THD *thd, Item *key, Item *value) : - Item_sum(thd, key, value) + Item_sum(thd, key, value), m_bad_pair(false), m_closed(false), + m_pairs_valid(true), m_pairs_depth(1), m_cut(false), m_row_count(0), + m_thd(NULL) { quick_group= FALSE; - result.append('{'); } Item_func_json_objectagg(THD *thd, Item_func_json_objectagg *item); @@ -817,6 +1187,16 @@ class Item_func_json_objectagg : public Item_sum { return Type_handler_json_common::json_type_handler_sum(this); } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } + /* + A document or nothing, for the reason given where the sister + aggregate declares the same. + */ + bool is_valid_json_static() const override { return true; } + /* Asked for rather than read off, also as in the sister aggregate. */ + bool is_null() override { update_null_value(); return null_value; } void clear() override; bool add() override; void reset_field() override { DBUG_ASSERT(0); } // not used diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index b4388f12e7f4c..2ed0911265f58 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -642,9 +642,6 @@ bool Item_func_concat::realloc_result(String *str, uint length) const if (str->alloced_length() >= length) return false; // Alloced space is big enough, nothing to do. - if (str->alloced_length() == 0) - return str->alloc(length); - /* Item_func_concat::val_str() makes sure the result length does not grow higher than max_allowed_packet. So "length" is limited to 1G here. @@ -653,6 +650,9 @@ bool Item_func_concat::realloc_result(String *str, uint length) const So multiplication by 2 can overflow, if args[0] for some reasons did not limit the result to max_alloced_packet. But it's not harmful, "str" will be reallocated exactly to "length" bytes in case of overflow. + It can even be zero, with "str" pointing to bytes that args[0] does not + own. realloc() moves those bytes into a buffer of our own, while alloc() + would throw them away: it empties the string it allocates for. */ uint new_length= MY_MAX(str->alloced_length() * 2, length); return str->realloc(new_length); @@ -3885,13 +3885,34 @@ String *Item_func_conv_charset::val_str(String *str) DBUG_ASSERT(fixed()); if (use_cached_value) return null_value ? 0 : &str_value; + m_marks.clear(); String *arg= args[0]->val_str(&tmp_value); String_copier_for_item copier(current_thd); - return ((null_value= args[0]->null_value || - copier.copy_with_warn(collation.collation, str, - arg->charset(), arg->ptr(), - arg->length(), arg->length()))) ? - 0 : str; + if ((null_value= args[0]->null_value || + copier.copy_with_warn(collation.collation, str, + arg->charset(), arg->ptr(), + arg->length(), arg->length()))) + return 0; + /* + Asked of the argument only now, after it has been evaluated: what it + answers is about the value it has just passed, which is the one + that was converted. A conversion that lost nothing kept the + characters it was given, and how deeply a value nests is a property + of characters like the other two, so all three carry across. + + Where the copy kept the BYTES and called them something else it + kept no such thing, and that is what the second test is for - see + String_copier::conversion_keeps_characters(), which reads the branch + this very copier took. Either end being the binary set is enough: + bytes going into it are relabelled, and bytes coming out of it are + read as whatever the other set makes of them. + */ + if (!copier.most_important_error_pos() && + String_copier::conversion_keeps_characters(collation.collation, + arg->charset())) + m_marks.set(str, args[0]->is_valid_json(), args[0]->is_nice_json(), + args[0]->last_depth()); + return str; } bool Item_func_conv_charset::fix_length_and_dec(THD *thd) diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index a0d8ea22136a7..9d61b758ec5f6 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -1982,6 +1982,21 @@ class Item_func_conv_charset :public Item_str_func { bool use_cached_value; String tmp_value; + /* + What can be said about the converted value. A conversion that lost + nothing kept the sequence of characters it was given, and JSON is a + grammar over characters, so a document that went in comes out a + document and comes out written the same way. A conversion that + substituted a character, or stopped on a byte it could not read, has + changed the value into something nothing here has looked at. + + Conversion to "binary" is the exception, and is refused below. It + is not a conversion but a relabelling: the bytes are kept, and each + one becomes a character of its own, so a multi-byte document comes + out as several times as many characters as it had. The copier + cannot report that, binary having no byte that it rejects. + */ + Json_result_marks m_marks; public: bool safe; Item_func_conv_charset(THD *thd, Item *a, CHARSET_INFO *cs): @@ -2004,6 +2019,16 @@ class Item_func_conv_charset :public Item_str_func use_cached_value= 1; str_value.mark_as_const(); safe= (errors == 0); + /* + The bytes are settled here and never computed again, so what is + said about them has to be settled here too. Asking the argument + later would pair an answer about a value it has since gone on to + produce with the one frozen above. + */ + if (!null_value && safe && + String_copier::conversion_keeps_characters(cs, str->charset())) + m_marks.set(&str_value, args[0]->is_valid_json(), + args[0]->is_nice_json(), args[0]->last_depth()); } else { @@ -2069,6 +2094,16 @@ class Item_func_conv_charset :public Item_str_func static LEX_CSTRING name= {STRING_WITH_LEN("convert") }; return name; } + /* + What this wrapper is put round. A caller looking through it for the + item underneath is told, rather than working out from the object + that this is what it was handed - see Item::conv_charset_arg(), + which says why that is worth a virtual of its own. + */ + Item *conv_charset_arg() const override { return args[0]; } + bool is_valid_json() const override { return m_marks.valid(); } + bool is_nice_json() const override { return m_marks.nice(); } + uint last_depth() const override { return m_marks.depth(); } void print(String *str, enum_query_type query_type) override; int save_in_field(Field*, bool) override; diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 0d4c2e5361a1a..070cc75a0db8d 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -1569,6 +1569,36 @@ bool Item_singlerow_subselect::val_native(THD *thd, Native *to) } +/* + A row subquery has more than one cache and none of them is the value, + which is what 'value' is NULL for; a subquery that has not been fixed + yet has none at all. +*/ + +bool Item_singlerow_subselect::is_valid_json() const +{ + return value && value->is_valid_json(); +} + + +bool Item_singlerow_subselect::is_nice_json() const +{ + return value && value->is_nice_json(); +} + + +uint Item_singlerow_subselect::last_depth() const +{ + return value ? value->last_depth() : JSON_DEPTH_UNKNOWN; +} + + +bool Item_singlerow_subselect::is_valid_json_static() const +{ + return value && value->is_valid_json_static(); +} + + my_decimal *Item_singlerow_subselect::val_decimal(my_decimal *decimal_value) { DBUG_ASSERT(fixed()); diff --git a/sql/item_subselect.h b/sql/item_subselect.h index ef2fa8b1fe168..9eb3fbf0707b2 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -323,6 +323,14 @@ class Item_singlerow_subselect :public Item_subselect my_decimal *val_decimal(my_decimal *) override; bool val_bool() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override; + /* + val_str() above returns what the cache holds, so the cache is what is + asked. Out of line because Item_cache is not a complete type here. + */ + bool is_valid_json() const override; + bool is_nice_json() const override; + uint last_depth() const override; + bool is_valid_json_static() const override; const Type_handler *type_handler() const override; bool fix_length_and_dec() override; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 1c2789f86b737..66da849474cd2 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -3786,7 +3786,7 @@ int group_concat_key_cmp_with_order_with_nulls(void *arg, } -static void report_cut_value_error(THD *thd, uint row_count, const char *fname) +void report_cut_value_error(THD *thd, uint row_count, const char *fname) { size_t fn_len= strlen(fname); char *fname_upper= (char *) my_alloca(fn_len + 1); @@ -3830,7 +3830,20 @@ int dump_leaf_key(void* key_arg, element_count count __attribute__((unused)), { Item_func_group_concat *item= (Item_func_group_concat *) item_arg; TABLE *table= item->table; - uint max_length= table->in_use->gconcat_max_len(); + /* + Room is kept back for whatever the caller still has to write round + this, so that the limit is reached where the answer reaches it + rather than two bytes further on. GROUP_CONCAT keeps back nothing + and is not moved by any of this. + + A limit too small to hold what the caller writes leaves nothing for + the rows, and the answer is then that wrapper by itself - as short + as the function goes, and still over the limit. Nothing can be done + about that here; the width the item declares carries room for it. + */ + uint reserved= item->reserved_result_length(); + uint gconcat_max_len= table->in_use->gconcat_max_len(); + uint max_length= gconcat_max_len > reserved ? gconcat_max_len - reserved : 0; String tmp((char *)table->record[1], table->s->reclength, default_charset_info); String tmp2; @@ -4498,6 +4511,16 @@ String* Item_func_group_concat::val_str(String* str) return &result; else DBUG_ASSERT(false); // Can't happen + + /* + The walk is what produces the result, and it has now happened. + dump_leaf_key() raises this itself for the first row it writes, + but a walk in which every row fell inside OFFSET writes none and + raises nothing, so the next caller would walk again - with the + offset already spent, and the rows it skipped the first time + appended to a result that has already been handed out once. + */ + result_finalized= true; } if (table && table->blob_storage && diff --git a/sql/item_sum.h b/sql/item_sum.h index 77e02cac30f1c..4c233d3e6ab74 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -1986,6 +1986,19 @@ int dump_leaf_key(void* key_arg, void* item_arg); C_MODE_END +/* + Says that a group was cut to fit group_concat_max_len, naming the row + the cut fell on. 'fname' is written into the message ahead of a ')' + the message supplies itself, so it is the function's name with its + opening bracket already on it. + + Shared with the JSON object aggregate, which cuts its own group and + has to say so the same way, having no GROUP_CONCAT machinery under it + to say it. +*/ +void report_cut_value_error(THD *thd, uint row_count, const char *fname); + + class Item_func_group_concat : public Item_sum { protected: @@ -2070,6 +2083,14 @@ class Item_func_group_concat : public Item_sum { return f->val_str(tmp, key + offset); } virtual void cut_max_length(String *result, uint old_length, uint max_length) const; + /* + How much of the length limit is spoken for by something that is not + in the buffer yet. What this accumulates is the whole of what + GROUP_CONCAT returns, so nothing; a subclass that writes anything + round the group once it is asked for is handing those bytes back as + well, and the limit is on the bytes returned. + */ + virtual uint32 reserved_result_length() const { return 0; } bool uses_non_standard_aggregator_for_distinct() const override { return distinct; } diff --git a/sql/json_table.cc b/sql/json_table.cc index e5ac60085bf19..97679d9755858 100644 --- a/sql/json_table.cc +++ b/sql/json_table.cc @@ -347,7 +347,15 @@ int ha_json_table::rnd_init(bool scan) Json_table_nested_path &p= m_jt->m_nested_path; DBUG_ENTER("ha_json_table::rnd_init"); - if ((m_js= m_jt->m_json->val_str(&m_tmps))) + /* + The scan is told where the document stands and reads from there + until it runs out of rows, so the bytes it is given have to outlive + everything the query does between one row and the next. Working + out a row can write to whatever the document was read from - a + package body's variable is the plain case - so the document is + asked for the way that returns bytes of the reader's own. + */ + if ((m_js= m_jt->m_json->val_json(&m_tmps))) { p.scan_start(m_js->charset(), (const uchar *) m_js->ptr(), (const uchar *) m_js->end()); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e980b79f5c5d5..0c1e348877881 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7691,6 +7691,9 @@ SHOW_VAR status_vars[]= { {"Handler_tmp_write", (char*) offsetof(STATUS_VAR, ha_tmp_write_count), SHOW_LONG_STATUS}, {"Handler_update", (char*) offsetof(STATUS_VAR, ha_update_count), SHOW_LONG_STATUS}, {"Handler_write", (char*) offsetof(STATUS_VAR, ha_write_count), SHOW_LONG_STATUS}, +#ifndef DBUG_OFF + {"Json_scans", (char*) offsetof(STATUS_VAR, json_scans), SHOW_LONG_STATUS}, +#endif SHOW_FUNC_ENTRY("Key", &show_default_keycache), {"optimizer_join_prefixes_check_calls", (char*) offsetof(STATUS_VAR, optimizer_join_prefixes_check_calls), SHOW_LONG_STATUS}, {"Last_query_cost", (char*) offsetof(STATUS_VAR, last_query_cost), SHOW_DOUBLE_STATUS}, diff --git a/sql/select_handler.cc b/sql/select_handler.cc index b0b8e58623de7..21126fb631780 100644 --- a/sql/select_handler.cc +++ b/sql/select_handler.cc @@ -64,6 +64,9 @@ TABLE *select_handler::create_tmp_table(THD *thd, SELECT_LEX *select) (ORDER *) 0, false, 0, TMP_TABLE_ALL_COLUMNS, 1, &empty_clex_str, true, false); + /* The engine the select was pushed down to writes the rows of this one. */ + if (table) + table->set_filled_by_engine(); DBUG_RETURN(table); } @@ -75,8 +78,28 @@ bool select_handler::prepare() Some engines (e.g. XPand) initialize "table" on their own. So we need to create a temporary table only if "table" is NULL. */ - if (!table && !(table= create_tmp_table(thd, select))) - DBUG_RETURN(true); + if (!table) + { + if (!(table= create_tmp_table(thd, select))) + DBUG_RETURN(true); + } + else + { + /* + A table the engine brought with it is a table the engine fills, so + it drops the attestations about its columns for the same reason + one built here does - see TABLE::set_filled_by_engine(). Done on + this arm too, and not only where the table is built, because what + has to hold is that EVERY way in has dropped them. + + No engine in the tree takes this arm, every one of them leaving + the table to be built above, so nothing here is exercised by a + test. It is written all the same: the arm exists for engines that + are not in the tree, and an answer that is only right because + nobody takes the arm is not an answer. + */ + table->set_filled_by_engine(); + } DBUG_RETURN(table->fill_item_list(&result_columns)); } @@ -128,6 +151,15 @@ int select_handler::execute() DBUG_ENTER("select_handler::execute"); + /* + Nothing attests to what an engine puts in a table it fills, so the + bitmaps are dropped where the table is given to the engine - see + TABLE::set_filled_by_engine(). Asserted here rather than there + because what has to hold is that EVERY way in has dropped them, and + the ways in all come through here. + */ + DBUG_ASSERT(!table->is_valid_json_static_set); + if ((err= init_scan())) goto error; diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 9f1dcec69a067..27be865b78ced 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -10758,3 +10758,13 @@ ER_CM_OPTION_MISSING_REQUIREMENT eng "CHANGE MASTER TO option '%s=%s' is missing requirement %s" ER_SLAVE_STATEMENT_TIMEOUT 70100 eng "Slave log event execution was interrupted (slave_max_statement_time exceeded)" +# A number a later series has already given out would mean one number +# with two meanings, so the next one starts above every number assigned +# in any of them +start-error-number 4269 +ER_WARN_JSON_VALID_OTHER_COLUMN + eng "CHECK constraint of column '%-.192s' calls JSON_VALID() on something other than '%-.192s'; the column is not a JSON column" +ER_WARN_JSON_VALID_NOT_REQUIRED + eng "CHECK constraint of column '%-.192s' can pass without JSON_VALID() holding; the column is not a JSON column" +ER_WARN_JSON_VALID_NOT_ALONE + eng "CHECK constraint of column '%-.192s' asks more than JSON_VALID() of it; the column is not a JSON column" diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 160d7d4776d60..34b52b0badb20 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -9155,6 +9155,17 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, if (fields.elements) table_arg->auto_increment_field_not_null= FALSE; + /* + The marks are about the row image built here and no other. Clearing + them as the image is begun, rather than trusting something later to + clear them once it is done with, is what keeps them from being read + with the NEXT row: a row whose writing ends before anything reads what + was written leaves its marks standing - an update that finds nothing to + change never reaches the reading, and a view whose own check refuses + the row returns before it. + */ + table_arg->clear_is_valid_json_marks(); + while ((fld= f++)) { field= fld->field_for_view_update(); @@ -9186,12 +9197,15 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, { if (!skip_sys_field) { - if (value->save_in_field(rfield, 0) < 0 && !ignore_errors) + int store_rc= value->save_in_field(rfield, 0); + if (store_rc < 0 && !ignore_errors) { my_message(ER_UNKNOWN_ERROR, ER_THD(thd, ER_UNKNOWN_ERROR), MYF(0)); goto err_unwind_fields; } rfield->set_has_explicit_value(); + if (table->has_own_json_valid_check) + rfield->set_is_valid_json(value, store_rc); } /* In sql MODE_SIMULTANEOUS_ASSIGNMENT, @@ -9434,6 +9448,10 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List &values, only one row. */ table->auto_increment_field_not_null= FALSE; + + /* See the same call in the other fill_record() above. */ + table->clear_is_valid_json_marks(); + while ((field = *ptr++) && ! thd->is_error()) { /* Ensure that all fields are from the same table */ @@ -9469,10 +9487,23 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List &values, continue; if (use_value) + { + /* + Nothing said how the store went, so nothing can be attested - + and there is nothing to take back either. The marks were cleared + where this row image was begun, a few lines above, and this route + sets none. + */ value->save_val(field); + } else - if (value->save_in_field(field, 0) < 0) + { + int store_rc= value->save_in_field(field, 0); + if (store_rc < 0) goto err; + if (table->has_own_json_valid_check) + field->set_is_valid_json(value, store_rc); + } field->set_has_explicit_value(); } /* Update virtual fields if there wasn't any errors */ diff --git a/sql/sql_class.h b/sql/sql_class.h index 20b3385b0582d..81aa78226c0ea 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -997,6 +997,22 @@ typedef struct system_status_var ulong filesort_scan_count_; ulong filesort_pq_sorts_; ulong optimizer_join_prefixes_check_calls; +#ifndef DBUG_OFF + /* + How many times the JSON functions have started the parser, over whole + documents and over single spliced values alike. + + It counts work rather than results, and that is the point of it: the + JSON functions are meant to give the same answers however many times + they read a value, so the number of readings is the one thing a test + can watch that is allowed to change. Nothing else would show a + reading that stopped happening. + + Debug builds only. There is nothing here for a released server to + offer anybody. + */ + ulong json_scans; +#endif /* Features used */ ulong feature_custom_aggregate_functions; /* +1 when custom aggregate @@ -8232,15 +8248,25 @@ class Type_holder: public Sql_alloc, { const TYPELIB *m_typelib; bool m_maybe_null; + bool m_is_valid_json_static; public: Type_holder() :m_typelib(NULL), - m_maybe_null(false) + m_maybe_null(false), + m_is_valid_json_static(false) { } void set_type_maybe_null(bool maybe_null_arg) override { m_maybe_null= maybe_null_arg; } bool get_maybe_null() const { return m_maybe_null; } + /* + Whether every producer gathered here returns a document each time + it is evaluated. One column is written by all of them and read as + one, so what can be said about it is what they all say - and an + answer about no producers at all is about nothing, which is why + having gathered none is a no rather than a yes about an empty set. + */ + bool get_is_valid_json_static() const { return m_is_valid_json_static; } decimal_digits_t decimal_precision() const override { @@ -8266,8 +8292,28 @@ class Type_holder: public Sql_alloc, bool aggregate_attributes(THD *thd) { static LEX_CSTRING union_name= { STRING_WITH_LEN("UNION") }; + m_is_valid_json_static= arg_count > 0; for (uint i= 0; i < arg_count; i++) + { m_maybe_null|= args[i]->maybe_null(); + /* + What this producer always answers, and not what the column's + store will keep of it. Only the first of those can be asked + here: the set the column will be in is settled by the call + below, so there is nothing yet to hold a branch's own set up + against - and a branch made to move into a set that keeps its + bytes without keeping its characters passes a document that + nothing reads as one, however faithfully it attests to what it + wrote. + + Asked at the store instead, where both sets are in hand - see + Field::confirm_is_valid_json_static(). Asked in both places it + would be one rule written twice, and this is the copy that could + be wrong: the field is made by create_tmp_field(), which is free + to land somewhere other than the set agreed on here. + */ + m_is_valid_json_static&= args[i]->is_valid_json_static(); + } return type_handler()->Item_hybrid_func_fix_attributes(thd, union_name, this, this, diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 8df98b09a7eea..49ae781b8a49f 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -3119,6 +3119,24 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) bitmaps_used*share->column_bitmap_size), share->fields); } + /* + Unlike has_value_set above this one is always initialised: it is read + for every row that is written, and the TABLE this was memcpy'd from + would otherwise leave it pointing at the client thread's bitmap. + The fourth block of the four allocated above is its own. + */ + my_bitmap_init(©->is_valid_json_set, + (my_bitmap_map*) (bitmap + + (bitmaps_used + 1)*share->column_bitmap_size), + share->fields); + /* + The table this was copied from is a base table and so has none, but + the copy is made by memcpy and would carry the pointers over rather + than have any storage of its own. Said outright, so that a table + which is not the server's own can never come to attest to what is + written into it. + */ + copy->set_filled_by_engine(); copy->tmp_set.bitmap= 0; // To catch errors bzero((char*) bitmap, share->column_bitmap_size * bitmaps_used); copy->read_set= ©->def_read_set; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index f0ef090d25059..f4428ca10fcc9 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3797,6 +3797,13 @@ bool JOIN::make_aggr_tables_info() /* Give storage engine access to temporary table */ gbh->table= table; + /* + And it is the engine that writes its rows - see + TABLE::set_filled_by_engine(), and select_handler:: + create_tmp_table(), which says the same of the table it + builds out of the same call. + */ + table->set_filled_by_engine(); pushdown_query->store_data_in_temp_table= need_tmp; pushdown_query->having= having; @@ -20781,6 +20788,9 @@ setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps, uint field_count) bitmaps+= bitmap_size; my_bitmap_init(&table->has_value_set, (my_bitmap_map*) bitmaps, field_count); + bitmaps+= bitmap_size; + my_bitmap_init(&table->is_valid_json_set, + (my_bitmap_map*) bitmaps, field_count); /* write_set and all_set are copies of read_set */ table->def_write_set= table->def_read_set; table->s->all_set= table->def_read_set; @@ -20893,6 +20903,11 @@ TABLE *Create_tmp_table::start(THD *thd, Field **reg_field; uint *blob_field; key_part_map *const_key_parts; + MY_BITMAP *json_static_valid_map; + uchar *json_static_valid_bits; + MY_BITMAP *json_static_nice_map; + uchar *json_static_nice_bits; + uint *json_static_depth; /* Treat sum functions as normal ones when loose index scan is used. */ m_save_sum_fields|= param->precomputed_group_by; DBUG_ENTER("Create_tmp_table::start"); @@ -20990,6 +21005,11 @@ TABLE *Create_tmp_table::start(THD *thd, &m_group_buff, (m_group && ! m_using_unique_constraint ? param->group_length : 0), &m_bitmaps, bitmap_buffer_size(field_count)*6, + &json_static_valid_map, sizeof(MY_BITMAP), + &json_static_valid_bits, bitmap_buffer_size(field_count), + &json_static_nice_map, sizeof(MY_BITMAP), + &json_static_nice_bits, bitmap_buffer_size(field_count), + &json_static_depth, sizeof(uint) * field_count, &const_key_parts, sizeof(*const_key_parts), NullS)) { @@ -21010,6 +21030,7 @@ TABLE *Create_tmp_table::start(THD *thd, bzero((char*) m_from_field, sizeof(Field*) * field_count); /* const_key_parts is used in sort_and_filter_keyuse */ bzero((char*) const_key_parts, sizeof(*const_key_parts)); + bzero((char*) json_static_depth, sizeof(uint) * field_count); table->mem_root= own_root; mem_root_save= thd->mem_root; @@ -21024,6 +21045,35 @@ TABLE *Create_tmp_table::start(THD *thd, table->temp_pool_slot= m_temp_pool_slot; table->copy_blobs= 1; table->in_use= thd; + + /* + A table built here is the server's own, and each of its fields has one + producer, known while the field is being made. So this is a table + that can attest to what will be written into it, and the bitmap + being here at all is what says so - see + TABLE::is_valid_json_static_set. It is sized for every field the + table may come to have, the fields being added after this. + */ + table->is_valid_json_static_set= json_static_valid_map; + my_bitmap_init(table->is_valid_json_static_set, + (my_bitmap_map*) json_static_valid_bits, field_count); + /* + The formatting of what is written there, kept beside it and made here + for the same reason - see TABLE::is_nice_json_static_set. The two + are made together and given up together, so nothing has to ask which + of them a table has. + */ + table->is_nice_json_static_set= json_static_nice_map; + my_bitmap_init(table->is_nice_json_static_set, + (my_bitmap_map*) json_static_nice_bits, field_count); + /* + And how deep it goes, kept beside the two of them and for the same + reason - see TABLE::json_static_depth. An entry is written when the + field it belongs to is added, so what is here now is only what a + field never added would have been read as, and nothing reads one of + those. + */ + table->json_static_depth= json_static_depth; table->no_rows_with_nulls= param->force_not_null_cols; table->group_concat= param->group_concat; table->expr_arena= thd; @@ -21141,6 +21191,7 @@ bool Create_tmp_table::add_fields(THD *thd, Item_field(thd, new_field))) goto err; ((Item_field*) tmp_item)->set_refers_to_temp_table(); + Item *producer= arg; arg= sum_item->set_arg(i, thd, tmp_item); thd->mem_root= &table->mem_root; @@ -21149,6 +21200,13 @@ bool Create_tmp_table::add_fields(THD *thd, m_field_count[current_counter]++; m_uneven_bit[current_counter]+= (m_uneven_bit_length - uneven_delta); + /* + Said after add_field() for the reason given at the other such + call below. The item that fills the field is kept aside above, + set_arg() having by now put the replacement in its place. + */ + new_field->set_is_valid_json_static(producer); + if (!(new_field->flags & NOT_NULL_FLAG)) { /* @@ -21234,6 +21292,13 @@ bool Create_tmp_table::add_fields(THD *thd, m_field_count[current_counter]++; m_uneven_bit[current_counter]+= (m_uneven_bit_length - uneven_delta); + /* + Said here rather than where the field was made, add_field() being + where the field learns its own number and so the first place it + can be spoken about. + */ + new_field->set_is_valid_json_static(item); + if (item->marker == MARKER_NULL_KEY && item->maybe_null()) { m_group_null_items++; @@ -21889,6 +21954,37 @@ bool Virtual_tmp_table::init(uint field_count) }; +/* + Where a stored program's variables keep what was said about the values + in them, and the only place one is left - see TABLE::json_held_marks. + + Given to the tables that hold such variables and to no others. Three + callers build a table through the init() above, and only one of them + builds one out of a stored program's variable definitions - the other + two are the table SUM(DISTINCT) counts in and the one row-based + replication converts through, neither of which holds anything anybody + assigns to. Both ways a stored program gets one come through that + caller, the row a cursor is read into among them, so there is no table + of this kind that has to be given room later. + + Said to be nothing before any of it is read, this being room off a + memory root and not a fresh page. +*/ + +bool Virtual_tmp_table::init_json_held_marks(uint field_count) +{ + DBUG_ENTER("Virtual_tmp_table::init_json_held_marks"); + if (!(json_held_marks= (Json_result_marks *) + alloc_root(in_use->mem_root, + field_count * + sizeof(Json_result_marks)))) + DBUG_RETURN(true); + for (uint i= 0; i < field_count; i++) + json_held_marks[i].clear(); + DBUG_RETURN(false); +} + + bool Virtual_tmp_table::add(List &field_list) { /* Create all fields and calculate the total length of record */ @@ -22011,7 +22107,19 @@ bool Virtual_tmp_table::sp_set_all_fields_from_item_list(THD *thd, for (uint i= 0 ; (item= it++) ; i++) { if (field[i]->sp_prepare_and_store_item(thd, &item)) + { + /* + The members after the one that failed were never reached, so what + they hold is what the assignment before this one left and what + was said about it then - an answer about to be read beside + members this assignment did write. Those go with the assignment + that did not finish. The ones already written keep theirs: each + of them went through the funnel and was attested to there, a + member being attested to one at a time. + */ + clear_json_held_marks_from(i); return true; + } } return false; } @@ -22024,7 +22132,11 @@ bool Virtual_tmp_table::sp_set_all_fields_from_item(THD *thd, Item *value) for (uint i= 0; i < value->cols(); i++) { if (field[i]->sp_prepare_and_store_item(thd, value->addr(i))) + { + /* A half-written row, as above. */ + clear_json_held_marks_from(i); return true; + } } return false; } @@ -28442,7 +28554,17 @@ copy_fields(TMP_TABLE_PARAM *param) DBUG_ASSERT((ptr != NULL && end >= ptr) || (ptr == NULL && end == NULL)); for (; ptr != end; ptr++) + { (*ptr->do_copy)(ptr); + /* + The other route that goes around field_conv() - see there. Whether + there is anything to confirm was settled where the entry was made: + an entry set up by Copy_field::set(uchar*, Field*) records neither + field and answers no, both kinds of entry sharing this array. + */ + if (unlikely(ptr->to_needs_confirm)) + ptr->to_field->confirm_is_valid_json_static_from(ptr->from_field); + } List_iterator_fast it(param->copy_funcs); Item_copy *item; diff --git a/sql/sql_select.h b/sql/sql_select.h index 1c607883be91e..317bef682553b 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2275,6 +2275,17 @@ class Virtual_tmp_table: public TABLE */ bool init(uint field_count); + /** + Give the table room to keep what was said about the value in each + of its members - see TABLE::json_held_marks. Only for a table that + holds a stored program's variables; the others init() builds hold + nothing anybody assigns to. + @param field_count - The number of fields we plan to add to the table. + @returns false - on success. + @returns true - on error. + */ + bool init_json_held_marks(uint field_count); + /** Add one Field to the end of the field array, update members: s->reclength, s->fields, s->blob_fields, s->null_fuelds. @@ -2318,10 +2329,25 @@ class Virtual_tmp_table: public TABLE */ bool open(); + /* + What was said about the value in each member, given up together. + Whatever writes a whole row at once writes it outside the funnel a + member is otherwise written through, and a member nobody wrote this + time is still holding what the last assignment left - so an answer + that was about that is taken back here rather than left standing + over bytes a later reader will pair it with. + */ + void clear_json_held_marks_from(uint first) + { + for (uint i= first; i < s->fields; i++) + field[i]->clear_json_held_marks(); + } + void clear_all_json_held_marks() { clear_json_held_marks_from(0); } void set_all_fields_to_null() { for (uint i= 0; i < s->fields; i++) field[i]->set_null(); + clear_all_json_held_marks(); } /** Set all fields from a compatible item list. @@ -2400,6 +2426,7 @@ create_virtual_tmp_table(THD *thd, List &field_list) DBUG_SET("+d,simulate_out_of_memory");); if (table->init(field_list.elements) || + table->init_json_held_marks(field_list.elements) || table->add(field_list) || table->open()) { diff --git a/sql/sql_string.cc b/sql/sql_string.cc index f5edb1a5414d9..0e761fdf9c897 100644 --- a/sql/sql_string.cc +++ b/sql/sql_string.cc @@ -1118,10 +1118,7 @@ String_copier::well_formed_copy(CHARSET_INFO *to_cs, const char *from, size_t from_length, size_t nchars) { - if ((to_cs == &my_charset_bin) || - (from_cs == &my_charset_bin) || - (to_cs == from_cs) || - my_charset_same(from_cs, to_cs)) + if (conversion_copies_bytes(to_cs, from_cs)) { m_cannot_convert_error_pos= NULL; return (uint) to_cs->copy_fix(to, to_length, from, from_length, diff --git a/sql/sql_string.h b/sql/sql_string.h index f0977dc0dff9a..3ddfa4d32107f 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -101,6 +101,38 @@ class String_copier: public String_copy_status, protected MY_STRCONV_STATUS { public: + /* + Whether a copy between these two character sets moves the bytes + across as they stand, rather than writing the characters again in + the destination's formatting. + + This is the branch well_formed_copy() takes, said where a caller can + read it as well. Sets that are the same are copied unchanged, and so + is anything with a binary set at either end - the bytes go across + untouched and are then read as whatever the other set makes of them. + */ + static bool conversion_copies_bytes(CHARSET_INFO *to_cs, + CHARSET_INFO *from_cs) + { + return to_cs == &my_charset_bin || from_cs == &my_charset_bin || + to_cs == from_cs || my_charset_same(from_cs, to_cs); + } + /* + Whether such a copy still puts down the characters it was given. + + Sets that are the same put them down unchanged and different sets + convert them, both of which keep the characters. A binary set at + either end is neither: the bytes are kept and called by the other + set's name, which is a different string of characters and, for a + document, usually not one at all. That is the one case the branch + above takes without converting, which is what this asks. + */ + static bool conversion_keeps_characters(CHARSET_INFO *to_cs, + CHARSET_INFO *from_cs) + { + return my_charset_same(from_cs, to_cs) || + !conversion_copies_bytes(to_cs, from_cs); + } const char *cannot_convert_error_pos() const { return m_cannot_convert_error_pos; } const char *most_important_error_pos() const diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc index 3fa4675574c48..deeaa1b4c0233 100644 --- a/sql/sql_tvc.cc +++ b/sql/sql_tvc.cc @@ -292,7 +292,9 @@ bool table_value_constr::prepare(THD *thd, SELECT_LEX *sl, Item_type_holder *new_holder= new (thd->mem_root) Item_type_holder(thd, item, holders[pos].type_handler(), &holders[pos]/*Type_all_attributes*/, - holders[pos].get_maybe_null()); + holders[pos].get_maybe_null(), + holders[pos]. + get_is_valid_json_static()); sl->item_list.push_back(new_holder); } if (arena) diff --git a/sql/sql_type_json.cc b/sql/sql_type_json.cc index 27072de2d559c..885f9cc6d3754 100644 --- a/sql/sql_type_json.cc +++ b/sql/sql_type_json.cc @@ -107,13 +107,214 @@ Type_handler_json_common::json_type_handler_sum(const Item_sum *item) } +/** + The column a JSON_VALID() call is about, where the expression is that + call and nothing else. + + NULL for anything else at all: another function, the call with more + than the one argument it takes, or an argument that is not a column. + Both questions below are that shape plus a way of naming the column, + so the shape is asked once. + + RESOLVED says whether the expression has been fixed. It has, wherever + there is a Field to compare against, and the argument is then reached + through Item::real_item() so that a reference standing in for the + column is followed to it; it has not while a table is being defined, + where following would be reaching through a pointer nothing has set. +*/ + +static const Item_field *json_valid_single_field_arg(const Item *expr, + bool resolved) +{ + const Item_func *func; + const Item *arg; + + if (!expr || expr->type() != Item::FUNC_ITEM) + return NULL; + func= static_cast(expr); + if (func->functype() != Item_func::JSON_VALID_FUNC || + func->argument_count() != 1) + return NULL; + arg= func->arguments()[0]; + if (resolved) + arg= arg->real_item(); + if (arg->type() != Item::FIELD_ITEM) + return NULL; + return static_cast(arg); +} + + +/** + Whether an expression asks nothing but whether one particular column + holds a document. + + The COLUMN, and not the field object standing for it. A table can + have more than one of those for the same column: the second and third + row images a trigger reads through OLD. and NEW. are reached through + copies made by Field::make_new_field(), which memdups the field and + points it at the other buffer. A copy carries the same check + constraint - the same pointer, to an expression that still names the + field it was fixed against - so comparing the objects makes every + copy of a JSON column stop being one, and what it holds is quoted + into a document instead of going in as a document. + + Same table and same position is the whole of the question and gives + nothing away. A column check can only read columns of its own table, + so a check that names a different position is a check about another + column - which is the thing this asks in order to refuse - and one + that names the same position in another table is a copy that was put + somewhere else, where the constraint it brought along says nothing. +*/ + +bool Type_handler_json_common::is_json_valid_of_field_expr(const Item *expr, + const Field *field) +{ + const Item_field *arg= json_valid_single_field_arg(expr, true); + + return arg && arg->field->table == field->table && + arg->field->field_index == field->field_index; +} + + +/** + Whether a column carries a check constraint that says that column holds + a document, which is what makes it a JSON column. + + The question has to be about this column: a constraint reading another + column, or none at all, promises nothing about what is in this one, and + a column typed JSON on such a promise has its contents put into + documents verbatim rather than quoted. + + A constraint the server wrote for a temporary table column carries no + vcol type, so there is nothing to ask about it beyond the expression; + a constraint belonging to the table rather than to a column never + reaches a column's check_constraint in the first place. +*/ + bool Type_handler_json_common::has_json_valid_constraint(const Field *field) { return field->check_constraint && - field->check_constraint->expr && - field->check_constraint->expr->type() == Item::FUNC_ITEM && - static_cast(field->check_constraint->expr)-> - functype() == Item_func::JSON_VALID_FUNC; + is_json_valid_of_field_expr(field->check_constraint->expr, field); +} + + +/** + The same question about a check constraint read from a table + definition, where it also has to be the column's own check rather than + the table's before its result can be trusted. +*/ + +bool Type_handler_json_common::is_json_valid_of_field(Virtual_column_info *check, + const Field *field) +{ + return check && check->get_vcol_type() == VCOL_CHECK_FIELD && + is_json_valid_of_field_expr(check->expr, field); +} + + +/** + The same question asked while a table is being defined. + + There the expression has not been fixed yet, so there is no Field to + compare against and the column is recognised by name instead. Do not + fold this into is_json_valid_of_field_expr(): that one runs on an open + table, where names have already been resolved and the identity of the + Field is the stronger answer. +*/ + +bool Type_handler_json_common::is_json_valid_of_name(const Item *expr, + const LEX_CSTRING &name) +{ + const Item_field *arg= json_valid_single_field_arg(expr, false); + + return arg && !my_strcasecmp(system_charset_info, arg->field_name.str, + name.str); +} + + +/** + Whether a check constraint cannot pass without the named column holding + a document. + + A conjunction has to hold in full, so a JSON_VALID() anywhere among its + parts is required; anywhere else in an expression it may be what the + constraint passes without. +*/ + +static bool json_valid_of_name_required(Item *expr, const LEX_CSTRING &name) +{ + Item_cond *cond; + List_iterator_fast li; + Item *item; + + if (Type_handler_json_common::is_json_valid_of_name(expr, name)) + return true; + if (!expr || expr->type() != Item::COND_ITEM) + return false; + cond= static_cast(expr); + if (cond->functype() != Item_func::COND_AND_FUNC) + return false; + li.init(*cond->argument_list()); + while ((item= li++)) + { + if (json_valid_of_name_required(item, name)) + return true; + } + return false; +} + + +/** + Say so when a check constraint mentions JSON_VALID() but leaves the + column an ordinary one. + + Writing JSON_VALID() in a column's check constraint is how a JSON column + is asked for, so a constraint that mentions it and still does not make + the column a JSON column is worth reporting: it reads something other + than this column, or it can be satisfied without the call holding, or it + asks something else as well. All three are quiet, and until the column + is used in a document none of them shows. + + What is asked here is what types the column and nothing weaker. A + conjunction cannot pass unless every part of it does, so JSON_VALID() + inside one is a promise that really is kept - but a promise kept is not + the same thing as a column typed, and has_json_valid_constraint() wants + the call and nothing else. Asking whether the promise is kept, which is + what this used to ask, left the commonest shape of all - JSON_VALID(c) + AND something - typed as text, warned about not at all, and reported to + the client as a JSON column by the protocol. +*/ + +void Type_handler_json_common::warn_if_json_valid_does_not_type( + THD *thd, + Virtual_column_info *check, + const LEX_CSTRING &name) +{ + Item_func::Functype json_valid= Item_func::JSON_VALID_FUNC; + + if (!check || !check->expr || is_json_valid_of_name(check->expr, name)) + return; + + if (check->expr->walk(&Item::json_valid_of_column_processor, 0, + const_cast(&name))) + { + /* + The call is there and it is about this column, so what stands in the + way is one of two things, and they are worth telling apart: the + constraint can be satisfied without the call holding, or it cannot + and the constraint is still not the call alone. Only the second is + a constraint that does what its writer meant and is refused anyway. + */ + uint code= json_valid_of_name_required(check->expr, name) ? + ER_WARN_JSON_VALID_NOT_ALONE : ER_WARN_JSON_VALID_NOT_REQUIRED; + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, code, + ER_THD(thd, code), name.str); + } + else if (check->expr->walk(&Item::find_function_processor, 0, &json_valid)) + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + ER_WARN_JSON_VALID_OTHER_COLUMN, + ER_THD(thd, ER_WARN_JSON_VALID_OTHER_COLUMN), + name.str, name.str); } diff --git a/sql/sql_type_json.h b/sql/sql_type_json.h index b7fe5c8aa64a3..77aae18542ef3 100644 --- a/sql/sql_type_json.h +++ b/sql/sql_type_json.h @@ -37,7 +37,16 @@ class Type_handler_json_common static const Type_handler *json_blob_type_handler_by_length_bytes(uint len); static const Type_handler *json_type_handler_sum(const Item_sum *sum); static const Type_handler *json_type_handler_from_generic(const Type_handler *th); + static bool is_json_valid_of_field_expr(const Item *expr, + const Field *field); + static bool is_json_valid_of_name(const Item *expr, + const LEX_CSTRING &name); + static void warn_if_json_valid_does_not_type(THD *thd, + Virtual_column_info *check, + const LEX_CSTRING &name); static bool has_json_valid_constraint(const Field *field); + static bool is_json_valid_of_field(Virtual_column_info *check, + const Field *field); static const Type_collection *type_collection(); static bool is_json_type_handler(const Type_handler *handler) { diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 0d87c830adf38..f5031cf86a3da 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -1261,13 +1261,22 @@ bool st_select_lex_unit::join_union_item_types(THD *thd_arg, */ bool pos_maybe_null= is_recursive ? true : holders[pos].get_maybe_null(); + /* + Only the non-recursive parts were joined above, so for a recursive + CTE the producers of the recursive parts were never asked, and the + ones that were must not answer in their place. + */ + bool pos_is_valid_json_static= is_recursive ? false : + holders[pos].get_is_valid_json_static(); + /* Error's in 'new' will be detected after loop */ types.push_back(new (thd_arg->mem_root) Item_type_holder(thd_arg, item_tmp, holders[pos].type_handler(), &holders[pos]/*Type_all_attributes*/, - pos_maybe_null)); + pos_maybe_null, + pos_is_valid_json_static)); } if (unlikely(thd_arg->is_fatal_error)) DBUG_RETURN(true); // out of memory diff --git a/sql/sql_update.cc b/sql/sql_update.cc index a50b52456474a..19b406b2c9c2e 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -3015,6 +3015,15 @@ int multi_update::do_updates() table->status|= STATUS_UPDATED; store_record(table,record[1]); + /* + A new row image begins here. Nothing on this pass goes through + fill_record() - the columns arrive by Copy_field, below - so this is + where the marks of the row before have to go; a trigger on this + table can set them, and a row that turns out to need no update + never reaches the reading that would clear them. + */ + table->clear_is_valid_json_marks(); + /* Copy data from temporary table to current table */ for (copy_field_ptr=copy_field; copy_field_ptr != copy_field_end; diff --git a/sql/table.cc b/sql/table.cc index f52f0590dea47..1b873c11d8ebd 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -42,6 +42,8 @@ #include "sql_view.h" #include "rpl_filter.h" #include "sql_cte.h" +#include "sql_type_json.h" // Type_handler_json_common +#include "item_jsonfunc.h" // Json_scans_unbilled #include "ha_sequence.h" #include "sql_show.h" #include "opt_trace.h" @@ -1256,6 +1258,17 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table, &((*field_ptr)->check_constraint), error_reported); *check_constraint_ptr++= (*field_ptr)->check_constraint; + /* + Work out once, here, whether this check asks anything besides + whether its own column holds a document. TABLE::verify_constraints + reads the answer per row and cannot afford to work it out there. + */ + if (vcol && + Type_handler_json_common::is_json_valid_of_field(vcol, *field_ptr)) + { + vcol->json_valid_field_index= (*field_ptr)->field_index; + table->has_own_json_valid_check= true; + } break; case VCOL_CHECK_TABLE: vcol= unpack_vcol_info_from_frm(thd, table, &expr_str, @@ -4461,7 +4474,7 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, /* Allocate bitmaps */ bitmap_size= share->column_bitmap_size; - bitmap_count= 7; + bitmap_count= 8; if (share->virtual_fields) bitmap_count++; @@ -4490,6 +4503,9 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, bitmaps+= bitmap_size; my_bitmap_init(&outparam->def_rpl_write_set, (my_bitmap_map*) bitmaps, share->fields); + bitmaps+= bitmap_size; + my_bitmap_init(&outparam->is_valid_json_set, + (my_bitmap_map*) bitmaps, share->fields); outparam->default_column_bitmaps(); outparam->cond_selectivity= 1.0; @@ -5861,6 +5877,12 @@ void TABLE::init(THD *thd, TABLE_LIST *tl) pos_in_table_list= tl; clear_column_bitmaps(); + /* + A cached TABLE, or one whose statement gave up between a write and the + reading of what was written, can arrive here with marks still on it. + They are about a row image that is over. + */ + clear_is_valid_json_marks(); for (Field **f_ptr= field ; *f_ptr ; f_ptr++) { (*f_ptr)->next_equal_field= NULL; @@ -6528,6 +6550,66 @@ int TABLE_LIST::view_check_option(THD *thd, bool ignore_failure) int TABLE::verify_constraints(bool ignore_failure) +{ + int rc= run_check_constraints(ignore_failure); + /* + The marks are about the bytes that are in record[0] right now, and this + is where the row image they belong to is done with. Draining them here + rather than at the callers is what makes them safe: whatever replaces + those bytes next - restore_record() between the two readings of + INSERT ... ON DUPLICATE KEY UPDATE is the sharp case - finds nothing + left over to be believed. + */ + clear_is_valid_json_marks(); + return rc; +} + + +#ifndef DBUG_OFF +/* + Reads back what a mark claimed, and stops a debug server if it was wrong. + + The mark says this column holds a document, so the check would pass and + is being left unrun. Nothing downstream ever finds out otherwise: a + wrong mark is a constraint that quietly stops being enforced, and the row + it lets through is indistinguishable from a row that passed. This is the + last place that can still tell the difference, so a debug build spends + the reading here to tell it. + + It is worth the cost because of the shape of the thing it polices. The + marks are SET at three places, which is a closed list, but they are + CLEARED at every place that writes record[0] without going through those + three - and that list is open. A site added later that forgets to clear + produces no wrong answer, no warning and no failing test: it produces a + check that silently stops running. With this here it produces an abort + on the first row that reaches it. + + The reading is taken back off Json_scans afterwards. It is the debug + build's work rather than the server's, and counting it would move a + number kept to watch the work a released server does. +*/ + +void TABLE::check_json_valid_mark(Virtual_column_info *check) +{ + bool held; + + /* + An error already standing makes the reading say nothing about the + mark, only about the error. + */ + if (in_use->is_error()) + return; + + { + Json_scans_unbilled unbilled(in_use); + held= check->expr->val_bool() || check->expr->null_value; + } + DBUG_ASSERT(held); +} +#endif + + +int TABLE::run_check_constraints(bool ignore_failure) { /* We have to check is_error() first as we are checking it for each @@ -6546,6 +6628,19 @@ int TABLE::verify_constraints(bool ignore_failure) StringBuffer field_error(system_charset_info); for (Virtual_column_info **chk= check_constraints ; *chk ; chk++) { + /* + A check that asks nothing but whether its own column holds a + document has nothing to find out where that column was written + from an item that already attested to the value. + */ + if ((*chk)->json_valid_field_index != NO_JSON_VALID_FIELD && + bitmap_is_set(&is_valid_json_set, (*chk)->json_valid_field_index)) + { +#ifndef DBUG_OFF + check_json_valid_mark(*chk); +#endif + continue; + } /* yes! NULL is ok. see 4.23.3.4 Table check constraints, part 2, SQL:2016 @@ -9179,6 +9274,13 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) # endif vcol_info->expr->save_in_field(vf, 0); DBUG_RESTORE_WRITE_SET(vf); + /* + Whatever was put into this field before, what stands there now came + from the column's own expression and nobody has attested to it. + A value assigned to a stored generated column is taken and then + written over exactly here. + */ + vf->clear_is_valid_json(); DBUG_PRINT("info", ("field '%s' - updated error: %d", vf->field_name.str, field_error)); if (swap_values && (vf->flags & BLOB_FLAG)) @@ -9238,6 +9340,8 @@ int TABLE::update_virtual_field(Field *vf, bool ignore_warnings) DBUG_FIX_WRITE_SET(vf); vf->vcol_info->expr->save_in_field(vf, 0); DBUG_RESTORE_WRITE_SET(vf); + /* See the same call in update_virtual_fields() above. */ + vf->clear_is_valid_json(); in_use->restore_active_arena(expr_arena, &backup_arena); in_use->pop_internal_handler(); if (ignore_warnings) diff --git a/sql/table.h b/sql/table.h index 6b8ebae10dcb6..90afa16549bf9 100644 --- a/sql/table.h +++ b/sql/table.h @@ -76,6 +76,7 @@ class Table_triggers_list; class TMP_TABLE_PARAM; class SEQUENCE; class Range_rowid_filter_cost_info; +class Json_result_marks; class derived_handler; class Pushdown_derived; struct Name_resolution_context; @@ -1355,6 +1356,135 @@ struct TABLE MY_BITMAP *read_set, *write_set, *rpl_write_set; /* On INSERT: fields that the user specified a value for */ MY_BITMAP has_value_set; + /* + Fields whose bytes in record[0] were written from an Item that answered + is_valid_json(), by a store that kept every character of it. A + json_valid() check over such a field can only find out what is already + known, so verify_constraints() leaves it unread. + + The bits last one row image and no longer. verify_constraints() reads + them and clears the whole bitmap on every way out of itself, and + TABLE::init() clears it at the start of a statement. Whatever writes + record[0] without setting a bit therefore leaves it clear, and the + check runs - which is what every writer other than the three store + sites does. + */ + MY_BITMAP is_valid_json_set; + /* + Whether any column of this table carries a check that asks nothing but + whether that same column holds a document - which is the only thing the + bitmap above is ever read for. + + Worked out once, where the checks are worked out, because the writers + would otherwise ask a field about every column of every row on the + chance that one of them was constrained. Where no column is, no bit + can ever be set and nothing reads one. + */ + bool has_own_json_valid_check; + /* + Fields of a temporary table the server built for itself, every value + of which came from an item that answers is_valid_json_static() and + was put down whole. A reader of such a field is reading a document, + whatever row it has in front of it. + + Unlike the bitmap above, these bits say nothing about one row image. + They are written once, while the table is being built and each + field's one producer is known, and they hold for every row the field + will ever be given - which is why a reader can still believe one long + after the row was written and the row image it was written in is + gone. + + A bit is taken back, for good, if a store into the field ever puts + down less than it was handed. That is the one way the promise could + come apart, and it is not a way the store will own up to: truncation + is reported only where count_cuted_fields is raised, which is not + where a temporary table is written. So the store is asked what it + kept rather than whether it minded. + + Only Create_tmp_table leaves a bitmap here. A base table and a table + the user asked for have nowhere to keep a yes, and so answer no by + construction - which is the whole of why their columns stay unread: + a row can hold bytes no check ever saw, and nothing in one records + where it came from. + */ + MY_BITMAP *is_valid_json_static_set; + /* + Of those same fields, the ones every value has so far reached + formatted the way json_nice() writes a document in its loose form. + + This one cannot be promised in advance the way the one above is. + Being a document is a property of the producer, which is why it can + be asked before a row exists; the FORMATTING is a property of each + value the producer passes, and there is nobody to ask about a + value that has not been made yet. So the bit starts as a yes + wherever the bit above was granted, and the first value that arrives + formatted otherwise takes it back for good. + + What it says when it is read is therefore "nothing put here so far + was formatted another way", which is what a reader of a row already + written needs of it: the value in front of that reader was put down + before the read, so a bit that still stands now also stood when that + value was written. + + Read only through the bitmap above, a formatting being nothing to say + about a column whose values are not known to be documents at all. + That is why is_valid_json_static_set alone decides whether a table + keeps these answers; this pointer is never tested on its own. + */ + MY_BITMAP *is_nice_json_static_set; + + /* + Of those same fields, how deep the deepest value that has reached + each of them goes. + + A number rather than a bit, because what reads this is composing: + it is putting the value inside something of its own and needs a + figure to add its own levels to, not a yes. The column is written + by many values and read as one, so what holds for all of them is the + deepest of them, and the entry only ever rises. + + Too large is the safe direction here, as saying no is for the two + above: a figure larger than the truth costs a reading that need not + have happened, while one smaller lets a document be composed that + goes past what this server can read back. JSON_DEPTH_UNKNOWN is + the largest of all and so absorbs everything - one value nobody + counted takes the column there and leaves it there, which is what + it means for a column to have stopped saying. + + Read through the first bitmap, like the second, a depth being + nothing to say about a column whose values are not known to be + documents at all. + */ + uint *json_static_depth; + /* + What was said about the value each field of this table is holding + right now, where the table is one a stored program keeps its + variables in. + + A variable is not a column and cannot be attested to the way the + three above attest to one. A column of a table the server built + for itself has one producer, settled while the column is being made, + so what will be written there is known before a row exists. A + variable has as many producers as there are assignments to it, each + one arriving at a different moment and saying something different, + so there is nothing to say in advance and everything to say each + time one lands. + + So these are written where a value is written, by the one funnel + every assignment goes through, and each of them is about the bytes + the field holds until the next assignment replaces both together. + That is a narrower promise than the standing one and a stronger one: + it is about a value that exists, and it can carry the formatting and + the depth exactly rather than as a bound over rows. + + Only Virtual_tmp_table leaves an array here, and it is the whole of + the discriminator, exactly as the pointer above it is: a base table + and a table the user asked for have nowhere to keep an answer and so + give none. A table a stored program keeps its variables in cannot + be named from SQL and is written by nothing but that one funnel, + which is what makes an answer about it worth keeping at all. + */ + Json_result_marks *json_held_marks; /* The ID of the query that opened and is using this table. Has different @@ -1623,6 +1753,70 @@ struct TABLE void mark_columns_used_by_virtual_fields(void); void mark_check_constraint_columns_for_read(void); int verify_constraints(bool ignore_failure); + /* + The constraint loop alone. Call verify_constraints() instead: it is + what clears is_valid_json_set afterwards, and every way out of this + one leaves the marks standing. + */ + int run_check_constraints(bool ignore_failure); + /* + Take back whatever was said about the bytes standing in record[0]. + + Asked only of a table some column of which carries a check that could + read a mark. Where none does no bit can ever be set, and the writers + below run on every row of every table in the instance: the guard is + what keeps a bitmap nothing ever reads from being walked by all of + them. + */ + void clear_is_valid_json_marks() + { + if (has_own_json_valid_check) + bitmap_clear_all(&is_valid_json_set); + } + /* + Put a whole row image into record[0]. This is what restore_record() + does, and the reason it is a method rather than the memcpy it reads + as. + + Whatever replaces a row image wholesale leaves nothing that was said + about the one before it still true, and the list of places that do it + is open where the list of places that say something is closed. Naming + them one at a time is how one gets missed, and a missed one is a check + that quietly stops being run. + */ + void restore_record_image(const uchar *from) + { + memcpy(record[0], from, (size_t) s->reclength); + clear_is_valid_json_marks(); + } + /* + Said of a table this server built and a storage engine fills: a + select, a grouping or a derived table pushed down whole borrows this + machinery for its columns and then has its rows written by the + engine, straight into record[0]. No store of ours ever sees what + goes into them, so none of the columns can attest to their values - + however well the item a column was made from attests to the values + IT makes. + + Taking the bitmaps away is how that is said, a table with nowhere to + keep a yes answering no to everything. All three go, the two behind + the first being read only through it but pointing at storage of + their own all the same. + + Said at the moment the engine is handed the table rather than when + it starts filling it: a table built out of this one asks these + columns what they promise while it is being built, which is before + any row of either exists. + */ + void set_filled_by_engine() + { + is_valid_json_static_set= NULL; + is_nice_json_static_set= NULL; + json_static_depth= NULL; + } +#ifndef DBUG_OFF + void check_json_valid_mark(Virtual_column_info *check); +#endif void free_engine_stats(); void update_engine_independent_stats(); inline void column_bitmaps_set(MY_BITMAP *read_set_arg) diff --git a/sql/unireg.h b/sql/unireg.h index fa657a267c2cb..d8369a2a32aa4 100644 --- a/sql/unireg.h +++ b/sql/unireg.h @@ -72,7 +72,7 @@ /* Extern defines */ #define store_record(A,B) memcpy((A)->B,(A)->record[0],(size_t) (A)->s->reclength) -#define restore_record(A,B) memcpy((A)->record[0],(A)->B,(size_t) (A)->s->reclength) +#define restore_record(A,B) (A)->restore_record_image((A)->B) #define cmp_record(A,B) memcmp((A)->record[0],(A)->B,(size_t) (A)->s->reclength) #define empty_record(A) { \ restore_record((A),s->default_values); \ diff --git a/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_deinit.inc b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_deinit.inc new file mode 100644 index 0000000000000..76b7582abfe49 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_deinit.inc @@ -0,0 +1,11 @@ +--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP +--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP +--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP +--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP +--disable_warnings +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_result_log +--enable_query_log +--enable_warnings diff --git a/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_init.inc b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_init.inc new file mode 100644 index 0000000000000..345c6fc501072 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/include/charset_conv_pushdown_init.inc @@ -0,0 +1,23 @@ +--disable_warnings +--disable_query_log +--disable_result_log +--source ../t/test_init.inc +--enable_result_log +--enable_query_log +--enable_warnings +--let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1 +let $MASTER_1_COMMENT_2_1= + COMMENT='table "tbl_a", srv "s_2_1"'; +--let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES +let $CHILD2_1_DROP_TABLES= + DROP TABLE IF EXISTS tbl_a; +--let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES +let $CHILD2_1_CREATE_TABLES= + CREATE TABLE tbl_a ( + pkey int NOT NULL, + txt varchar(16) NOT NULL, + PRIMARY KEY (pkey) + ) $CHILD2_1_ENGINE DEFAULT CHARACTER SET utf8; +--let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES +let $CHILD2_1_SELECT_TABLES= + SELECT pkey, txt FROM tbl_a ORDER BY pkey; diff --git a/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown,group_by_handler.rdiff b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown,group_by_handler.rdiff new file mode 100644 index 0000000000000..c49606197b51c --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown,group_by_handler.rdiff @@ -0,0 +1,11 @@ +--- charset_conv_pushdown.result 2026-08-15 16:02:58.308945369 -0400 ++++ charset_conv_pushdown.reject 2026-08-15 16:03:11.788881446 -0400 +@@ -43,7 +43,7 @@ + SELECT argument FROM mysql.general_log + WHERE command_type != 'Execute' AND argument LIKE 'select %'; + argument +-select `pkey`,`txt` from `auto_test_remote`.`tbl_a` where ((convert(`txt` using latin1)) = 'beta') ++select t0.`pkey` `pkey`,t0.`txt` `txt` from `auto_test_remote`.`tbl_a` t0 where ((convert(t0.`txt` using latin1)) = 'beta') + SELECT argument FROM mysql.general_log + WHERE command_type != 'Execute' AND argument LIKE 'select %' + diff --git a/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown.result b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown.result new file mode 100644 index 0000000000000..a827a9cf8de58 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/charset_conv_pushdown.result @@ -0,0 +1,63 @@ +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 + +# A character-set conversion in a pushed-down condition keeps its +# conversion. The data node is told which set to convert into; a +# condition that arrived there without the USING clause would +# compare the bytes as they stand and answer a different row. + +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +connection child2_1; +SET @old_log_output = @@global.log_output; +SET GLOBAL log_output = 'TABLE,FILE'; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +connection child2_1; +CHILD2_1_CREATE_TABLES +connection master_1; +CREATE TABLE tbl_a ( +pkey int NOT NULL, +txt varchar(16) NOT NULL, +PRIMARY KEY (pkey) +) ENGINE=Spider COMMENT='table "tbl_a", srv "s_2_1"' +INSERT INTO tbl_a (pkey, txt) VALUES +(1, 'alpha'), +(2, 'beta'), +(3, 'gamma'), +(4, 'delta'); +connection child2_1; +TRUNCATE TABLE mysql.general_log; +connection master_1; +SELECT pkey, txt FROM tbl_a WHERE CONVERT(txt USING latin1) = 'beta'; +pkey txt +2 beta + +# The statement the data node was sent +connection child2_1; +SELECT argument FROM mysql.general_log +WHERE command_type != 'Execute' AND argument LIKE 'select %'; +argument +select `pkey`,`txt` from `auto_test_remote`.`tbl_a` where ((convert(`txt` using latin1)) = 'beta') +SELECT argument FROM mysql.general_log +WHERE command_type != 'Execute' AND argument LIKE 'select %' + +deinit +connection master_1; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +DROP DATABASE IF EXISTS auto_test_remote; +SET GLOBAL log_output = @old_log_output; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 + +end of test diff --git a/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.cnf b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.cnf new file mode 100644 index 0000000000000..05dfd8a0bcea9 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.cnf @@ -0,0 +1,3 @@ +!include include/default_mysqld.cnf +!include ../my_1_1.cnf +!include ../my_2_1.cnf diff --git a/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.test b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.test new file mode 100644 index 0000000000000..38e651c42333e --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/charset_conv_pushdown.test @@ -0,0 +1,76 @@ +--source ../include/charset_conv_pushdown_init.inc +--echo +--echo # A character-set conversion in a pushed-down condition keeps its +--echo # conversion. The data node is told which set to convert into; a +--echo # condition that arrived there without the USING clause would +--echo # compare the bytes as they stand and answer a different row. +--echo + +--connection master_1 +--disable_warnings +CREATE DATABASE auto_test_local; +USE auto_test_local; + +--connection child2_1 +SET @old_log_output = @@global.log_output; +SET GLOBAL log_output = 'TABLE,FILE'; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +--enable_warnings + +--connection child2_1 +--disable_query_log +echo CHILD2_1_CREATE_TABLES; +eval $CHILD2_1_CREATE_TABLES; +--enable_query_log + +--connection master_1 +--disable_query_log +echo CREATE TABLE tbl_a ( + pkey int NOT NULL, + txt varchar(16) NOT NULL, + PRIMARY KEY (pkey) +) $MASTER_1_ENGINE $MASTER_1_COMMENT_2_1; +eval CREATE TABLE tbl_a ( + pkey int NOT NULL, + txt varchar(16) NOT NULL, + PRIMARY KEY (pkey) +) $MASTER_1_ENGINE $MASTER_1_COMMENT_2_1; +--enable_query_log +INSERT INTO tbl_a (pkey, txt) VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'), + (4, 'delta'); + +--connection child2_1 +TRUNCATE TABLE mysql.general_log; + +--connection master_1 +--disable_ps2_protocol +SELECT pkey, txt FROM tbl_a WHERE CONVERT(txt USING latin1) = 'beta'; +--enable_ps2_protocol + +--echo +--echo # The statement the data node was sent +--connection child2_1 +--disable_view_protocol +--disable_ps2_protocol +SELECT argument FROM mysql.general_log + WHERE command_type != 'Execute' AND argument LIKE 'select %'; +--enable_ps2_protocol +--enable_view_protocol + +--echo +--echo deinit +--disable_warnings +--connection master_1 +DROP DATABASE IF EXISTS auto_test_local; + +--connection child2_1 +DROP DATABASE IF EXISTS auto_test_remote; +SET GLOBAL log_output = @old_log_output; +--enable_warnings +--source ../include/charset_conv_pushdown_deinit.inc +--echo +--echo end of test diff --git a/strings/json_lib.c b/strings/json_lib.c index 01df72bd522a7..c0e718754f624 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -40,6 +40,7 @@ void json_string_set_cs(json_string_t *s, CHARSET_INFO *i_cs) { s->cs= i_cs; s->error= 0; + s->error_pos= NULL; s->wc= i_cs->cset->mb_wc; } @@ -123,7 +124,7 @@ typedef int (*json_state_handler)(json_engine_t *); /* The string is broken. */ static int unexpected_eos(json_engine_t *j) { - j->s.error= JE_EOS; + json_error(&j->s, JE_EOS); return 1; } @@ -131,21 +132,37 @@ static int unexpected_eos(json_engine_t *j) /* This symbol here breaks the JSON syntax. */ static int syntax_error(json_engine_t *j) { - j->s.error= JE_SYN; + json_error(&j->s, JE_SYN); return 1; } +/* + The four functions below open a nested structure, and each refuses to + open one deeper than the scanner can keep track of. + + A refusal must leave the nesting counter where it was. It indexes + 'stack', it is declared immediately after 'stack', and 'stack' is + exactly JSON_DEPTH_LIMIT long, so a counter moved past the end of the + stack addresses the counter itself: a caller that reads + j->stack[j->stack_p] then gets JSON_DEPTH_LIMIT back as though it were + a state, and acts on it. Callers are not all obliged to stop the + moment they are told the document is too deep, so this cannot be left + for them to get right - which is also why json_error() keeps the first + refusal and the reading taken at it, rather than the scanner being + stopped where it failed. +*/ + /* Value of object. */ static int mark_object(json_engine_t *j) { j->state= JST_OBJ_START; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_OBJ_CONT; + j->stack[++j->stack_p]= JST_OBJ_CONT; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -156,12 +173,12 @@ static int read_obj(json_engine_t *j) j->state= JST_OBJ_START; j->value_type= JSON_VALUE_OBJECT; j->value= j->value_begin; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_OBJ_CONT; + j->stack[++j->stack_p]= JST_OBJ_CONT; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -170,13 +187,13 @@ static int read_obj(json_engine_t *j) static int mark_array(json_engine_t *j) { j->state= JST_ARRAY_START; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_ARRAY_CONT; + j->stack[++j->stack_p]= JST_ARRAY_CONT; j->value= j->value_begin; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -186,12 +203,12 @@ static int read_array(json_engine_t *j) j->state= JST_ARRAY_START; j->value_type= JSON_VALUE_ARRAY; j->value= j->value_begin; - if (++j->stack_p < JSON_DEPTH_LIMIT) + if (j->stack_p + 1 < JSON_DEPTH_LIMIT) { - j->stack[j->stack_p]= JST_ARRAY_CONT; + j->stack[++j->stack_p]= JST_ARRAY_CONT; return 0; } - j->s.error= JE_DEPTH; + json_error(&j->s, JE_DEPTH); return 1; } @@ -264,10 +281,10 @@ static int read_4_hexdigits(json_string_t *s, uchar *dest) for (i=0; i<4; i++) { if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); if (s->c_next >= 128 || (t= json_instr_chr_map[s->c_next]) > S_F) - return s->error= JE_SYN; + return json_error(s, JE_SYN); s->c_str+= c_len; dest[i/2]+= (i % 2) ? t : t*16; @@ -281,7 +298,7 @@ static int json_handle_esc(json_string_t *s) int t, c_len; if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); s->c_str+= c_len; switch (s->c_next) @@ -306,7 +323,7 @@ static int json_handle_esc(json_string_t *s) if (s->c_next < 128 && (t= json_instr_chr_map[s->c_next]) == S_ERR) { s->c_str-= c_len; - return s->error= JE_ESCAPING; + return json_error(s, JE_ESCAPING); } @@ -328,18 +345,18 @@ static int json_handle_esc(json_string_t *s) return 0; if (c_len != MY_CS_TOOSMALL4) - return s->error= JE_BAD_CHR; + return json_error(s, JE_BAD_CHR); if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); if (s->c_next != '\\') - return s->error= JE_SYN; + return json_error(s, JE_SYN); s->c_str+= c_len; if ((c_len= json_next_char(s)) <= 0) - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); if (s->c_next != 'u') - return s->error= JE_SYN; + return json_error(s, JE_SYN); s->c_str+= c_len; if (read_4_hexdigits(s, code+2)) @@ -348,7 +365,7 @@ static int json_handle_esc(json_string_t *s) if ((c_len= my_utf16_uni(0, &s->c_next, code, code+4)) == 4) return 0; } - return s->error= JE_BAD_CHR; + return json_error(s, JE_BAD_CHR); } @@ -361,7 +378,7 @@ int json_read_string_const_chr(json_string_t *js) js->c_str+= c_len; return (js->c_next == '\\') ? json_handle_esc(js) : 0; } - js->error= json_eos(js) ? JE_EOS : JE_BAD_CHR; + json_error(js, json_eos(js) ? JE_EOS : JE_BAD_CHR); return 1; } @@ -387,10 +404,10 @@ static int skip_str_constant(json_engine_t *j) continue; } /* Symbol not allowed in JSON. */ - return j->s.error= JE_NOT_JSON_CHR; + return json_error(&j->s, JE_NOT_JSON_CHR); } else - return j->s.error= json_eos(&j->s) ? JE_EOS : JE_BAD_CHR; + return json_error(&j->s, json_eos(&j->s) ? JE_EOS : JE_BAD_CHR); } j->state= j->stack[j->stack_p]; @@ -528,7 +545,7 @@ static int skip_num_constant(json_engine_t *j) break; } - if ((j->s.error= + if (json_error(&j->s, json_eos(&j->s) ? json_num_states[state][N_END] : JE_BAD_CHR) < 0) return 1; else @@ -574,9 +591,9 @@ static int skip_string_verbatim(json_string_t *s, const char *str) s->c_str+= c_len; continue; } - return s->error= JE_SYN; + return json_error(s, JE_SYN); } - return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR; + return json_error(s, json_eos(s) ? JE_EOS : JE_BAD_CHR); } return 0; @@ -649,7 +666,7 @@ static int read_true(json_engine_t *j) /* Disallowed character. */ static int not_json_chr(json_engine_t *j) { - j->s.error= JE_NOT_JSON_CHR; + json_error(&j->s, JE_NOT_JSON_CHR); return 1; } @@ -657,7 +674,7 @@ static int not_json_chr(json_engine_t *j) /* Bad character. */ static int bad_chr(json_engine_t *j) { - j->s.error= JE_BAD_CHR; + json_error(&j->s, JE_BAD_CHR); return 1; } @@ -722,9 +739,9 @@ static int next_key(json_engine_t *j) return 0; } - j->s.error= (t_next == C_EOS) ? JE_EOS : - ((t_next == C_BAD) ? JE_BAD_CHR : - JE_SYN); + json_error(&j->s, (t_next == C_EOS) ? JE_EOS : + ((t_next == C_BAD) ? JE_BAD_CHR : + JE_SYN)); return 1; } @@ -804,11 +821,38 @@ static json_state_handler json_actions[NR_JSON_STATES][NR_C_CLASSES]= +#ifndef DBUG_OFF +/* + Called whenever a value is about to be read, if anybody has asked to + be told. + + Every reading of a JSON value begins here - the calls that look like + they start one of their own (json_valid(), json_valid_engine(), + json_get_path_start(), and the normalizing build) all come through + this function - so this is the one place where a count of readings is + a count of readings rather than a list of the ways of asking for one. + Counting anywhere else means keeping such a list, and a list of that + kind is wrong the day something is added to it. + + Nobody is told unless somebody asks: the server sets this and nothing + else does, so the tools and the tests that link this code in are + unaffected. Debug builds only, and there is nothing here for a + released server to do. +*/ +void (*json_scan_start_hook)(void)= NULL; +#endif + + int json_scan_start(json_engine_t *je, CHARSET_INFO *i_cs, const uchar *str, const uchar *end) { static const uint32_t no_time_to_die= 0; +#ifndef DBUG_OFF + if (json_scan_start_hook) + json_scan_start_hook(); +#endif + json_string_setup(&je->s, i_cs, str, end); je->stack[0]= JST_DONE; je->stack_p= 0; @@ -831,9 +875,9 @@ static int skip_colon(json_engine_t *j) return json_actions[JST_VALUE][t_next](j); } - j->s.error= (t_next == C_EOS) ? JE_EOS : - ((t_next == C_BAD) ? JE_BAD_CHR: - JE_SYN); + json_error(&j->s, (t_next == C_EOS) ? JE_EOS : + ((t_next == C_BAD) ? JE_BAD_CHR: + JE_SYN)); return 1; } @@ -850,7 +894,7 @@ static int skip_key(json_engine_t *j) while (json_read_keyname_chr(j) == 0) {} - if (j->s.error) + if (unlikely(j->s.error)) return 1; get_first_nonspace(&j->s, &t_next, &c_len); @@ -927,10 +971,10 @@ int json_read_keyname_chr(json_engine_t *j) j->s.c_str+= c_len; continue; } - j->s.error= JE_SYN; + json_error(&j->s, JE_SYN); break; } - j->s.error= json_eos(&j->s) ? JE_EOS : JE_BAD_CHR; + json_error(&j->s, json_eos(&j->s) ? JE_EOS : JE_BAD_CHR); break; } return 1; @@ -938,11 +982,11 @@ int json_read_keyname_chr(json_engine_t *j) return json_handle_esc(&j->s); case S_ERR: j->s.c_str-= c_len; - j->s.error= JE_STRING_CONST; + json_error(&j->s, JE_STRING_CONST); return 1; } } - j->s.error= json_eos(&j->s) ? JE_EOS : JE_BAD_CHR; + json_error(&j->s, json_eos(&j->s) ? JE_EOS : JE_BAD_CHR); return 1; } @@ -956,7 +1000,7 @@ int json_read_value(json_engine_t *j) { while (json_read_keyname_chr(j) == 0) {} - if (j->s.error) + if (unlikely(j->s.error)) return 1; } @@ -976,7 +1020,7 @@ int json_scan_next(json_engine_t *j) get_first_nonspace(&j->s, &t_next, &j->sav_c_len); if (j->killed_ptr && *j->killed_ptr) { - j->s.error= JE_KILLED; + json_error(&j->s, JE_KILLED); return 1; } return json_actions[j->state][t_next](j); @@ -1149,19 +1193,19 @@ int json_path_setup(json_path_t *p, t_next= (p->s.c_next >= 128) ? P_ETC : json_path_chr_map[p->s.c_next]; if ((state= json_path_transitions[state][t_next]) < 0) - return p->s.error= state; + return json_error(&p->s, state); p->s.c_str+= c_len; switch (state) { case PS_LAX: - if ((p->s.error= skip_string_verbatim(&p->s, "ax"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "ax"))) return 1; p->mode_strict= FALSE; continue; case PS_SCT: - if ((p->s.error= skip_string_verbatim(&p->s, "rict"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "rict"))) return 1; p->mode_strict= TRUE; state= PS_LAX; @@ -1204,7 +1248,7 @@ int json_path_setup(json_path_t *p, is_negative_index= 0; is_last= 0; if (p->last_step - p->steps >= JSON_DEPTH_LIMIT) - return p->s.error= JE_DEPTH; + return json_error(&p->s, JE_DEPTH); p->types_used|= p->last_step->type= JSON_PATH_KEY | double_wildcard; double_wildcard= JSON_PATH_KEY_NULL; /* fall through */ @@ -1222,7 +1266,7 @@ int json_path_setup(json_path_t *p, prev_value= 0; is_negative_index= 0; if (p->last_step - p->steps >= JSON_DEPTH_LIMIT) - return p->s.error= JE_DEPTH; + return json_error(&p->s, JE_DEPTH); p->types_used|= p->last_step->type= JSON_PATH_ARRAY | double_wildcard; double_wildcard= JSON_PATH_KEY_NULL; p->last_step->n_item= 0; @@ -1249,7 +1293,7 @@ int json_path_setup(json_path_t *p, is_negative_index= 1; continue; case PS_LAST: - if ((p->s.error= skip_string_verbatim(&p->s, "ast"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "ast"))) return 1; p->types_used|= JSON_PATH_NEGATIVE_INDEX; is_last= 1; @@ -1259,7 +1303,7 @@ int json_path_setup(json_path_t *p, p->last_step->n_item= -1; continue; case PS_T: - if ((p->s.error= skip_string_verbatim(&p->s, "o"))) + if (json_error(&p->s, skip_string_verbatim(&p->s, "o"))) return 1; is_to= 1; is_negative_index= 0; @@ -1272,7 +1316,7 @@ int json_path_setup(json_path_t *p, }; } while (state != PS_OK); - return double_wildcard ? (p->s.error= JE_SYN) : 0; + return double_wildcard ? json_error(&p->s, JE_SYN) : 0; } @@ -1316,7 +1360,15 @@ int json_skip_array_and_count(json_engine_t *je, int *n_items) res= json_skip_level_and_count(&j, n_items); if (res) + { + /* + The copy scanned the same string, so where it stopped is a + position in the caller's string too, and it has to come back with + the code: what reports the refusal reads the two together. + */ je->s.error= j.s.error; + je->s.error_pos= j.s.error_pos; + } return res; } @@ -1769,7 +1821,7 @@ int json_escape(CHARSET_INFO *str_cs, { /* We have to use /uXXXX escaping. */ uchar utf16buf[4]; - uchar code_str[8]; + uchar code_str[4]; int u_len= my_uni_utf16(0, c_chr, utf16buf, utf16buf + 4); code_str[0]= hexconv[utf16buf[0] >> 4]; @@ -1777,22 +1829,48 @@ int json_escape(CHARSET_INFO *str_cs, code_str[2]= hexconv[utf16buf[1] >> 4]; code_str[3]= hexconv[utf16buf[1] & 15]; - if (u_len > 2) + if ((c_len= json_append_ascii(json_cs, json, json_end, + code_str, code_str + 4)) <= 0) { - code_str[4]= hexconv[utf16buf[2] >> 4]; - code_str[5]= hexconv[utf16buf[2] & 15]; - code_str[6]= hexconv[utf16buf[3] >> 4]; - code_str[7]= hexconv[utf16buf[3] & 15]; + /* JSON buffer is depleted. */ + return JSON_ERROR_OUT_OF_SPACE; } - - if ((c_len= json_append_ascii(json_cs, json, json_end, - code_str, code_str+u_len*2)) > 0) + json+= c_len; + + if (u_len > 2) { + /* + A character outside the first plane is two UTF-16 units, and + each unit is written as an escape of its own: a reader takes + \uXXXX one at a time and puts the pair back together itself. + The second unit's figures after the first escape and nothing + to introduce them is not a second escape - it is four more + characters of the string, and the reader stops at the one + escape with half a character in hand. + */ + code_str[0]= hexconv[utf16buf[2] >> 4]; + code_str[1]= hexconv[utf16buf[2] & 15]; + code_str[2]= hexconv[utf16buf[3] >> 4]; + code_str[3]= hexconv[utf16buf[3] & 15]; + + if ((c_len= my_ci_wc_mb(json_cs, '\\', json, json_end)) <= 0 || + (c_len= my_ci_wc_mb(json_cs, 'u', json+= c_len, + json_end)) <= 0) + { + /* JSON buffer is depleted. */ + return JSON_ERROR_OUT_OF_SPACE; + } + json+= c_len; + + if ((c_len= json_append_ascii(json_cs, json, json_end, + code_str, code_str + 4)) <= 0) + { + /* JSON buffer is depleted. */ + return JSON_ERROR_OUT_OF_SPACE; + } json+= c_len; - continue; } - /* JSON buffer is depleted. */ - return JSON_ERROR_OUT_OF_SPACE; + continue; } } else /* c_len == 0, an illegal symbol. */ diff --git a/strings/json_normalize.c b/strings/json_normalize.c index 5243bd3c7064b..b74cd21a35bb6 100644 --- a/strings/json_normalize.c +++ b/strings/json_normalize.c @@ -931,7 +931,17 @@ json_normalize_engine(json_engine_t *je, DYNAMIC_STRING *result, { /* resulting error offset is mapped to original string */ if (je->s.error) + { je->s.c_str= (const uchar *) (s + (ptrdiff_t)(je->s.c_str - (const uchar *) s_utf8)); + /* + And the reading taken where the refusal happened, which is the + one reported. The converted copy is freed on the next line, so + a position left pointing into it is not merely in the wrong + string but in no string at all. + */ + je->s.error_pos= (const uchar *) + (s + (ptrdiff_t)(je->s.error_pos - (const uchar *) s_utf8)); + } my_free(s_utf8); } return err; diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index b8682de74c304..6f216a4dfbad6 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -34,3 +34,7 @@ MY_ADD_TEST(mf_iocache) ADD_EXECUTABLE(my_json_writer-t my_json_writer-t.cc dummy_builtins.cc) TARGET_LINK_LIBRARIES(my_json_writer-t sql mytap) MY_ADD_TEST(my_json_writer) + +ADD_EXECUTABLE(copy_field-t copy_field-t.cc dummy_builtins.cc) +TARGET_LINK_LIBRARIES(copy_field-t sql mytap) +MY_ADD_TEST(copy_field) diff --git a/unittest/sql/copy_field-t.cc b/unittest/sql/copy_field-t.cc new file mode 100644 index 0000000000000..7bd0a379f943d --- /dev/null +++ b/unittest/sql/copy_field-t.cc @@ -0,0 +1,136 @@ +/* + Copyright (c) 2026, MariaDB Foundation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA +*/ + +/** + Unit test for the state a Copy_field starts life in. + + Copy_field has two set() overloads. The field to field one records the + two fields it works between; the field to string one has no destination + field to record and records neither, and set(Field*, Field*, bool) + returns early for a MYSQL_TYPE_NULL destination without recording them + either. Entries of both kinds are put in one array, which callers walk + with a single loop, so from_field and to_field have to read as absent + where they were never filled in rather than as whatever the memory the + array was built on happened to hold. The same goes for the bit the + destination carries, which the field to string overload never records + either and which decides whether the copy confirms the attestation. +*/ + +#include +#include +#include +#include +#include + +/* + Any byte but zero: memory the constructor leaves alone must be + distinguishable from memory it cleared. +*/ +static const int POISON= 0xAB; + + +/** + Whether a bool reads back as false. + + Asked of the byte rather than of the value. A bool holding neither 0 + nor 1 makes any comparison of it undefined, and ok() takes anything + but zero for a pass, so a plain comparison would silently pass over + exactly the memory this file is about. +*/ + +static bool starts_false(const bool *flag) +{ + unsigned char raw; + + memcpy(&raw, flag, sizeof(raw)); + return raw == 0; +} + + +/** + Construct over memory known to be dirty. +*/ + +static void test_construction_over_dirty_memory() +{ + union + { + char bytes[2 * sizeof(Copy_field)]; + longlong align; + } storage; + Copy_field *first, *second; + + memset(storage.bytes, POISON, sizeof(storage.bytes)); + first= ::new ((void*) storage.bytes) Copy_field; + second= ::new ((void*) (storage.bytes + sizeof(Copy_field))) Copy_field; + + ok(first->from_field == NULL, "first from_field starts absent"); + ok(first->to_field == NULL, "first to_field starts absent"); + ok(starts_false(&first->to_needs_confirm), + "first to_needs_confirm starts false"); + ok(second->from_field == NULL, "second from_field starts absent"); + ok(second->to_field == NULL, "second to_field starts absent"); + ok(starts_false(&second->to_needs_confirm), + "second to_needs_confirm starts false"); + + second->~Copy_field(); + first->~Copy_field(); +} + + +/** + Build an array the way the server builds one, on a MEM_ROOT. + + The root is dirtied and its blocks marked free rather than released, so + the array is built on bytes that are known not to be zero. +*/ + +static void test_array_on_mem_root() +{ + const uint count= 3; + const size_t dirty= count * sizeof(Copy_field) + 64; + MEM_ROOT mem_root; + Copy_field *copy; + uint i; + + init_alloc_root(PSI_NOT_INSTRUMENTED, &mem_root, 4096, 0, MYF(0)); + memset(alloc_root(&mem_root, dirty), POISON, dirty); + free_root(&mem_root, MYF(MY_MARK_BLOCKS_FREE)); + + copy= new (&mem_root) Copy_field[count]; + for (i= 0; i < count; i++) + { + ok(copy[i].from_field == NULL, "copy[%u] from_field starts absent", i); + ok(copy[i].to_field == NULL, "copy[%u] to_field starts absent", i); + ok(starts_false(©[i].to_needs_confirm), + "copy[%u] to_needs_confirm starts false", i); + } + free_root(&mem_root, MYF(0)); +} + + +int main(int argc __attribute__((unused)), char *argv[]) +{ + MY_INIT(argv[0]); + plan(15); + + test_construction_over_dirty_memory(); + test_array_on_mem_root(); + + my_end(0); + return exit_status(); +}