MDEV-40642 Project "JSON Phoenix" - redundant computation elimination

include/json_lib.h

@@ -12,7 +12,7 @@ extern "C" {
 
 /*
   When error happens, the c_next of the JSON engine contains the
-  character that caused the error, and the c_str is the position
+  character that caused the error, and the error_pos is the position
   in string where the error occurs.
 */
 enum json_errors {
@@ -41,6 +41,7 @@ typedef struct st_json_string_t
   my_wc_t c_next;        /* UNICODE of the last read character */
   int c_next_len;        /* character lenght of the last read character. */
   int error;             /* error code. */
+  const uchar *error_pos; /* Where in the string that error happened. */
 
   CHARSET_INFO *cs;      /* Character set of the JSON string. */
 
@@ -52,6 +53,43 @@ typedef struct st_json_string_t
 void json_string_set_cs(json_string_t *s, CHARSET_INFO *i_cs);
 void json_string_set_str(json_string_t *s,
                          const uchar *str, const uchar *end);
+
+/*
+  Refuse the string, and remember where the refusing happened.
+
+  The position is taken here rather than read off c_str at the point
+  the refusal is reported, because the scanner does not come to rest
+  where it failed.  A caller is free to go on asking for the next token
+  after being told no - not all of them are obliged to stop, and see
+  where the depth refusals are written for why they cannot be made to -
+  and c_str then walks past the place the refusal happened, so a reading
+  taken later describes somewhere else.
+
+  The first refusal is the one kept, for the same reason: a scan that
+  carries on after failing reaches handlers that refuse it again on
+  their own account, and what the caller is told has to be what stopped
+  the scan rather than whatever it ran into afterwards.
+
+  Returns the code it was given, so the sites written as
+  `return s->error= JE_SYN` keep their shape.  A code of zero refuses
+  nothing, which is what the sites that pass a result along need.
+
+  Everything that a position is reported for is refused through here,
+  so error and error_pos are set together and mean each other.  The
+  geometry reader puts codes of its own into error without coming here;
+  it reports those by name and never asks where, and it means to
+  replace whatever the scanner said, which is why it still assigns.
+*/
+static inline int json_error(json_string_t *s, int code)
+{
+  if (code && !s->error)
+  {
+    s->error= code;
+    s->error_pos= s->c_str;
+  }
+  return code;
+}
+
 #define json_next_char(j) \
   ((j)->c_next_len= (j)->wc((j)->cs, &(j)->c_next, (j)->c_str, (j)->str_end))
 #define json_eos(j) ((j)->c_str >= (j)->str_end)
@@ -235,6 +273,14 @@ typedef struct st_json_engine_t
 } json_engine_t;
 
 
+#ifndef DBUG_OFF
+/*
+  Told about every reading of a value, when somebody has asked to be.
+  See json_scan_start() for why this is where the telling happens.
+*/
+extern void (*json_scan_start_hook)(void);
+#endif
+
 int json_scan_start(json_engine_t *je,
                         CHARSET_INFO *i_cs, const uchar *str, const uchar *end);
 int json_scan_next(json_engine_t *j);

mysql-test/main/func_concat.result

@@ -304,3 +304,72 @@ f
 NULL
 DROP TABLE t1;
 # End of 10.5 tests
+# Start of 10.11 tests
+#
+# CONCAT keeps the value of its first argument when that argument
+# answers in the buffer it was handed, pointing at bytes it does
+# not own
+#
+SELECT JSON_TYPE('{"a":1}');
+JSON_TYPE('{"a":1}')
+OBJECT
+SELECT CONCAT(JSON_TYPE('{"a":1}'), 'X');
+CONCAT(JSON_TYPE('{"a":1}'), 'X')
+OBJECTX
+SELECT CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B');
+CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B')
+OBJECTAB
+SELECT CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]'));
+CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]'))
+OBJECTARRAY
+SELECT CONCAT(JSON_TYPE('{"a":1}'), '');
+CONCAT(JSON_TYPE('{"a":1}'), '')
+OBJECT
+SELECT LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X'));
+LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X'))
+7
+SELECT GET_FORMAT(DATE, 'USA');
+GET_FORMAT(DATE, 'USA')
+%m.%d.%Y
+SELECT CONCAT(GET_FORMAT(DATE, 'USA'), 'X');
+CONCAT(GET_FORMAT(DATE, 'USA'), 'X')
+%m.%d.%YX
+SELECT CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO'));
+CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO'))
+%m.%d.%Y%Y-%m-%d
+# Every other argument position was always right
+SELECT CONCAT('P', JSON_TYPE('{"a":1}'));
+CONCAT('P', JSON_TYPE('{"a":1}'))
+POBJECT
+SELECT CONCAT('P', GET_FORMAT(DATE, 'USA'));
+CONCAT('P', GET_FORMAT(DATE, 'USA'))
+P%m.%d.%Y
+SELECT CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X');
+CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X')
+OBJECT-X
+# The same over table data
+CREATE TABLE t1 (j JSON);
+INSERT INTO t1 VALUES ('{"a":1}'), ('[1,2]');
+SELECT CONCAT(JSON_TYPE(j), '.') AS c FROM t1;
+c
+OBJECT.
+ARRAY.
+SELECT CONCAT(JSON_TYPE(j), JSON_TYPE(j)) AS c FROM t1;
+c
+OBJECTOBJECT
+ARRAYARRAY
+DROP TABLE t1;
+# The || operator in sql_mode=ORACLE concatenates the same way
+SET @save_sql_mode=@@sql_mode;
+SET sql_mode=ORACLE;
+SELECT JSON_TYPE('{"a":1}') || 'X';
+JSON_TYPE('{"a":1}') || 'X'
+OBJECTX
+SELECT GET_FORMAT(DATE, 'USA') || 'X';
+GET_FORMAT(DATE, 'USA') || 'X'
+%m.%d.%YX
+SELECT 'P' || JSON_TYPE('{"a":1}');
+'P' || JSON_TYPE('{"a":1}')
+POBJECT
+SET sql_mode=@save_sql_mode;
+# End of 10.11 tests

mysql-test/main/func_concat.test

@@ -281,3 +281,51 @@ SELECT CONCAT_WS(' ', a, b, PASSWORD(c)) AS f FROM t1 GROUP BY f WITH ROLLUP;
 DROP TABLE t1;
 
 --echo # End of 10.5 tests
+
+--echo # Start of 10.11 tests
+
+--echo #
+--echo # CONCAT keeps the value of its first argument when that argument
+--echo # answers in the buffer it was handed, pointing at bytes it does
+--echo # not own
+--echo #
+
+SELECT JSON_TYPE('{"a":1}');
+SELECT CONCAT(JSON_TYPE('{"a":1}'), 'X');
+SELECT CONCAT(JSON_TYPE('{"a":1}'), 'A', 'B');
+SELECT CONCAT(JSON_TYPE('{"a":1}'), JSON_TYPE('[1]'));
+SELECT CONCAT(JSON_TYPE('{"a":1}'), '');
+SELECT LENGTH(CONCAT(JSON_TYPE('{"a":1}'), 'X'));
+
+SELECT GET_FORMAT(DATE, 'USA');
+SELECT CONCAT(GET_FORMAT(DATE, 'USA'), 'X');
+SELECT CONCAT(GET_FORMAT(DATE, 'USA'), GET_FORMAT(DATE, 'ISO'));
+
+--echo # Every other argument position was always right
+
+SELECT CONCAT('P', JSON_TYPE('{"a":1}'));
+SELECT CONCAT('P', GET_FORMAT(DATE, 'USA'));
+SELECT CONCAT_WS('-', JSON_TYPE('{"a":1}'), 'X');
+
+--echo # The same over table data
+
+CREATE TABLE t1 (j JSON);
+INSERT INTO t1 VALUES ('{"a":1}'), ('[1,2]');
+SELECT CONCAT(JSON_TYPE(j), '.') AS c FROM t1;
+SELECT CONCAT(JSON_TYPE(j), JSON_TYPE(j)) AS c FROM t1;
+DROP TABLE t1;
+
+--echo # The || operator in sql_mode=ORACLE concatenates the same way
+# A wrapping view is created on a connection of its own, which does not
+# carry this session's sql_mode, so || would be read as an OR there.
+--disable_view_protocol
+
+SET @save_sql_mode=@@sql_mode;
+SET sql_mode=ORACLE;
+SELECT JSON_TYPE('{"a":1}') || 'X';
+SELECT GET_FORMAT(DATE, 'USA') || 'X';
+SELECT 'P' || JSON_TYPE('{"a":1}');
+SET sql_mode=@save_sql_mode;
+--enable_view_protocol
+
+--echo # End of 10.11 tests

mysql-test/main/func_gconcat.result

@@ -1556,3 +1556,68 @@ Warning	1260	Row 1 was cut by GROUP_CONCAT()
 disconnect u;
 connection default;
 # End of 10.6 tests
+#
+# A group with an OFFSET past its last row, asked for more than once
+#
+# Nothing says how often a statement asks for the result of a group.
+# HAVING on the alias is the shortest one that asks twice, and the
+# answer must not depend on how often it was asked.
+#
+CREATE TABLE t1 (g INT, a VARCHAR(10));
+INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f');
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1;
+v
+
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+
+# two conditions, so it is asked a third time
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%' AND v NOT LIKE 'a%';
+v
+
+# DISTINCT reaches the same walk by the other route
+SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+
+# a group per row of output, so more than one group is replayed
+SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1
+GROUP BY g HAVING v LIKE '%' ORDER BY g;
+g	v
+1	
+2	
+# an offset that stops inside the group, which must not move
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+b,c
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 3) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+d
+# and no LIMIT at all
+SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1 HAVING v LIKE '%';
+v
+a,b,c,d
+# the same through the aggregate that is built on this one, where an
+# element written a second time lands outside the brackets already
+# put round the group
+SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1;
+v
+[]
+SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+[]
+SELECT JSON_VALID(JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4)) AS v
+FROM t1 WHERE g = 1 HAVING v LIKE '%';
+v
+1
+SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1
+HAVING v LIKE '%';
+v
+["b","c"]
+DROP TABLE t1;
+# End of 10.11 tests

mysql-test/main/func_gconcat.test

@@ -1137,3 +1137,46 @@ disconnect u;
 connection default;
 
 --echo # End of 10.6 tests
+
+--echo #
+--echo # A group with an OFFSET past its last row, asked for more than once
+--echo #
+--echo # Nothing says how often a statement asks for the result of a group.
+--echo # HAVING on the alias is the shortest one that asks twice, and the
+--echo # answer must not depend on how often it was asked.
+--echo #
+CREATE TABLE t1 (g INT, a VARCHAR(10));
+INSERT INTO t1 VALUES (1,'a'),(1,'b'),(1,'c'),(1,'d'),(2,'e'),(2,'f');
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1;
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+--echo # two conditions, so it is asked a third time
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%' AND v NOT LIKE 'a%';
+--echo # DISTINCT reaches the same walk by the other route
+SELECT GROUP_CONCAT(DISTINCT a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+--echo # a group per row of output, so more than one group is replayed
+SELECT g, GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1
+  GROUP BY g HAVING v LIKE '%' ORDER BY g;
+--echo # an offset that stops inside the group, which must not move
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+SELECT GROUP_CONCAT(a ORDER BY a LIMIT 2 OFFSET 3) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+--echo # and no LIMIT at all
+SELECT GROUP_CONCAT(a ORDER BY a) AS v FROM t1 WHERE g = 1 HAVING v LIKE '%';
+
+--echo # the same through the aggregate that is built on this one, where an
+--echo # element written a second time lands outside the brackets already
+--echo # put round the group
+SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1;
+SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+SELECT JSON_VALID(JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 4)) AS v
+  FROM t1 WHERE g = 1 HAVING v LIKE '%';
+SELECT JSON_ARRAYAGG(a ORDER BY a LIMIT 2 OFFSET 1) AS v FROM t1 WHERE g = 1
+  HAVING v LIKE '%';
+DROP TABLE t1;
+
+--echo # End of 10.11 tests

mysql-test/main/func_json.result

@@ -280,7 +280,7 @@ create table t1 as select json_object('id', 87, 'name', 'carrot') as f;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `f` varchar(46) DEFAULT NULL
+  `f` varchar(52) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 select * from t1;
 f
@@ -305,7 +305,7 @@ json_quote('foo')
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `json_quote('foo')` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
+  `json_quote('foo')` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop table t1;
 select json_merge('string');
@@ -770,8 +770,8 @@ JSON_QUOTE(_utf8'foo') AS c2;
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `c1` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
-  `c2` varchar(38) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
+  `c1` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL,
+  `c2` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 DROP TABLE t1;
 #
@@ -780,6 +780,8 @@ DROP TABLE t1;
 select json_array(1,user(),compress(5.140264e+307));
 json_array(1,user(),compress(5.140264e+307))
 NULL
+Warnings:
+Note	4035	Broken JSON string in argument 3 to function 'json_array' at position 0
 #
 # MDEV-16869 String functions don't respect character set of JSON_VALUE.
 #
@@ -834,7 +836,7 @@ CREATE TABLE t2 SELECT JSON_ARRAY_INSERT(fld, '$.[0]', '0') FROM t1;
 SHOW CREATE TABLE t2;
 Table	Create Table
 t2	CREATE TABLE `t2` (
-  `JSON_ARRAY_INSERT(fld, '$.[0]', '0')` varchar(21) DEFAULT NULL
+  `JSON_ARRAY_INSERT(fld, '$.[0]', '0')` varchar(40) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 DROP TABLE t1, t2;
 SET sql_mode=default;
@@ -1615,7 +1617,7 @@ insert into t1 values (concat('x64-', repeat('b', 60)));
 insert into t1 values (concat('x64-', repeat('c', 60)));
 select json_arrayagg(a) from t1;
 json_arrayagg(a)
-["x64-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]
+[]
 Warnings:
 Warning	1260	Row 1 was cut by JSON_ARRAYAGG()
 drop table t1;
@@ -1741,9 +1743,7 @@ SET @save_collation_connection= @@collation_connection;
 SET collation_connection='utf16_bin';
 SELECT JSON_EXTRACT('{"a": 1,"b": 2}','$.a');
 JSON_EXTRACT('{"a": 1,"b": 2}','$.a')
-NULL
-Warnings:
-Warning	4036	Character disallowed in JSON in argument 1 to function 'json_extract' at position 2
+1
 SET @@collation_connection= @save_collation_connection;
 # End of 10.5 tests
 #
@@ -1832,9 +1832,9 @@ create temporary table t (c varchar(20) character set latin1);
 insert into t values ('ab'), ('cd'), ('ef');
 select json_arrayagg(c order by c) from t;
 json_arrayagg(c order by c)
-["ab","cd"]
+["ab"]
 Warnings:
-Warning	1260	Row 3 was cut by JSON_ARRAYAGG()
+Warning	1260	Row 2 was cut by JSON_ARRAYAGG()
 drop temporary table t;
 # End of 10.6 tests
 SELECT json_extract(t.j, '$')