Skip to content

MDEV-40802 COUNT(DISTINCT <blob>) fails when its tmp table converts #5561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
arcivanov wants to merge 1 commit into
base: bb-blob-main-monty
Choose a base branch
from
Open

MDEV-40802 COUNT(DISTINCT <blob>) fails when its tmp table converts #5561

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 171 additions & 0 deletions mysql-test/suite/heap/count_distinct_blob_convert.result
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
#
# COUNT(DISTINCT <blob>) must not fail with a unique constraint error
# when its in-memory temporary table overflows.
#
# The aggregate collects the distinct values in a temporary table with
# a unique constraint over its arguments and treats a duplicate key
# error from the write as "value already seen", not as a failure. For a
# blob argument the record holds only a pointer to the value, so the
# values cannot be compared as raw record bytes and the in-memory tree
# that normally does the deduplication is not used; every value goes
# through the write instead.
#
# When such a write overflows the in-memory table, the row it carried
# is handed to the conversion to the on-disk engine, which writes it to
# the converted table once all the stored rows have been copied. If
# that row is a duplicate of one of them, the conversion has to report
# a duplicate back to the aggregate, exactly as the write itself does,
# rather than a fatal write error.
#
set @save_tmp_memory_table_size=@@tmp_memory_table_size;
set @save_max_heap_table_size=@@max_heap_table_size;
set @small_heap=65536;
#
# Setup: 1000 distinct values, each present twice, with the two copies
# adjacent.
#
# A write that is rejected as a duplicate returns the record it had
# allocated to the free list and never gets as far as allocating space
# for the blob value, so only the first copy of a value makes the
# in-memory table grow. The write that finds the table full is
# therefore the second copy of the value stored last, a duplicate of a
# row the conversion has already copied.
#
# That holds only while a record slot is what the table runs out of
# first. Blob values come out of the same space, and only a write that
# is not a duplicate ever allocates one. When a blob allocation is the
# one that hits the limit, the pending row is not a duplicate and these
# cases no longer cover the conversion's handling of one.
#
# Which of the two allocations hits the limit first follows from how
# records and blob values pack together, not from any threshold on the
# value width. The widths used here were measured to overflow on a
# record slot; they were not derived from a rule, and neighbouring
# widths do not all behave the same way. The handling of a duplicate
# pending row is covered on its own, without depending on any of this,
# in count_distinct_blob_convert_debug.
#
CREATE TABLE t1 (id INT PRIMARY KEY, v TEXT, w TEXT, s VARCHAR(64))
ENGINE=MyISAM;
INSERT INTO t1
SELECT seq,
LPAD((seq+1) DIV 2, 6, 'x'),
LPAD((seq+1) DIV 2, 10, 'y'),
LPAD((seq+1) DIV 2, 6, 'z')
FROM seq_1_to_2000;
SELECT COUNT(*) AS rows_stored FROM t1;
rows_stored
2000
#
# ================================================================
# Run 1: the temporary table overflows and is converted
# ================================================================
#
set @@tmp_memory_table_size=@small_heap;
set @@max_heap_table_size=@small_heap;
# --- one blob argument ---
FLUSH STATUS;
SELECT COUNT(DISTINCT v) AS distinct_values FROM t1;
distinct_values
1000
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
ON
# --- one blob argument of a different width ---
FLUSH STATUS;
SELECT COUNT(DISTINCT w) AS distinct_values FROM t1;
distinct_values
1000
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
ON
# --- two blob arguments: the unique constraint spans both ---
FLUSH STATUS;
SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1;
distinct_values
1000
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
ON
# --- a blob and a non-blob argument ---
FLUSH STATUS;
SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1;
distinct_values
1000
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
ON
# --- control: no blob argument, deduplicated by the in-memory tree ---
FLUSH STATUS;
SELECT COUNT(DISTINCT s) AS distinct_values FROM t1;
distinct_values
1000
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
OFF
# --- grouped: one temporary table, reused for every group ---
#
# There is one aggregate, and it keeps one temporary table that is
# emptied between groups rather than created again. Once an earlier
# group has converted it, the later groups go on using the converted
# table, so this covers the aggregate reading a table it did not
# create in memory.
#
FLUSH STATUS;
SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values
FROM t1 GROUP BY g ORDER BY g;
g distinct_values
0 1000
1 1000
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
ON
#
# ================================================================
# Run 2: the same aggregates with enough memory to stay in memory
# ================================================================
#
set @@tmp_memory_table_size=1024*1024*512;
set @@max_heap_table_size=1024*1024*512;
FLUSH STATUS;
SELECT COUNT(DISTINCT v) AS distinct_values FROM t1;
distinct_values
1000
SELECT COUNT(DISTINCT w) AS distinct_values FROM t1;
distinct_values
1000
SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1;
distinct_values
1000
SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1;
distinct_values
1000
SELECT COUNT(DISTINCT s) AS distinct_values FROM t1;
distinct_values
1000
SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values
FROM t1 GROUP BY g ORDER BY g;
g distinct_values
0 1000
1 1000
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
OFF
# Clean up
set @@tmp_memory_table_size=@save_tmp_memory_table_size;
set @@max_heap_table_size=@save_max_heap_table_size;
DROP TABLE t1;
136 changes: 136 additions & 0 deletions mysql-test/suite/heap/count_distinct_blob_convert.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
--source include/have_sequence.inc

--echo #
--echo # COUNT(DISTINCT <blob>) must not fail with a unique constraint error
--echo # when its in-memory temporary table overflows.
--echo #
--echo # The aggregate collects the distinct values in a temporary table with
--echo # a unique constraint over its arguments and treats a duplicate key
--echo # error from the write as "value already seen", not as a failure. For a
--echo # blob argument the record holds only a pointer to the value, so the
--echo # values cannot be compared as raw record bytes and the in-memory tree
--echo # that normally does the deduplication is not used; every value goes
--echo # through the write instead.
--echo #
--echo # When such a write overflows the in-memory table, the row it carried
--echo # is handed to the conversion to the on-disk engine, which writes it to
--echo # the converted table once all the stored rows have been copied. If
--echo # that row is a duplicate of one of them, the conversion has to report
--echo # a duplicate back to the aggregate, exactly as the write itself does,
--echo # rather than a fatal write error.
--echo #

set @save_tmp_memory_table_size=@@tmp_memory_table_size;
set @save_max_heap_table_size=@@max_heap_table_size;
set @small_heap=65536;

--echo #
--echo # Setup: 1000 distinct values, each present twice, with the two copies
--echo # adjacent.
--echo #
--echo # A write that is rejected as a duplicate returns the record it had
--echo # allocated to the free list and never gets as far as allocating space
--echo # for the blob value, so only the first copy of a value makes the
--echo # in-memory table grow. The write that finds the table full is
--echo # therefore the second copy of the value stored last, a duplicate of a
--echo # row the conversion has already copied.
--echo #
--echo # That holds only while a record slot is what the table runs out of
--echo # first. Blob values come out of the same space, and only a write that
--echo # is not a duplicate ever allocates one. When a blob allocation is the
--echo # one that hits the limit, the pending row is not a duplicate and these
--echo # cases no longer cover the conversion's handling of one.
--echo #
--echo # Which of the two allocations hits the limit first follows from how
--echo # records and blob values pack together, not from any threshold on the
--echo # value width. The widths used here were measured to overflow on a
--echo # record slot; they were not derived from a rule, and neighbouring
--echo # widths do not all behave the same way. The handling of a duplicate
--echo # pending row is covered on its own, without depending on any of this,
--echo # in count_distinct_blob_convert_debug.
--echo #

CREATE TABLE t1 (id INT PRIMARY KEY, v TEXT, w TEXT, s VARCHAR(64))
ENGINE=MyISAM;
INSERT INTO t1
SELECT seq,
LPAD((seq+1) DIV 2, 6, 'x'),
LPAD((seq+1) DIV 2, 10, 'y'),
LPAD((seq+1) DIV 2, 6, 'z')
FROM seq_1_to_2000;

SELECT COUNT(*) AS rows_stored FROM t1;

--echo #
--echo # ================================================================
--echo # Run 1: the temporary table overflows and is converted
--echo # ================================================================
--echo #

set @@tmp_memory_table_size=@small_heap;
set @@max_heap_table_size=@small_heap;

--echo # --- one blob argument ---
FLUSH STATUS;
SELECT COUNT(DISTINCT v) AS distinct_values FROM t1;
--source count_distinct_converted.inc

--echo # --- one blob argument of a different width ---
FLUSH STATUS;
SELECT COUNT(DISTINCT w) AS distinct_values FROM t1;
--source count_distinct_converted.inc

--echo # --- two blob arguments: the unique constraint spans both ---
FLUSH STATUS;
SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1;
--source count_distinct_converted.inc

--echo # --- a blob and a non-blob argument ---
FLUSH STATUS;
SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1;
--source count_distinct_converted.inc

--echo # --- control: no blob argument, deduplicated by the in-memory tree ---
FLUSH STATUS;
SELECT COUNT(DISTINCT s) AS distinct_values FROM t1;
--source count_distinct_converted.inc

--echo # --- grouped: one temporary table, reused for every group ---
--echo #
--echo # There is one aggregate, and it keeps one temporary table that is
--echo # emptied between groups rather than created again. Once an earlier
--echo # group has converted it, the later groups go on using the converted
--echo # table, so this covers the aggregate reading a table it did not
--echo # create in memory.
--echo #
FLUSH STATUS;
SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values
FROM t1 GROUP BY g ORDER BY g;
--source count_distinct_converted.inc

--echo #
--echo # ================================================================
--echo # Run 2: the same aggregates with enough memory to stay in memory
--echo # ================================================================
--echo #

set @@tmp_memory_table_size=1024*1024*512;
set @@max_heap_table_size=1024*1024*512;

FLUSH STATUS;
SELECT COUNT(DISTINCT v) AS distinct_values FROM t1;
SELECT COUNT(DISTINCT w) AS distinct_values FROM t1;
SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1;
SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1;
SELECT COUNT(DISTINCT s) AS distinct_values FROM t1;
SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values
FROM t1 GROUP BY g ORDER BY g;
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';

--echo # Clean up

set @@tmp_memory_table_size=@save_tmp_memory_table_size;
set @@max_heap_table_size=@save_max_heap_table_size;
DROP TABLE t1;
60 changes: 60 additions & 0 deletions mysql-test/suite/heap/count_distinct_blob_convert_debug.result
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#
# A duplicate on the row that overflowed the in-memory temporary table
# of COUNT(DISTINCT <blob>) must be reported back to the aggregate as
# "value already seen", not as a fatal write error.
#
# Which write is the one that overflows the table, and so whether the
# row the conversion has left to write is a duplicate of a row it has
# already copied, depends on the order in which the in-memory engine
# happens to allocate record slots and blob values. That order is not
# stable across platforms, so the duplicate is injected here instead:
# the write of the pending row reports one whatever the row holds. The
# aggregate has to answer with the number of distinct values either way.
#
SET @save_tmp_memory_table_size= @@tmp_memory_table_size;
SET @save_max_heap_table_size= @@max_heap_table_size;
SET @save_debug_dbug= @@debug_dbug;
SET @small_heap= 65536;
#
# 1000 distinct values, each present twice. The injected duplicate
# discards the pending row, and whichever of the two copies that was,
# the other one is still written, so the answer is the same either way.
#
# The values are wide enough that the table is certain to overflow.
#
CREATE TABLE t1 (id INT PRIMARY KEY, v TEXT, w TEXT) ENGINE=MyISAM;
INSERT INTO t1
SELECT seq,
LPAD((seq+1) DIV 2, 200, 'x'),
LPAD((seq+1) DIV 2, 300, 'y')
FROM seq_1_to_2000;
SET @@tmp_memory_table_size= @small_heap;
SET @@max_heap_table_size= @small_heap;
# --- one blob argument ---
FLUSH STATUS;
SET @@debug_dbug= '+d,dup_pending_tmp_table_row';
SELECT COUNT(DISTINCT v) AS distinct_values FROM t1;
distinct_values
1000
SET @@debug_dbug= @save_debug_dbug;
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
ON
# --- two blob arguments: the unique constraint spans both ---
FLUSH STATUS;
SET @@debug_dbug= '+d,dup_pending_tmp_table_row';
SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1;
distinct_values
1000
SET @@debug_dbug= @save_debug_dbug;
SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED
FROM INFORMATION_SCHEMA.SESSION_STATUS
WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES';
CONVERTED
ON
# Clean up
SET @@tmp_memory_table_size= @save_tmp_memory_table_size;
SET @@max_heap_table_size= @save_max_heap_table_size;
DROP TABLE t1;
Loading
Loading