diff --git a/mysql-test/suite/heap/count_distinct_blob_convert.result b/mysql-test/suite/heap/count_distinct_blob_convert.result new file mode 100644 index 0000000000000..8cc248fc23b0b --- /dev/null +++ b/mysql-test/suite/heap/count_distinct_blob_convert.result @@ -0,0 +1,171 @@ +# +# COUNT(DISTINCT ) must not fail with a unique constraint error +# when its in-memory temporary table overflows. +# +# The aggregate collects the distinct values in a temporary table with +# a unique constraint over its arguments and treats a duplicate key +# error from the write as "value already seen", not as a failure. For a +# blob argument the record holds only a pointer to the value, so the +# values cannot be compared as raw record bytes and the in-memory tree +# that normally does the deduplication is not used; every value goes +# through the write instead. +# +# When such a write overflows the in-memory table, the row it carried +# is handed to the conversion to the on-disk engine, which writes it to +# the converted table once all the stored rows have been copied. If +# that row is a duplicate of one of them, the conversion has to report +# a duplicate back to the aggregate, exactly as the write itself does, +# rather than a fatal write error. +# +set @save_tmp_memory_table_size=@@tmp_memory_table_size; +set @save_max_heap_table_size=@@max_heap_table_size; +set @small_heap=65536; +# +# Setup: 1000 distinct values, each present twice, with the two copies +# adjacent. +# +# A write that is rejected as a duplicate returns the record it had +# allocated to the free list and never gets as far as allocating space +# for the blob value, so only the first copy of a value makes the +# in-memory table grow. The write that finds the table full is +# therefore the second copy of the value stored last, a duplicate of a +# row the conversion has already copied. +# +# That holds only while a record slot is what the table runs out of +# first. Blob values come out of the same space, and only a write that +# is not a duplicate ever allocates one. When a blob allocation is the +# one that hits the limit, the pending row is not a duplicate and these +# cases no longer cover the conversion's handling of one. +# +# Which of the two allocations hits the limit first follows from how +# records and blob values pack together, not from any threshold on the +# value width. The widths used here were measured to overflow on a +# record slot; they were not derived from a rule, and neighbouring +# widths do not all behave the same way. The handling of a duplicate +# pending row is covered on its own, without depending on any of this, +# in count_distinct_blob_convert_debug. +# +CREATE TABLE t1 (id INT PRIMARY KEY, v TEXT, w TEXT, s VARCHAR(64)) +ENGINE=MyISAM; +INSERT INTO t1 +SELECT seq, +LPAD((seq+1) DIV 2, 6, 'x'), +LPAD((seq+1) DIV 2, 10, 'y'), +LPAD((seq+1) DIV 2, 6, 'z') +FROM seq_1_to_2000; +SELECT COUNT(*) AS rows_stored FROM t1; +rows_stored +2000 +# +# ================================================================ +# Run 1: the temporary table overflows and is converted +# ================================================================ +# +set @@tmp_memory_table_size=@small_heap; +set @@max_heap_table_size=@small_heap; +# --- one blob argument --- +FLUSH STATUS; +SELECT COUNT(DISTINCT v) AS distinct_values FROM t1; +distinct_values +1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- one blob argument of a different width --- +FLUSH STATUS; +SELECT COUNT(DISTINCT w) AS distinct_values FROM t1; +distinct_values +1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- two blob arguments: the unique constraint spans both --- +FLUSH STATUS; +SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1; +distinct_values +1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- a blob and a non-blob argument --- +FLUSH STATUS; +SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1; +distinct_values +1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- control: no blob argument, deduplicated by the in-memory tree --- +FLUSH STATUS; +SELECT COUNT(DISTINCT s) AS distinct_values FROM t1; +distinct_values +1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +OFF +# --- grouped: one temporary table, reused for every group --- +# +# There is one aggregate, and it keeps one temporary table that is +# emptied between groups rather than created again. Once an earlier +# group has converted it, the later groups go on using the converted +# table, so this covers the aggregate reading a table it did not +# create in memory. +# +FLUSH STATUS; +SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values +FROM t1 GROUP BY g ORDER BY g; +g distinct_values +0 1000 +1 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# +# ================================================================ +# Run 2: the same aggregates with enough memory to stay in memory +# ================================================================ +# +set @@tmp_memory_table_size=1024*1024*512; +set @@max_heap_table_size=1024*1024*512; +FLUSH STATUS; +SELECT COUNT(DISTINCT v) AS distinct_values FROM t1; +distinct_values +1000 +SELECT COUNT(DISTINCT w) AS distinct_values FROM t1; +distinct_values +1000 +SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1; +distinct_values +1000 +SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1; +distinct_values +1000 +SELECT COUNT(DISTINCT s) AS distinct_values FROM t1; +distinct_values +1000 +SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values +FROM t1 GROUP BY g ORDER BY g; +g distinct_values +0 1000 +1 1000 +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +OFF +# Clean up +set @@tmp_memory_table_size=@save_tmp_memory_table_size; +set @@max_heap_table_size=@save_max_heap_table_size; +DROP TABLE t1; diff --git a/mysql-test/suite/heap/count_distinct_blob_convert.test b/mysql-test/suite/heap/count_distinct_blob_convert.test new file mode 100644 index 0000000000000..d8e59371bf3fb --- /dev/null +++ b/mysql-test/suite/heap/count_distinct_blob_convert.test @@ -0,0 +1,136 @@ +--source include/have_sequence.inc + +--echo # +--echo # COUNT(DISTINCT ) must not fail with a unique constraint error +--echo # when its in-memory temporary table overflows. +--echo # +--echo # The aggregate collects the distinct values in a temporary table with +--echo # a unique constraint over its arguments and treats a duplicate key +--echo # error from the write as "value already seen", not as a failure. For a +--echo # blob argument the record holds only a pointer to the value, so the +--echo # values cannot be compared as raw record bytes and the in-memory tree +--echo # that normally does the deduplication is not used; every value goes +--echo # through the write instead. +--echo # +--echo # When such a write overflows the in-memory table, the row it carried +--echo # is handed to the conversion to the on-disk engine, which writes it to +--echo # the converted table once all the stored rows have been copied. If +--echo # that row is a duplicate of one of them, the conversion has to report +--echo # a duplicate back to the aggregate, exactly as the write itself does, +--echo # rather than a fatal write error. +--echo # + +set @save_tmp_memory_table_size=@@tmp_memory_table_size; +set @save_max_heap_table_size=@@max_heap_table_size; +set @small_heap=65536; + +--echo # +--echo # Setup: 1000 distinct values, each present twice, with the two copies +--echo # adjacent. +--echo # +--echo # A write that is rejected as a duplicate returns the record it had +--echo # allocated to the free list and never gets as far as allocating space +--echo # for the blob value, so only the first copy of a value makes the +--echo # in-memory table grow. The write that finds the table full is +--echo # therefore the second copy of the value stored last, a duplicate of a +--echo # row the conversion has already copied. +--echo # +--echo # That holds only while a record slot is what the table runs out of +--echo # first. Blob values come out of the same space, and only a write that +--echo # is not a duplicate ever allocates one. When a blob allocation is the +--echo # one that hits the limit, the pending row is not a duplicate and these +--echo # cases no longer cover the conversion's handling of one. +--echo # +--echo # Which of the two allocations hits the limit first follows from how +--echo # records and blob values pack together, not from any threshold on the +--echo # value width. The widths used here were measured to overflow on a +--echo # record slot; they were not derived from a rule, and neighbouring +--echo # widths do not all behave the same way. The handling of a duplicate +--echo # pending row is covered on its own, without depending on any of this, +--echo # in count_distinct_blob_convert_debug. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, v TEXT, w TEXT, s VARCHAR(64)) +ENGINE=MyISAM; +INSERT INTO t1 +SELECT seq, + LPAD((seq+1) DIV 2, 6, 'x'), + LPAD((seq+1) DIV 2, 10, 'y'), + LPAD((seq+1) DIV 2, 6, 'z') +FROM seq_1_to_2000; + +SELECT COUNT(*) AS rows_stored FROM t1; + +--echo # +--echo # ================================================================ +--echo # Run 1: the temporary table overflows and is converted +--echo # ================================================================ +--echo # + +set @@tmp_memory_table_size=@small_heap; +set @@max_heap_table_size=@small_heap; + +--echo # --- one blob argument --- +FLUSH STATUS; +SELECT COUNT(DISTINCT v) AS distinct_values FROM t1; +--source count_distinct_converted.inc + +--echo # --- one blob argument of a different width --- +FLUSH STATUS; +SELECT COUNT(DISTINCT w) AS distinct_values FROM t1; +--source count_distinct_converted.inc + +--echo # --- two blob arguments: the unique constraint spans both --- +FLUSH STATUS; +SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1; +--source count_distinct_converted.inc + +--echo # --- a blob and a non-blob argument --- +FLUSH STATUS; +SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1; +--source count_distinct_converted.inc + +--echo # --- control: no blob argument, deduplicated by the in-memory tree --- +FLUSH STATUS; +SELECT COUNT(DISTINCT s) AS distinct_values FROM t1; +--source count_distinct_converted.inc + +--echo # --- grouped: one temporary table, reused for every group --- +--echo # +--echo # There is one aggregate, and it keeps one temporary table that is +--echo # emptied between groups rather than created again. Once an earlier +--echo # group has converted it, the later groups go on using the converted +--echo # table, so this covers the aggregate reading a table it did not +--echo # create in memory. +--echo # +FLUSH STATUS; +SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values +FROM t1 GROUP BY g ORDER BY g; +--source count_distinct_converted.inc + +--echo # +--echo # ================================================================ +--echo # Run 2: the same aggregates with enough memory to stay in memory +--echo # ================================================================ +--echo # + +set @@tmp_memory_table_size=1024*1024*512; +set @@max_heap_table_size=1024*1024*512; + +FLUSH STATUS; +SELECT COUNT(DISTINCT v) AS distinct_values FROM t1; +SELECT COUNT(DISTINCT w) AS distinct_values FROM t1; +SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1; +SELECT COUNT(DISTINCT v, s) AS distinct_values FROM t1; +SELECT COUNT(DISTINCT s) AS distinct_values FROM t1; +SELECT id MOD 2 AS g, COUNT(DISTINCT v) AS distinct_values +FROM t1 GROUP BY g ORDER BY g; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; + +--echo # Clean up + +set @@tmp_memory_table_size=@save_tmp_memory_table_size; +set @@max_heap_table_size=@save_max_heap_table_size; +DROP TABLE t1; diff --git a/mysql-test/suite/heap/count_distinct_blob_convert_debug.result b/mysql-test/suite/heap/count_distinct_blob_convert_debug.result new file mode 100644 index 0000000000000..3226bf3a10891 --- /dev/null +++ b/mysql-test/suite/heap/count_distinct_blob_convert_debug.result @@ -0,0 +1,60 @@ +# +# A duplicate on the row that overflowed the in-memory temporary table +# of COUNT(DISTINCT ) must be reported back to the aggregate as +# "value already seen", not as a fatal write error. +# +# Which write is the one that overflows the table, and so whether the +# row the conversion has left to write is a duplicate of a row it has +# already copied, depends on the order in which the in-memory engine +# happens to allocate record slots and blob values. That order is not +# stable across platforms, so the duplicate is injected here instead: +# the write of the pending row reports one whatever the row holds. The +# aggregate has to answer with the number of distinct values either way. +# +SET @save_tmp_memory_table_size= @@tmp_memory_table_size; +SET @save_max_heap_table_size= @@max_heap_table_size; +SET @save_debug_dbug= @@debug_dbug; +SET @small_heap= 65536; +# +# 1000 distinct values, each present twice. The injected duplicate +# discards the pending row, and whichever of the two copies that was, +# the other one is still written, so the answer is the same either way. +# +# The values are wide enough that the table is certain to overflow. +# +CREATE TABLE t1 (id INT PRIMARY KEY, v TEXT, w TEXT) ENGINE=MyISAM; +INSERT INTO t1 +SELECT seq, +LPAD((seq+1) DIV 2, 200, 'x'), +LPAD((seq+1) DIV 2, 300, 'y') +FROM seq_1_to_2000; +SET @@tmp_memory_table_size= @small_heap; +SET @@max_heap_table_size= @small_heap; +# --- one blob argument --- +FLUSH STATUS; +SET @@debug_dbug= '+d,dup_pending_tmp_table_row'; +SELECT COUNT(DISTINCT v) AS distinct_values FROM t1; +distinct_values +1000 +SET @@debug_dbug= @save_debug_dbug; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# --- two blob arguments: the unique constraint spans both --- +FLUSH STATUS; +SET @@debug_dbug= '+d,dup_pending_tmp_table_row'; +SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1; +distinct_values +1000 +SET @@debug_dbug= @save_debug_dbug; +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; +CONVERTED +ON +# Clean up +SET @@tmp_memory_table_size= @save_tmp_memory_table_size; +SET @@max_heap_table_size= @save_max_heap_table_size; +DROP TABLE t1; diff --git a/mysql-test/suite/heap/count_distinct_blob_convert_debug.test b/mysql-test/suite/heap/count_distinct_blob_convert_debug.test new file mode 100644 index 0000000000000..2c7f75e75ff6b --- /dev/null +++ b/mysql-test/suite/heap/count_distinct_blob_convert_debug.test @@ -0,0 +1,59 @@ +--source include/have_debug.inc +--source include/have_sequence.inc + +--echo # +--echo # A duplicate on the row that overflowed the in-memory temporary table +--echo # of COUNT(DISTINCT ) must be reported back to the aggregate as +--echo # "value already seen", not as a fatal write error. +--echo # +--echo # Which write is the one that overflows the table, and so whether the +--echo # row the conversion has left to write is a duplicate of a row it has +--echo # already copied, depends on the order in which the in-memory engine +--echo # happens to allocate record slots and blob values. That order is not +--echo # stable across platforms, so the duplicate is injected here instead: +--echo # the write of the pending row reports one whatever the row holds. The +--echo # aggregate has to answer with the number of distinct values either way. +--echo # + +SET @save_tmp_memory_table_size= @@tmp_memory_table_size; +SET @save_max_heap_table_size= @@max_heap_table_size; +SET @save_debug_dbug= @@debug_dbug; +SET @small_heap= 65536; + +--echo # +--echo # 1000 distinct values, each present twice. The injected duplicate +--echo # discards the pending row, and whichever of the two copies that was, +--echo # the other one is still written, so the answer is the same either way. +--echo # +--echo # The values are wide enough that the table is certain to overflow. +--echo # + +CREATE TABLE t1 (id INT PRIMARY KEY, v TEXT, w TEXT) ENGINE=MyISAM; +INSERT INTO t1 +SELECT seq, + LPAD((seq+1) DIV 2, 200, 'x'), + LPAD((seq+1) DIV 2, 300, 'y') +FROM seq_1_to_2000; + +SET @@tmp_memory_table_size= @small_heap; +SET @@max_heap_table_size= @small_heap; + +--echo # --- one blob argument --- +FLUSH STATUS; +SET @@debug_dbug= '+d,dup_pending_tmp_table_row'; +SELECT COUNT(DISTINCT v) AS distinct_values FROM t1; +SET @@debug_dbug= @save_debug_dbug; +--source count_distinct_converted.inc + +--echo # --- two blob arguments: the unique constraint spans both --- +FLUSH STATUS; +SET @@debug_dbug= '+d,dup_pending_tmp_table_row'; +SELECT COUNT(DISTINCT v, w) AS distinct_values FROM t1; +SET @@debug_dbug= @save_debug_dbug; +--source count_distinct_converted.inc + +--echo # Clean up + +SET @@tmp_memory_table_size= @save_tmp_memory_table_size; +SET @@max_heap_table_size= @save_max_heap_table_size; +DROP TABLE t1; diff --git a/mysql-test/suite/heap/count_distinct_converted.inc b/mysql-test/suite/heap/count_distinct_converted.inc new file mode 100644 index 0000000000000..fc7975456270a --- /dev/null +++ b/mysql-test/suite/heap/count_distinct_converted.inc @@ -0,0 +1,26 @@ +# Report whether the aggregate that ran just before this file was sourced +# converted its temporary table to the on-disk engine. +# +# Reading INFORMATION_SCHEMA materializes a temporary table of its own, and +# its VARIABLE_VALUE column is wide enough to be stored as a blob, so under +# the shrunken in-memory limit that table can overflow and be converted as +# well, and would then report its own conversion rather than the aggregate's. +# The limit is restored for the duration of the read and shrunk again +# afterwards. +# +# Expects @save_tmp_memory_table_size, @save_max_heap_table_size and +# @small_heap to be set by the test. + +--disable_query_log +SET @@tmp_memory_table_size= @save_tmp_memory_table_size; +SET @@max_heap_table_size= @save_max_heap_table_size; +--enable_query_log + +SELECT IF(VARIABLE_VALUE > 0, 'ON', 'OFF') AS CONVERTED +FROM INFORMATION_SCHEMA.SESSION_STATUS +WHERE VARIABLE_NAME = 'CREATED_TMP_DISK_TABLES'; + +--disable_query_log +SET @@tmp_memory_table_size= @small_heap; +SET @@max_heap_table_size= @small_heap; +--enable_query_log diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 64c06f45bf6b9..88d7462bdf491 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -1012,11 +1012,18 @@ bool Aggregator_distinct::add() converts it to an on-disk engine and copies all rows plus the overflow row (record[0]). For any other error it reports a fatal error and returns 1. + + The overflow row has not been checked against the unique + constraint of the converted table yet, and it may well be a + duplicate of a row that was copied there before it. That is the + same condition the write above returns FALSE for, so ignore it + here as well; the value it holds is already counted, which is + why *is_duplicate is not read afterwards. */ if (create_internal_tmp_table_from_heap(table->in_use, table, tmp_table_param->start_recinfo, &tmp_table_param->recinfo, - error, 0, &is_duplicate, NULL)) + error, 1, &is_duplicate, NULL)) return TRUE; } return FALSE; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index d5b6565d626c2..95877662a5171 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -24037,7 +24037,10 @@ int Tmp_table_default_copier::copy_rows(TABLE *from, TABLE *to) } /* copy row that filled HEAP table */ - if (unlikely((write_err= to->file->ha_write_tmp_row(from->record[0])))) + write_err= to->file->ha_write_tmp_row(from->record[0]); + DBUG_EXECUTE_IF("dup_pending_tmp_table_row", + write_err= HA_ERR_FOUND_DUPP_UNIQUE ;); + if (unlikely(write_err)) { if (to->file->is_fatal_error(write_err, HA_CHECK_DUP) || !ignore_last_dupp_key_error)