MySQL派生表查询大数据量无结果问题分析与解决-优快云博客

本文链接：https://blog.youkuaiyun.com/GreatSQL2021/article/details/148784040

MySQL派生表查询大数据量无结果问题分析与解决

一、问题发现

在客户现场的一次问题报告中发现某个带有派生表进行查询的时候，数据量少的时候有结果，但是数据量大的时候返回无记录。看下面例子：

1、准备表

CREATE TABLE `cmdb_item` (
  `cm_item_id` varchar(350) NOT NULL,
  `cm_model_id` varchar(350) NOT NULL,
  PRIMARY KEY (`cm_item_id`,`cm_model_id`)
);
CREATE TABLE `sys_auth_role_cmdb` (
  `id` varchar(60) NOT NULL,
  `auth_type_id` varchar(60) DEFAULT NULL,
  `cmdb_item_id` varchar(60) DEFAULT NULL,
  `cmdb_model_id` varchar(60) DEFAULT NULL,
  `role_id` varchar(60) DEFAULT NULL,
  PRIMARY KEY (`id`)
);

2、一条数据场合

如下所示插入一条数据并且进行派生表查询，可以发现结果有一条数据，符合预期。

greatsql> SELECT b.* FROM( SELECT cmdb_item_id FROM sys_auth_role_cmdb where1=0 union SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' and
cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';

+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| cm_item_id                                                                                                                                                                                                                                                       | cm_model_id                                                                                                                                                                                                                                                      |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J | PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (3.50 sec)

3、插入大量数据的场合

-- 首先设置tmp_table_size参数为合适的值
SET LOCAL tmp_table_size=2000000;

set sql_mode="oracle";
delimiter $$
CREATE or replace PROCEDURE p1() as
BEGIN
  for i in 1 .. 5000 loop
    insert into cmdb_item values('8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J'|| i,'PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' || i);
  end loop;
END$$
delimiter ;
--插入5000条数据
call p1();

--执行跟上面一样的查询，可以发现结果为空，此处为问题
greatsql> SELECT b.* FROM( SELECT cmdb_item_id FROM sys_auth_role_cmdb where1=0 union SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' and cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';
Empty set (0.00 sec)

4、tmp_table_size值最小的场合

-- 设置tmp_table_size参数为最小值
SET LOCAL tmp_table_size=1024;

-- 执行跟上面一样的查询，可以发现结果跟只有一条数据时候一样，出现一条数据
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| cm_item_id                                                                                                                                                                                                                                                       | cm_model_id                                                                                                                                                                                                                                                      |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J | PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (3.50 sec)

5、对比执行计划

查看上面几个场合的执行计划，结果如下：

数据量为1的场合执行计划：

greatsql> explain analyze SELECT b.* FROM( SELECT cmdb_item_id FROM sys_auth_role_cmdb where1=0 union SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' and cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| EXPLAIN                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| -> Constant row from a  (cost=1207.97..1207.97 rows=1) (actual time=1107.026..1107.027 rows=1 loops=1)
    -> Union materialize with deduplication  (cost=1207.62..1207.62 rows=4553) (actual time=1106.831..1106.831 rows=5001 loops=1)
        -> Zero rows (Impossible WHERE)  (cost=0.00..0.00 rows=0) (actual time=0.000..0.000 rows=0 loops=1)
        -> Covering index scan on cmdb_item using PRIMARY  (cost=752.32 rows=4553) (actual time=0.723..69.643 rows=5001 loops=1)
 |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (1.11 sec)

数据量为5001的场合执行计划，可以看到下面的执行计划跟上面是一样的，不同点在于最后一步Constant row from a 这段的rows=0，说明执行计划没问题，但是执行阶段有问题。

greatdb> explain analyze SELECT b.* FROM ( SELECT cmdb_item_id FROM sys_auth_role_cmdb WHERE 1=0 UNION SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' AND cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| EXPLAIN                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| -> Constant row from a  (cost=1053.17..1053.17 rows=1) (actual time=1118.948..1118.948 rows=0 loops=1)这个地方rows=0说明问题在这里
    -> Union materialize with deduplication  (cost=1052.82..1052.82 rows=4553) (actual time=1118.946..1118.946 rows=5001 loops=1)
        -> Zero rows (Impossible WHERE)  (cost=0.00..0.00 rows=0) (actual time=0.001..0.001 rows=0 loops=1)
        -> Covering index scan on cmdb_item using PRIMARY  (cost=597.52 rows=4553) (actual time=2.315..70.077 rows=5001 loops=1)
 |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (1.13 sec)

二、问题调查过程

查询带有派生表的时候内部会创建临时表用于保存临时数据，因此先看一下上面几个场合的临时表情况：

tmp_table_size值	数据量	查询开始时候临时表情况	查询过程中临时表情况	说明
2000000	1条数据	内存表	内存表	没问题
2000000	5001条数据	内存表	转为落盘表	问题
1024	5001条数据	落盘表	落盘表	没问题

先看一下读取数据的代码：

int ConstIterator::Read() {
  if (!m_first_record_since_init) {
    return -1;
  }
  m_first_record_since_init = false;
  // 这里打印err结果为-1，跳进read_const看一下为何没有数据返回
  int err = read_const(table(), m_ref);
  if (err == 0 && m_examined_rows != nullptr) {
    ++*m_examined_rows;
  }
  table()->const_table = true;
  return err;
}

int read_const(TABLE *table, Index_lookup *ref) {
  int error;
  DBUG_TRACE;
  // 数据为一条的时候会进入这个if线正常读数据，但是数据为5001条的时候却没进入这个if线，可见问题在table的这个状态。
  // 这里要求table->m_status = STATUS_NOT_STARTED才会进入
  if (!table->is_started())  // If first read
    这里面读取表数据
  } else if (table->has_row() && table->is_nullable()) {}
  return table->has_row() ? 0 : -1;
}

考虑上面的现象，数据量少和数据量多的场合结果不一样，这里面的区别在于union在查询阶段会创建临时物化表用于储存union结果，数据量少的时候在内存就可以创建物化表出结果了，但是如果数据量大的话，因为内存不够放，因此需要重新创建落盘表用于储存查询结果。

又考虑到上面<<tmp_table_size值最小的场合>>这节，把内存表值改小结果又正常，因此可以推测是内存临时表改为落盘表后table的is_started()状态值改变了导致结果为空。这里可以从创建落盘表入手。找到创建落盘表的函数create_ondisk_from_heap()并且打个断点，看看堆栈：

#0  create_ondisk_from_heap (
    thd=0x55555accb1a0 <native_rw_unlock(native_rw_lock_t*)+24>, wtable=0x7fffc83eddc0, 
    error=0, insert_last_record=false, ignore_last_dup=false, is_duplicate=0x0)
    at sql/sql_tmp_table.cc:2777
#1  0x0000555559b98fb2 in MaterializeIterator<DummyIteratorProfiler>::MaterializeQueryBlock (this=0x7fff2023f390, query_block=..., stored_rows=0x7fffc83eef10)
    at sql/iterators/composite_iterators.cc:1433
#2  0x0000555559b9649e in MaterializeIterator<DummyIteratorProfiler>::Init (
    this=0x7fff2023f390)
    at sql/iterators/composite_iterators.cc:975
#3  0x00005555597dae49 in Query_expression::ExecuteIteratorQuery (this=0x7fff2026d310, 
    thd=0x7fff20001050)
    at sql/sql_union.cc:1814
#4  0x00005555597db287 in Query_expression::execute (this=0x7fff2026d310, 
    thd=0x7fff20001050)
    at sql/sql_union.cc:1877
#5  0x0000555559700334 in Sql_cmd_dml::execute_inner (this=0x7fff20224010, 
    thd=0x7fff20001050)
    at sql/sql_select.cc:872
#6  0x00005555596ff502 in Sql_cmd_dml::execute (this=0x7fff20224010, thd=0x7fff20001050)
    at sql/sql_select.cc:612
#7  0x00005555596621f0 in mysql_execute_command (thd=0x7fff20001050, first_level=true)
    at sql/sql_parse.cc:5227
#8  0x0000555559664838 in dispatch_sql_command (thd=0x7fff20001050, 
    parser_state=0x7fffc83f0760, update_userstat=false)
    at sql/sql_parse.cc:5905
#9  0x0000555559658b03 in dispatch_command (thd=0x7fff20001050, com_data=0x7fffc83f1210, 
    command=COM_QUERY)

查看MaterializeIterator::MaterializeQueryBlock函数代码，追踪一下table->is_started()状态值：

bool MaterializeIterator<Profiler>::MaterializeQueryBlock(
    error = t->file->ha_write_row(t->record[0]);

    if (error == 0) {
      ++*stored_rows;
      continue;
    }
    // create_ondisk_from_heap will generate error if needed.
    if (!t->file->is_ignorable_error(error)) {
      bool is_duplicate;
      // 这里table->m_status = STATUS_NOT_STARTED | STATUS_NOT_FOUND，正常初始值，符合预期
      if (create_ondisk_from_heap(thd(), t, error,
                                  /*insert_last_record=*/true,
                                  /*ignore_last_dup=*/true, &is_duplicate))
        return true; /* purecov: inspected */
      // 这里执行完create_ondisk_from_heap以后table->m_status = STATUS_NOT_FOUND，可见在create_ondisk_from_heap函数内部改变了这个m_status
}
查看create_ondisk_from_heap代码，看一下哪个地方改变了table->m_status值。
bool create_ondisk_from_heap(THD *thd, TABLE *wtable, int error,
                             bool insert_last_record, bool ignore_last_dup,
                             bool *is_duplicate) {
        // 执行下面这句的时候因为调用了table->set_row_status_from_handler(result)改变了m_status值，先从3变为0最后变为2
        // 状态迁移情况：STATUS_NOT_STARTED | STATUS_NOT_FOUND ==> 0 ==> STATUS_NOT_FOUND
        while (!table->file->ha_rnd_next(new_table.record[1])) {
          write_err = new_table.file->ha_write_row(new_table.record[1]);
          DBUG_EXECUTE_IF("raise_error", write_err = HA_ERR_FOUND_DUPP_KEY;);
          if (write_err) goto err_after_open;
        }
}

发现果然是create_ondisk_from_heap函数改变了table->is_started()状态值，所以导致了结果为空。

三、问题解决

结合上面分析，我们可以把代码作如下修复，就可以解决这个问题了。

bool MaterializeIterator<Profiler>::MaterializeQueryBlock(
    error = t->file->ha_write_row(t->record[0]);

    if (error == 0) {
      ++*stored_rows;
      continue;
    }
    // create_ondisk_from_heap will generate error if needed.
    if (!t->file->is_ignorable_error(error)) {
      bool is_duplicate;
      if (create_ondisk_from_heap(thd(), t, error,
                                  /*insert_last_record=*/true,
                                  /*ignore_last_dup=*/true, &is_duplicate))
        return true; /* purecov: inspected */
      // 这里把table->is_started()状态复原即可，问题解决。
      t->set_not_started();
}

接着执行上面的查询，发现可以查出结果了。

四、问题总结

通过以上分析我们可以发现，带有派生表的查询会在内部创建临时表用于储存中间数据，根据tmp_table_size值设置会影响临时表存放地方，如果tmp_table_size设置小了那么一开始就会创建落盘表，如果设置大了那么创建的就是内存表，随着数据的增加后面可能会转为落盘表，因此才有了本文章提及的bug。分析类似问题的时候，需要结合查询计划的执行和临时表创建情况分析，才能更好解决问题。