MySQL派生表查询大数据量无结果问题分析与解决

关注

原创最新推荐文章于 2026-08-16 21:42:49 发布 438 阅读 AI 写同款· GEO 优化 ›

通过子查询优化MySQL复杂数据统计查询的性能技巧例如，统计每个部门的员工数量时，可以使用派生表代替WHERE子句中的子查询，这样MySQL优化器可以更有效地制定执行计划，减少重复计算。在统计查询中，如果子查询需要访问的列都包含在索引中，MySQL可以直接使用索引完成查询而无需访问数据表，这被称为覆盖索引扫描，能极大减少磁盘I/O操作。性能瓶颈主要出现在相关子查询对外部查询的每一行都执行一次内部查询，造成所谓的N+1查询问题。对于大数据集，这种差异会显著影响查询速度。此外，确保EXISTS子查询中的条件列有适当的索引，可以进一步加速查询执行过程。阅读详情

MySQL派生表查询大数据量无结果问题分析与解决

一、问题发现

在客户现场的一次问题报告中发现某个带有派生表进行查询的时候，数据量少的时候有结果，但是数据量大的时候返回无记录。看下面例子：

1、准备表

CREATE TABLE `cmdb_item` (
  `cm_item_id` varchar(350) NOT NULL,
  `cm_model_id` varchar(350) NOT NULL,
  PRIMARY KEY (`cm_item_id`,`cm_model_id`)
);
CREATE TABLE `sys_auth_role_cmdb` (
  `id` varchar(60) NOT NULL,
  `auth_type_id` varchar(60) DEFAULT NULL,
  `cmdb_item_id` varchar(60) DEFAULT NULL,
  `cmdb_model_id` varchar(60) DEFAULT NULL,
  `role_id` varchar(60) DEFAULT NULL,
  PRIMARY KEY (`id`)
);

2、一条数据场合

如下所示插入一条数据并且进行派生表查询，可以发现结果有一条数据，符合预期。

greatsql> SELECT b.* FROM( SELECT cmdb_item_id FROM sys_auth_role_cmdb where1=0 union SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' and
cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';

+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| cm_item_id                                                                                                                                                                                                                                                       | cm_model_id                                                                                                                                                                                                                                                      |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J | PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (3.50 sec)

3、插入大量数据的场合

-- 首先设置tmp_table_size参数为合适的值
SET LOCAL tmp_table_size=2000000;

set sql_mode="oracle";
delimiter $$
CREATE or replace PROCEDURE p1() as
BEGIN
  for i in 1 .. 5000 loop
    insert into cmdb_item values('8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J'|| i,'PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' || i);
  end loop;
END$$
delimiter ;
--插入5000条数据
call p1();

--执行跟上面一样的查询，可以发现结果为空，此处为问题
greatsql> SELECT b.* FROM( SELECT cmdb_item_id FROM sys_auth_role_cmdb where1=0 union SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' and cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';
Empty set (0.00 sec)

4、tmp_table_size值最小的场合

-- 设置tmp_table_size参数为最小值
SET LOCAL tmp_table_size=1024;

-- 执行跟上面一样的查询，可以发现结果跟只有一条数据时候一样，出现一条数据
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| cm_item_id                                                                                                                                                                                                                                                       | cm_model_id                                                                                                                                                                                                                                                      |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J | PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq |
+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (3.50 sec)

5、对比执行计划

查看上面几个场合的执行计划，结果如下：

数据量为1的场合执行计划：

greatsql> explain analyze SELECT b.* FROM( SELECT cmdb_item_id FROM sys_auth_role_cmdb where1=0 union SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' and cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| EXPLAIN                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| -> Constant row from a  (cost=1207.97..1207.97 rows=1) (actual time=1107.026..1107.027 rows=1 loops=1)
    -> Union materialize with deduplication  (cost=1207.62..1207.62 rows=4553) (actual time=1106.831..1106.831 rows=5001 loops=1)
        -> Zero rows (Impossible WHERE)  (cost=0.00..0.00 rows=0) (actual time=0.000..0.000 rows=0 loops=1)
        -> Covering index scan on cmdb_item using PRIMARY  (cost=752.32 rows=4553) (actual time=0.723..69.643 rows=5001 loops=1)
 |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (1.11 sec)

数据量为5001的场合执行计划，可以看到下面的执行计划跟上面是一样的，不同点在于最后一步Constant row from a 这段的rows=0，说明执行计划没问题，但是执行阶段有问题。

greatdb> explain analyze SELECT b.* FROM ( SELECT cmdb_item_id FROM sys_auth_role_cmdb WHERE 1=0 UNION SELECT cm_item_id FROM cmdb_item) a LEFT JOIN cmdb_item b ON a.cmdb_item_id = b.cm_item_id WHERE b.cm_model_id='PmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmqPmqECfANKxmbpgt3XBWaxzF6dx32dLmq' AND cm_item_id='8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J8b0zIlHsiarv8Ls6NejWji6Cu4h3iN2J';
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| EXPLAIN                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| -> Constant row from a  (cost=1053.17..1053.17 rows=1) (actual time=1118.948..1118.948 rows=0 loops=1)这个地方rows=0说明问题在这里
    -> Union materialize with deduplication  (cost=1052.82..1052.82 rows=4553) (actual time=1118.946..1118.946 rows=5001 loops=1)
        -> Zero rows (Impossible WHERE)  (cost=0.00..0.00 rows=0) (actual time=0.001..0.001 rows=0 loops=1)
        -> Covering index scan on cmdb_item using PRIMARY  (cost=597.52 rows=4553) (actual time=2.315..70.077 rows=5001 loops=1)
 |
+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
1 row in set (1.13 sec)

二、问题调查过程

查询带有派生表的时候内部会创建临时表用于保存临时数据，因此先看一下上面几个场合的临时表情况：

tmp_table_size值	数据量	查询开始时候临时表情况	查询过程中临时表情况	说明
2000000	1条数据	内存表	内存表	没问题
2000000	5001条数据	内存表	转为落盘表	问题
1024	5001条数据	落盘表	落盘表	没问题

先看一下读取数据的代码：

int ConstIterator::Read() {
  if (!m_first_record_since_init) {
    return -1;
  }
  m_first_record_since_init = false;
  // 这里打印err结果为-1，跳进read_const看一下为何没有数据返回
  int err = read_const(table(), m_ref);
  if (err == 0 && m_examined_rows != nullptr) {
    ++*m_examined_rows;
  }
  table()->const_table = true;
  return err;
}

int read_const(TABLE *table, Index_lookup *ref) {
  int error;
  DBUG_TRACE;
  // 数据为一条的时候会进入这个if线正常读数据，但是数据为5001条的时候却没进入这个if线，可见问题在table的这个状态。
  // 这里要求table->m_status = STATUS_NOT_STARTED才会进入
  if (!table->is_started())  // If first read
    这里面读取表数据
  } else if (table->has_row() && table->is_nullable()) {}
  return table->has_row() ? 0 : -1;
}

考虑上面的现象，数据量少和数据量多的场合结果不一样，这里面的区别在于union在查询阶段会创建临时物化表用于储存union结果，数据量少的时候在内存就可以创建物化表出结果了，但是如果数据量大的话，因为内存不够放，因此需要重新创建落盘表用于储存查询结果。

又考虑到上面<<tmp_table_size值最小的场合>>这节，把内存表值改小结果又正常，因此可以推测是内存临时表改为落盘表后table的is_started()状态值改变了导致结果为空。这里可以从创建落盘表入手。找到创建落盘表的函数create_ondisk_from_heap()并且打个断点，看看堆栈：

#0  create_ondisk_from_heap (
    thd=0x55555accb1a0 <native_rw_unlock(native_rw_lock_t*)+24>, wtable=0x7fffc83eddc0, 
    error=0, insert_last_record=false, ignore_last_dup=false, is_duplicate=0x0)
    at sql/sql_tmp_table.cc:2777
#1  0x0000555559b98fb2 in MaterializeIterator<DummyIteratorProfiler>::MaterializeQueryBlock (this=0x7fff2023f390, query_block=..., stored_rows=0x7fffc83eef10)
    at sql/iterators/composite_iterators.cc:1433
#2  0x0000555559b9649e in MaterializeIterator<DummyIteratorProfiler>::Init (
    this=0x7fff2023f390)
    at sql/iterators/composite_iterators.cc:975
#3  0x00005555597dae49 in Query_expression::ExecuteIteratorQuery (this=0x7fff2026d310, 
    thd=0x7fff20001050)
    at sql/sql_union.cc:1814
#4  0x00005555597db287 in Query_expression::execute (this=0x7fff2026d310, 
    thd=0x7fff20001050)
    at sql/sql_union.cc:1877
#5  0x0000555559700334 in Sql_cmd_dml::execute_inner (this=0x7fff20224010, 
    thd=0x7fff20001050)
    at sql/sql_select.cc:872
#6  0x00005555596ff502 in Sql_cmd_dml::execute (this=0x7fff20224010, thd=0x7fff20001050)
    at sql/sql_select.cc:612
#7  0x00005555596621f0 in mysql_execute_command (thd=0x7fff20001050, first_level=true)
    at sql/sql_parse.cc:5227
#8  0x0000555559664838 in dispatch_sql_command (thd=0x7fff20001050, 
    parser_state=0x7fffc83f0760, update_userstat=false)
    at sql/sql_parse.cc:5905
#9  0x0000555559658b03 in dispatch_command (thd=0x7fff20001050, com_data=0x7fffc83f1210, 
    command=COM_QUERY)

查看MaterializeIterator::MaterializeQueryBlock函数代码，追踪一下table->is_started()状态值：

bool MaterializeIterator<Profiler>::MaterializeQueryBlock(
    error = t->file->ha_write_row(t->record[0]);

    if (error == 0) {
      ++*stored_rows;
      continue;
    }
    // create_ondisk_from_heap will generate error if needed.
    if (!t->file->is_ignorable_error(error)) {
      bool is_duplicate;
      // 这里table->m_status = STATUS_NOT_STARTED | STATUS_NOT_FOUND，正常初始值，符合预期
      if (create_ondisk_from_heap(thd(), t, error,
                                  /*insert_last_record=*/true,
                                  /*ignore_last_dup=*/true, &is_duplicate))
        return true; /* purecov: inspected */
      // 这里执行完create_ondisk_from_heap以后table->m_status = STATUS_NOT_FOUND，可见在create_ondisk_from_heap函数内部改变了这个m_status
}
查看create_ondisk_from_heap代码，看一下哪个地方改变了table->m_status值。
bool create_ondisk_from_heap(THD *thd, TABLE *wtable, int error,
                             bool insert_last_record, bool ignore_last_dup,
                             bool *is_duplicate) {
        // 执行下面这句的时候因为调用了table->set_row_status_from_handler(result)改变了m_status值，先从3变为0最后变为2
        // 状态迁移情况：STATUS_NOT_STARTED | STATUS_NOT_FOUND ==> 0 ==> STATUS_NOT_FOUND
        while (!table->file->ha_rnd_next(new_table.record[1])) {
          write_err = new_table.file->ha_write_row(new_table.record[1]);
          DBUG_EXECUTE_IF("raise_error", write_err = HA_ERR_FOUND_DUPP_KEY;);
          if (write_err) goto err_after_open;
        }
}

发现果然是create_ondisk_from_heap函数改变了table->is_started()状态值，所以导致了结果为空。

三、问题解决

结合上面分析，我们可以把代码作如下修复，就可以解决这个问题了。

bool MaterializeIterator<Profiler>::MaterializeQueryBlock(
    error = t->file->ha_write_row(t->record[0]);

    if (error == 0) {
      ++*stored_rows;
      continue;
    }
    // create_ondisk_from_heap will generate error if needed.
    if (!t->file->is_ignorable_error(error)) {
      bool is_duplicate;
      if (create_ondisk_from_heap(thd(), t, error,
                                  /*insert_last_record=*/true,
                                  /*ignore_last_dup=*/true, &is_duplicate))
        return true; /* purecov: inspected */
      // 这里把table->is_started()状态复原即可，问题解决。
      t->set_not_started();
}

接着执行上面的查询，发现可以查出结果了。

四、问题总结

通过以上分析我们可以发现，带有派生表的查询会在内部创建临时表用于储存中间数据，根据tmp_table_size值设置会影响临时表存放地方，如果tmp_table_size设置小了那么一开始就会创建落盘表，如果设置大了那么创建的就是内存表，随着数据的增加后面可能会转为落盘表，因此才有了本文章提及的bug。分析类似问题的时候，需要结合查询计划的执行和临时表创建情况分析，才能更好解决问题。