【音视频】使用ffmpeg gdigrab采集桌面和程序窗口（4-1）

最近在研究程序窗口截流，因为之前用过ffmpeg的gdigrab，所以知道gdigrab也可以截取窗口图片，但实际上没用过。经过研究，通过将avformat_open_input中url设置为“title={程序名}"获取format context从而得到图片。

1、初始化gdigrab采集器

int GdigrabCaptor::init(const std::string& deviceId, const int fps)
{
	int err = ERROR_CODE_OK;
	if (m_inited) {
		return err;
	}

	do {
		if (deviceId.empty()) {
			err = ERROR_CODE_PARAMS_ERROR;
			break;
		}

		m_deviceId = deviceId;

		RECT rect = {};
		err = initDevice(m_deviceId, m_deviceName, rect);
		HCMDR_ERROR_CODE_BREAK(err);

		avdevice_register_all();
		avformat_network_init();
		m_inputFmt = av_find_input_format("gdigrab");
		if (m_inputFmt == nullptr) {
			err = ERROR_CODE_FFMPEG_FIND_INPUT_FORMAT_FAILED;
			break;
		}

		m_rect = rect;
		m_fps = fps;
		if (m_fps < 0) {
			m_fps = GDIGRAB_CAPTOR_MIN_FRAMERATE;
		}
		else if (m_fps > GDIGRAB_CAPTOR_MAX_FRAMERATE) {
			m_fps = GDIGRAB_CAPTOR_MAX_FRAMERATE;
		}
		err = initGdigrab(m_deviceName, m_rect, m_fps);
		HCMDR_ERROR_CODE_BREAK(err);

		m_pixelFmt = AV_PIX_FMT_BGRA;
		m_timebase = { 1, AV_TIME_BASE };

		m_inited = true;
	} while (0);

	if (err != ERROR_CODE_OK) {
		LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] init camera captor error: %s, last error: %lu",
			__FUNCTION__, HCMDR_GET_ERROR_DESC(err), GetLastError());
		cleanup();
	}

	return err;
}

其中initDevice中，根据deviceId获取HMONITOR或者HWND，然后获取显示器或者窗口大小、名称。需要注意的是，采集窗口时发现窗口最小化了，先将其显示出来，再获取其大小，也便于采集编码。

int GdigrabCaptor::initDevice(const std::string& deviceId, std::string& deviceName, RECT& deviceRect)
{
	int err = ERROR_CODE_OK;

	do {
		long handle = atol(deviceId.c_str());
		HMONITOR monitor = reinterpret_cast<HMONITOR>(handle);
		HWND hwnd = reinterpret_cast<HWND>(handle);
		MONITORINFO mi;
		mi.cbSize = sizeof(mi);
		if (GetMonitorInfo(monitor, &mi)) {					
			deviceName = "desktop";
			deviceRect = mi.rcMonitor;
			m_isDesktop = true;
		}
		else {
			TCHAR title[_MAX_FNAME] = { 0 };
			RECT rect = {};
			// 将最小化窗口显示出来
			if (::IsIconic(hwnd)) {
				::ShowWindow(hwnd, SW_SHOWNOACTIVATE);
			}
			BOOL ret = GetWindowText(hwnd, title, sizeof(title)) && GetClientRect(hwnd, &rect);
			if (!ret) {
				err = ERROR_CODE_DEVICE_GET_MONITOR_FAILED;
				break;
			}
#ifdef UNICODE
			deviceName = HELPER::StringConverter::convertUnicodeToAscii(title);
#else
			deviceName = title;
#endif // UNICODE
			deviceRect = rect;
			m_isDesktop = false;
		}
	} while (0);

	return err;
}

再初始化格式上下文和解码上下文。初始化AVFormatContext时，采集桌面时url名称是”desktop“，然后根据offset_x和offset_y设置左上角位置，根据video_size设置采集大小，这样可以实现采集多屏幕的需求；采集程序窗口时url的名称为“title={程序名}"，同样根据video_size设置采集大小，同时根据draw_mouse设置不采集鼠标。

int GdigrabCaptor::initGdigrab(const std::string& deviceName, const RECT& deviceRect, int fps)
{
	int err = ERROR_CODE_OK;

	do {
		if (m_formatCtx == nullptr) {
			m_formatCtx = avformat_alloc_context();
		}
		if (m_formatCtx == nullptr) {
			err = ERROR_CODE_FFMPEG_ALLOC_CONTEXT_FAILED;
			break;
		}
		if (m_decodeCtx == nullptr) {
			m_decodeCtx = avcodec_alloc_context3(nullptr);
		}
		if (m_decodeCtx == nullptr) {
			err = ERROR_CODE_FFMPEG_ALLOC_CONTEXT_FAILED;
			break;
		}

		AVDictionary* dict = nullptr;
		std::string url(deviceName);
		std::string videoSize;
		videoSize.assign(std::to_string(deviceRect.right - deviceRect.left)).append("x").append(std::to_string(deviceRect.bottom - deviceRect.top));
		av_dict_set_int(&dict, "framerate", fps, 0);
		av_dict_set(&dict, "video_size", videoSize.c_str(), 0);
		if (deviceName == "desktop") {
			av_dict_set_int(&dict, "offset_x", deviceRect.left, 0);
			av_dict_set_int(&dict, "offset_y", deviceRect.top, 0);
		}
		else {
			// 采集程序时不采集鼠标
			av_dict_set_int(&dict, "draw_mouse", 0, 0);
			url.assign("title=").append(deviceName);
		}
		int ret = avformat_open_input(&m_formatCtx, url.c_str(), m_inputFmt, &dict);
		if (dict != nullptr) {
			av_dict_free(&dict);
		}
		if (ret != 0 || m_formatCtx == nullptr) {
			err = ERROR_CODE_FFMPEG_OPEN_INPUT_FAILED;
			break;
		}

		for (int i = 0; i < m_formatCtx->nb_streams; i++) {
			if (m_formatCtx->streams[i] != nullptr &&
				m_formatCtx->streams[i]->codecpar != nullptr &&
				m_formatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
				m_streamIndex = i;
				break;
			}
		}
		if (m_streamIndex < 0) {
			err = ERROR_CODE_FFMPEG_FIND_BEST_STREAM_FAILED;
			break;
		}

		ret = avcodec_parameters_to_context(m_decodeCtx, m_formatCtx->streams[m_streamIndex]->codecpar);
		if (ret < 0) {
			err = ERROR_CODE_FFMPEG_PARAMS_TO_CONTEXT_FAILED;
			break;
		}
		m_decodeCtx->pkt_timebase = m_formatCtx->streams[m_streamIndex]->time_base;

		AVCodec* codec = avcodec_find_decoder(m_decodeCtx->codec_id);
		if (codec == 0) {
			err = ERROR_CODE_FFPMEG_FIND_DECODER_FAILED;
			break;
		}
		m_decodeCtx->flags2 |= AV_CODEC_FLAG2_FAST;

		AVDictionary* opts = nullptr;
		av_dict_set(&opts, "threads", "auto", 0);
		av_dict_set(&opts, "refcounted_frames", "0", 0);
		ret = avcodec_open2(m_decodeCtx, codec, &opts);
		if (opts != nullptr) {
			av_dict_free(&opts);
		}
		if (ret < 0) {
			err = ERROR_CODE_FFMPEG_OPEN_CODEC_FAILED;
			break;
		}
		m_formatCtx->streams[m_streamIndex]->discard = AVDISCARD_DEFAULT;
	} while (0);

	if (err != ERROR_CODE_OK) {
		if (m_formatCtx != nullptr) {
			avformat_close_input(&m_formatCtx);
			m_formatCtx = nullptr;
		}
		if (m_decodeCtx != nullptr) {
			avcodec_free_context(&m_decodeCtx);
			m_decodeCtx = nullptr;
		}
	}

	return err;
}

2、开启采集线程

int GdigrabCaptor::start()
{
	int err = ERROR_CODE_OK;

	if (m_running) {
		LOGGER::Logger::log(LOGGER::LOG_TYPE_WARN, "[%s] gdi captor already running", __FUNCTION__);
		return err;
	}

	if (!m_inited) {
		err = ERROR_CODE_UNINITIALIZED;
		LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] gdi captor not yet initialized: %s",
			__FUNCTION__, HCMDR_GET_ERROR_DESC(err));
		return err;
	}

	m_running = true;
	m_thread = std::thread(std::bind(&GdigrabCaptor::captureProcess, this));

	return err;
}

其中采集线程实现如下。这里要注意的是如果采集的是窗口，需要判断窗口被关闭或者最小化的情况，被关闭了就不需要再采集了，最小化了就不发送数据出去。

void GdigrabCaptor::captureProcess()
{
	int err = ERROR_CODE_OK;

	AVPacket packet;
	AVFrame* frame = av_frame_alloc();
	while (m_running) {
		/* 重要：先处理采集程序异常情况 */
		if (!m_isDesktop) {
			long handle = atol(m_deviceId.c_str());
			HWND hwnd = reinterpret_cast<HWND>(handle);
			if (!IsWindow(hwnd)) {
				err = ERROR_CODE_NO_CAPTURE_SOURCE;
				LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] window is closed: %s", __FUNCTION__,
					HCMDR_GET_ERROR_DESC(err));
				if (m_onVideoCaptureError != nullptr) {
					m_onVideoCaptureError(err, m_index);
				}
				break;
			}
			if (IsIconic(hwnd)) {
				LOGGER::Logger::log(LOGGER::LOG_TYPE_INFO, "[%s] window is not visible", __FUNCTION__);
				av_usleep(AV_TIME_BASE / m_fps);
				continue;
			}
		}
		/* 判断是否重新初始化 */
		err = reinit();
		if (err != ERROR_CODE_OK) {
			err = ERROR_CODE_NO_CAPTURE_SOURCE;
			LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] reinit capture failed: %s", __FUNCTION__,
				HCMDR_GET_ERROR_DESC(err));
			if (m_onVideoCaptureError != nullptr) {
				m_onVideoCaptureError(err, m_index);
			}
			break;
		}
		
		int ret = av_read_frame(m_formatCtx, &packet);
		if (ret < 0) {
			err = ERROR_CODE_NO_CAPTURE_SOURCE;
			LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] read frame error: %s", __FUNCTION__,
				HCMDR_GET_ERROR_DESC(err));
			if (m_onVideoCaptureError != nullptr) {
				m_onVideoCaptureError(err, m_index);
			}
			break;
		}

		ret = avcodec_send_packet(m_decodeCtx, &packet);
		while (ret >= 0) {
			ret = avcodec_receive_frame(m_decodeCtx, frame);
			if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
				break;
			}
			else if (ret < 0) {
				err = ERROR_CODE_FFMPEG_DECODE_FAILED;
				LOGGER::Logger::log(LOGGER::LOG_TYPE_ERROR, "[%s] decode packet error: %s", __FUNCTION__,
					HCMDR_GET_ERROR_DESC(err));
				if (m_onVideoCaptureError != nullptr) {
					m_onVideoCaptureError(err, m_index);
				}
				break;
			}

			/* 重要：设置时间戳，否则生成的文件播放太慢 */
			frame->pts = frame->pkt_dts = av_gettime_relative();
			if (m_onVideoCaptureData != nullptr) {
				m_onVideoCaptureData(frame, m_index);
			}
		}
		if (ret == AVERROR_EOF) {
			avcodec_flush_buffers(m_decodeCtx);
		}
		av_packet_unref(&packet);
	}
	av_frame_free(&frame);
}

还需要特别注意一点的是重新初始化采集器的情况。当发现桌面分辨率或者窗口大小（经常会变化）变化时，需要重新初始化采集器。这样，可以实现窗口大小或者桌面大小变化时，实时采集的需求。

int GdigrabCaptor::reinit()
{
	int err = ERROR_CODE_OK;

	do {
		bool reinit = false;
		RECT rect = { 0 };
		long handle = atol(m_deviceId.c_str());
		if (m_isDesktop) {
			HMONITOR monitor = reinterpret_cast<HMONITOR>(handle);
			MONITORINFO mi;
			mi.cbSize = sizeof(mi);
			if (!GetMonitorInfo(monitor, &mi)) {
				err = ERROR_CODE_DEVICE_GET_MONITOR_FAILED;
				break;
			}
			rect = mi.rcMonitor;
			if (rect.left != m_rect.left || rect.top != m_rect.top ||
				rect.right - rect.left != m_rect.right - m_rect.left ||
				rect.bottom - rect.top != m_rect.bottom - m_rect.top) {
				reinit = true;
			}
		}
		else {
			HWND hwnd = reinterpret_cast<HWND>(handle);
			BOOL ret = GetClientRect(hwnd, &rect);
			if (!ret) {
				err = ERROR_CODE_DEVICE_GET_MONITOR_FAILED;
				break;
			}
			if (rect.right - rect.left != m_rect.right - m_rect.left ||
				rect.bottom - rect.top != m_rect.bottom - m_rect.top) {
				reinit = true;
			}
		}

		if (reinit) {
			cleanup();
			err = init(m_deviceId, m_fps);
			HCMDR_ERROR_CODE_BREAK(err);
		}
	} while (0);

	return err;
}

3、结束采集

int GdigrabCaptor::stop()
{
	int err = ERROR_CODE_OK;

	if (!m_running) {
		return err;
	}

	m_running = false;
	if (m_thread.joinable()) {
		m_thread.join();
	}

	return err;
}

4、适配视频转换器（参考《【音视频】视频转码-SWS（二）》）

在转换函数中也需要添加重启转换器的函数reinit

int FfmpegSwsTranscoder::transcode(const AVFrame* srcFrame, AVFrame** dstFrame, uint8_t** dstData, uint32_t* dstLen)
{
	int err = ERROR_CODE_OK;

	if (!m_inited || m_swsContext == nullptr) {
		err = ERROR_CODE_UNINITIALIZED;
		return err;
	}
	if (srcFrame == nullptr || dstFrame == nullptr || dstData == nullptr || dstLen == nullptr) {
		err = ERROR_CODE_PARAMS_ERROR;
		return err;
	}

	err = reinit(static_cast<AVPixelFormat>(srcFrame->format), srcFrame->width, srcFrame->height,
		static_cast<AVPixelFormat>(m_frame->format), m_frame->width, m_frame->height);
	HCMDR_ERROR_CODE_RETURN(err);

	int ret = sws_scale(m_swsContext, srcFrame->data, srcFrame->linesize, 0, srcFrame->height, 
		m_frame->data, m_frame->linesize);
	if (ret <= 0) {
		err = ERROR_CODE_FFMPEG_SWS_SCALE_FAILED;
		return err;
	}

	m_frame->pict_type = srcFrame->pict_type;
	m_frame->pts = srcFrame->pts;
	m_frame->pkt_dts = srcFrame->pkt_dts;

	*dstFrame = m_frame;
	*dstData = m_buffer;
	*dstLen = m_bufferSize;

	return err;
}

reinit函数如下。当新的一帧的像素格式、像素宽、像素高跟上一帧不一样时，需要重启转换器。

int FfmpegSwsTranscoder::reinit(AVPixelFormat srcFmt, int srcWidth, int srcHeight,
			AVPixelFormat dstFmt, int dstWidth, int dstHeight)
{
	int err = ERROR_CODE_OK;

	if (m_srcPixelFmt != srcFmt || m_srcWidth != srcWidth || m_srcHeight != srcHeight) {
		if (m_swsContext != nullptr) {
			sws_freeContext(m_swsContext);
		}
		m_swsContext = sws_getContext(srcWidth, srcHeight, srcFmt, dstWidth, dstHeight, dstFmt,
			SWS_BICUBIC, nullptr, nullptr, nullptr);
		if (m_swsContext == nullptr) {
			err = ERROR_CODE_FFMPEG_GET_SWS_CONTEXT_FAILED;
			return err;
		}
		m_srcPixelFmt = srcFmt;
		m_srcWidth = srcWidth;
		m_srcHeight = srcHeight;
	}

	return err;
}

5、总结一下ffmpeg gdigrab使用感受

满足采集桌面、程序的需求：
1、可以满足采集桌面（包括采集辅屏）的需求
2、可以实现采集窗口的需求（包括被遮挡或屏幕外的窗口）
但是缺点也比较明显：
1、采集的帧率不会太高，一般不会超过20帧
2、采集窗口的缺陷尤为明显，比如，
（1）当有两个同名窗口存在时，它只会采集最后被激活的窗口，采集不到另一个窗口
（2）只能采集某些窗口的客户区域，如果使用GetWindowRect传入给video_size，采集会报错，所以我一律使用GetClientRect获取大小
（3）采集某些窗口报错，我在采集”Windows Media Player“的时候，有时候采集不到，或者刚开始可以采集，改变窗口大小时就采集不到了（不知道是不是我电脑的问题？）
总的来说，ffmpeg gdigrab算是还可以的桌面采集器（程序采集兼容性不太好），如果对采集桌面的帧率不会太高或者采集程序窗口要求较低，可以考虑使用它。