UVCCamera-Android 预览画面采集帧率提高

UVCCamera 使用过程的画面帧率优化实践

引言

框架项目地址：

saki4510t/UVCCamera: library and sample to access to UVC web camera on non-rooted Android device

问题背景：

        在使用此框架过程进行USB画面采集过程，在3568项目上，进行1080p分辨率预览，发现其帧率大概在17-18FPS范围，无法满足实际项目需求

技术实现方法

        经过框架源码的学习，定位到以下与图形编解码相关的代码片段

void UVCPreview::do_preview(uvc_stream_ctrl_t *ctrl) {
	ENTER();

	uvc_frame_t *frame = NULL;
	uvc_frame_t *frame_mjpeg = NULL;
	uvc_error_t result = uvc_start_streaming_bandwidth(
		mDeviceHandle, ctrl, uvc_preview_frame_callback, (void *)this, requestBandwidth, 0);

	if (LIKELY(!result)) {
		clearPreviewFrame();
		pthread_create(&capture_thread, NULL, capture_thread_func, (void *)this);

#if LOCAL_DEBUG
		LOGI("Streaming...");
#endif
		if (frameMode) {
			// MJPEG mode
			for ( ; LIKELY(isRunning()) ; ) {
				frame_mjpeg = waitPreviewFrame();
				if (LIKELY(frame_mjpeg)) {
					frame = get_frame(frame_mjpeg->width * frame_mjpeg->height * 2);
					result = uvc_mjpeg2yuyv(frame_mjpeg, frame);   // MJPEG => yuyv
					recycle_frame(frame_mjpeg);
					if (LIKELY(!result)) {
						frame = draw_preview_one(frame, &mPreviewWindow, uvc_any2rgbx, 4);
						addCaptureFrame(frame);
					} else {
						recycle_frame(frame);
					}
				}
			}
		} else {
			// yuvyv mode
			for ( ; LIKELY(isRunning()) ; ) {
				frame = waitPreviewFrame();
				if (LIKELY(frame)) {
					frame = draw_preview_one(frame, &mPreviewWindow, uvc_any2rgbx, 4);
					addCaptureFrame(frame);
				}
			}
		}
		pthread_cond_signal(&capture_sync);
#if LOCAL_DEBUG
		LOGI("preview_thread_func:wait for all callbacks complete");
#endif
		uvc_stop_streaming(mDeviceHandle);
#if LOCAL_DEBUG
		LOGI("Streaming finished");
#endif
	} else {
		uvc_perror(result, "failed start_streaming");
	}

	EXIT();
}

当使用 MJPEG 编码方式，会进入此分支

for ( ; LIKELY(isRunning()) ; ) {
				frame_mjpeg = waitPreviewFrame();
				if (LIKELY(frame_mjpeg)) {
					frame = get_frame(frame_mjpeg->width * frame_mjpeg->height * 2);
					result = uvc_mjpeg2yuyv(frame_mjpeg, frame);   // MJPEG => yuyv
					recycle_frame(frame_mjpeg);
					if (LIKELY(!result)) {
						frame = draw_preview_one(frame, &mPreviewWindow, uvc_any2rgbx, 4);
						addCaptureFrame(frame);
					} else {
						recycle_frame(frame);
					}
				}
			}

源作者的处理方式为：MJPEG ----> YUYV ----> RGBX

此过程进行了两次数据转码过程，

我的优化思路为：MJPEG ----> RGBX

 

for ( ; LIKELY(isRunning()) ; ) {
				frame_mjpeg = waitPreviewFrame();
				if (LIKELY(frame_mjpeg)) {
					frame = get_frame(frame_mjpeg->width * frame_mjpeg->height * 2);
					result = uvc_mjpeg2rgbx(frame_mjpeg, frame);   // MJPEG => RGBX
					recycle_frame(frame_mjpeg);
					if (LIKELY(!result)) {
						frame = draw_preview_one(frame, &mPreviewWindow, nullptr, 4);
						addCaptureFrame(frame);
					} else {
						recycle_frame(frame);
					}
				}
			}

在修改完此代码片段后，我发现预览过程，会存在 画面闪屏/程序空指针崩溃 的异常报错，

经过代码定位后，发现是frame-mjpeg.c 文件内部 uvc_mjpeg2rgbx 函数存在bug，故进行了以下修改【此为修改后的代码，可对比发现修改点】

/** @brief Convert an MJPEG frame to RGBX
 * @ingroup frame
 *
 * @param in MJPEG frame
 * @param out RGBX frame
 */
uvc_error_t uvc_mjpeg2rgbx(uvc_frame_t *in, uvc_frame_t *out) {
	struct jpeg_decompress_struct dinfo;
	struct error_mgr jerr;
	size_t lines_read;

	int num_scanlines, i;
	lines_read = 0;
	unsigned char *buffer[MAX_READLINE];

	out->actual_bytes = 0;	// XXX
	if (UNLIKELY(in->frame_format != UVC_FRAME_FORMAT_MJPEG))
		return UVC_ERROR_INVALID_PARAM;

	if (uvc_ensure_frame_size(out, in->width * in->height * 4) < 0)
		return UVC_ERROR_NO_MEM;

	out->width = in->width;
	out->height = in->height;
	out->frame_format = UVC_FRAME_FORMAT_RGBX;	// XXX
	out->step = in->width * 4;
	out->sequence = in->sequence;
	out->capture_time = in->capture_time;
	out->source = in->source;

	dinfo.err = jpeg_std_error(&jerr.super);
	jerr.super.error_exit = _error_exit;

    // local copy
    uint8_t *data = out->data;
    const int out_step = out->step;

	if (setjmp(jerr.jmp)) {
		goto fail;
	}

	jpeg_create_decompress(&dinfo);
	jpeg_mem_src(&dinfo, in->data, in->actual_bytes/*in->data_bytes*/);	// XXX
	jpeg_read_header(&dinfo, TRUE);

	if (dinfo.dc_huff_tbl_ptrs[0] == NULL) {
		/* This frame is missing the Huffman tables: fill in the standard ones */
		insert_huff_tables(&dinfo);
	}

	dinfo.out_color_space = JCS_EXT_RGBA;
	dinfo.dct_method = JDCT_IFAST;

	jpeg_start_decompress(&dinfo);

	if (LIKELY(dinfo.output_height == out->height)) {
		for (; dinfo.output_scanline < dinfo.output_height ;) {
			buffer[0] = data + (lines_read) * out_step;
			for (i = 1; i < MAX_READLINE; i++)
				buffer[i] = buffer[i-1] + out_step;
			num_scanlines = jpeg_read_scanlines(&dinfo, buffer, MAX_READLINE);
			lines_read += num_scanlines;
		}
		out->actual_bytes = in->width * in->height * 4;	// XXX
	}
	jpeg_finish_decompress(&dinfo);
	jpeg_destroy_decompress(&dinfo);
	return lines_read == out->height ? UVC_SUCCESS : UVC_ERROR_OTHER;	// XXX

fail:
	jpeg_destroy_decompress(&dinfo);
	return UVC_ERROR_OTHER+1;
}

修改测试后，发现解决了预览闪屏的问题，也提高了帧率；经过稳定性测试，能够稳定预览；

故障解决，

对于作者为什么要进行多次转码的原因我也无法探究，猜测是出于图像质量方面的考虑，这方面我也没有进行深入研究；