SDL2绘制ffmpeg解析的mp4文件

Alex1_Code

已于 2023-10-05 16:01:30 修改

阅读量1.2k

点赞数 1

CC 4.0 BY-SA版权

分类专栏： C++ 项目文章标签： ffmpeg

于 2023-10-03 18:49:40 首次发布

本文链接：https://blog.youkuaiyun.com/Wwc_code/article/details/133524976

C++ 同时被 2 个专栏收录

36 篇文章

订阅专栏

项目

10 篇文章

订阅专栏

文章目录

1.FFMPEG利用命令行将mp4转yuv420
2.ffmpeg将mp4解析为yuv数据
- 2.1 核心api:
3.SDL2进行yuv绘制到屏幕
- 3.1 核心api
4.完整代码
5.效果展示
6.SDL2事件响应补充
6.1 处理方式-01
6.2 处理方式-02

本项目采用生产者消费者模型，生产者线程：使用ffmpeg将mp4格式数据解析为yuv的帧，消费者线程：利用sdl2将解析的yuv的帧进行消费，绘制到屏幕上。未完成的部分：1.解析音频数据，并与视频数据同步。2.增加界面：暂停，播放按钮，支持视频前进和后退。
学习音视频的参考资料与项目：
playdemo_github
雷神的csdn博客

1.FFMPEG利用命令行将mp4转yuv420

ffmpeg -i input.mp4 -c:v rawvideo -pix_fmt yuv420p output.yuv

在这里插入图片描述

2.ffmpeg将mp4解析为yuv数据

2.1 核心api:

av_read_frame:读取一帧数据
avcodec_send_packet:将数据包发送给解码器
avcodec_receive_frame:将数据包从解码器中取
sws_scale:格式转换，将解码后的帧数据转为yuv数据，存储在data[0],data[1],data[2]中

void readFrame()
{
	AVPacket* avPacket = av_packet_alloc();
	AVFrame* frame = av_frame_alloc();
	
	FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data","wb+") ;
	while (av_read_frame(formatContext, avPacket) >= 0 && fp)
	{
		if (avPacket->stream_index == videoStreamIndex)
		{
			if (avcodec_send_packet(codecContext, avPacket) < 0) {
				std::cerr << "发送数据包到解码器失败" << std::endl;
				break;
			}
			/*解码*/
			int ret = avcodec_receive_frame(codecContext, frame);
			printf("ret:%d\n", ret);
			if (ret >= 0)
			{
				ret = sws_scale(swsContext, frame->data, frame->linesize, 0, codecContext->height, yuvFrame->data, yuvFrame->linesize);
				printf("sws_scale ret=%d\n", ret);
				std::lock_guard<std::mutex>lck(mtx);
				isFinished = false;
				memcpy(yuvBuf, yuvFrame->data[0], yuvFrame->width * yuvFrame->height);
				memcpy(yuvBuf + yuvFrame->width * yuvFrame->height, yuvFrame->data[1], yuvFrame->width * yuvFrame->height / 4);
				memcpy(yuvBuf + yuvFrame->width * yuvFrame->height*5/4, yuvFrame->data[2], yuvFrame->width * yuvFrame->height / 4);
				isFinished = true;
				condvar.notify_one();
				//保存y分量
				//fwrite(yuvFrame->data[0], 1, yuvFrame->width * yuvFrame->height, fp);
				//保存uv分量
				//fwrite(yuvFrame->data[1], 1, yuvFrame->width * yuvFrame->height/4, fp);
				//fwrite(yuvFrame->data[2], 1, yuvFrame->width * yuvFrame->height / 4, fp);
			}
			
		}
	}
	fclose(fp);
	av_frame_unref(yuvFrame);
	av_packet_free(&avPacket);
	av_frame_unref(frame);
}

3.SDL2进行yuv绘制到屏幕

3.1 核心api

SDL_Init
SDL_CreateWindow
SDL_CreateRenderer
SDL_CreateTexture
SDL_UpdateTexture
SDL_RenderCopy
SDL_RenderPresent
SDL_Delay:控制帧率

int sdl_display()
{
	if (SDL_Init(SDL_INIT_VIDEO)) {
		printf("sdl init failed\n");
		return -1;
	}
	SDL_Window* window = SDL_CreateWindow("sdl_demo", 200, 200, codecContext->width, codecContext->height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);
	if (!window) {
		SDL_Quit();
		return -1;
	}
	SDL_Renderer* renderer = SDL_CreateRenderer(window, -1, 0);
	if (!renderer)
	{
		SDL_DestroyWindow(window);
		SDL_Quit();
		return -1;
	}
	SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
	SDL_RenderClear(renderer);
	Uint32 pixformat = SDL_PIXELFORMAT_IYUV;
	SDL_Texture* sdlTexture = SDL_CreateTexture(renderer, pixformat, SDL_TEXTUREACCESS_STREAMING, codecContext->width, codecContext->height);
	//FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data", "rb+");
	
	
	while (1) {
		//int ret = fread(yuvBuf, 1, yuvlen, fp);
		//if (ret <= 0) {
		//	break;
		//}
		std::unique_lock<std::mutex>lck(mtx);
		if (condvar.wait_for(lck, std::chrono::seconds(1), [] {
			return isFinished;}))
		{
			isFinished = false;
			SDL_UpdateTexture(sdlTexture, NULL, yuvBuf, codecContext->width);
			SDL_RenderCopy(renderer, sdlTexture, NULL, NULL);
			SDL_RenderPresent(renderer);
			//控制帧率25fps
			SDL_Delay(40);
		}
		else {
			printf("sdl thread exit!\n");
			break;
		}
	}
	SDL_Quit();
	return 0;
}

4.完整代码

-使用两个线程，生产者消费者模型

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include <thread>
#include <fstream>

#include <mutex>
#include <condition_variable>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/avutil.h>
#include <libswscale/swscale.h>

#include <SDL.h>
}
#undef main
#pragma warning(disable:4996)
AVFormatContext* formatContext = nullptr;
AVCodecContext* codecContext = nullptr;
SwsContext* swsContext = nullptr;
int videoStreamIndex = -1;
AVFrame* yuvFrame;
unsigned char* yuvBuf;
bool isReady = false;
bool isFinished = false;

std::mutex mtx;
std::condition_variable condvar;
void readFrame()
{
	AVPacket* avPacket = av_packet_alloc();
	AVFrame* frame = av_frame_alloc();
	
	FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data","wb+") ;
	while (av_read_frame(formatContext, avPacket) >= 0 && fp)
	{
		if (avPacket->stream_index == videoStreamIndex)
		{
			if (avcodec_send_packet(codecContext, avPacket) < 0) {
				std::cerr << "发送数据包到解码器失败" << std::endl;
				break;
			}
			/*解码*/
			int ret = avcodec_receive_frame(codecContext, frame);
			printf("ret:%d\n", ret);
			if (ret >= 0)
			{
				ret = sws_scale(swsContext, frame->data, frame->linesize, 0, codecContext->height, yuvFrame->data, yuvFrame->linesize);
				printf("sws_scale ret=%d\n", ret);
				std::lock_guard<std::mutex>lck(mtx);
				isFinished = false;
				memcpy(yuvBuf, yuvFrame->data[0], yuvFrame->width * yuvFrame->height);
				memcpy(yuvBuf + yuvFrame->width * yuvFrame->height, yuvFrame->data[1], yuvFrame->width * yuvFrame->height / 4);
				memcpy(yuvBuf + yuvFrame->width * yuvFrame->height*5/4, yuvFrame->data[2], yuvFrame->width * yuvFrame->height / 4);
				isFinished = true;
				condvar.notify_one();
				//保存y分量
				//fwrite(yuvFrame->data[0], 1, yuvFrame->width * yuvFrame->height, fp);
				//保存uv分量
				//fwrite(yuvFrame->data[1], 1, yuvFrame->width * yuvFrame->height/4, fp);
				//fwrite(yuvFrame->data[2], 1, yuvFrame->width * yuvFrame->height / 4, fp);
			}
			
		}
	}
	fclose(fp);
	av_frame_unref(yuvFrame);
	av_packet_free(&avPacket);
	av_frame_unref(frame);
}

int sdl_display()
{
	if (SDL_Init(SDL_INIT_VIDEO)) {
		printf("sdl init failed\n");
		return -1;
	}
	SDL_Window* window = SDL_CreateWindow("sdl_demo", 200, 200, codecContext->width, codecContext->height, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);
	if (!window) {
		SDL_Quit();
		return -1;
	}
	SDL_Renderer* renderer = SDL_CreateRenderer(window, -1, 0);
	if (!renderer)
	{
		SDL_DestroyWindow(window);
		SDL_Quit();
		return -1;
	}
	SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
	SDL_RenderClear(renderer);
	Uint32 pixformat = SDL_PIXELFORMAT_IYUV;
	SDL_Texture* sdlTexture = SDL_CreateTexture(renderer, pixformat, SDL_TEXTUREACCESS_STREAMING, codecContext->width, codecContext->height);
	//FILE* fp = fopen("F:/VS_Project/ffmpeg_demo/yuv.data", "rb+");
	
	
	while (1) {
		//int ret = fread(yuvBuf, 1, yuvlen, fp);
		//if (ret <= 0) {
		//	break;
		//}
		std::unique_lock<std::mutex>lck(mtx);
		if (condvar.wait_for(lck, std::chrono::seconds(1), [] {
			return isFinished;}))
		{
			isFinished = false;
			SDL_UpdateTexture(sdlTexture, NULL, yuvBuf, codecContext->width);
			SDL_RenderCopy(renderer, sdlTexture, NULL, NULL);
			SDL_RenderPresent(renderer);
			//控制帧率25fps
			SDL_Delay(40);
		}
		else {
			printf("sdl thread exit!\n");
			break;
		}
	}
	SDL_Quit();
	return 0;
}

/*ffmpeg -i input.mp4 -c:v rawvideo -pix_fmt yuv420p output.yuv*/
int main(int argc, char* argv[])
{
	/*if (argc != 2) {
		std::cerr << "文件名未指定" << std::endl;
		return -1;
	}*/
	std::string filename = "F:/VS_Project/ffmpeg_demo/1.mkv";
	if (avformat_open_input(&formatContext, filename.c_str(), nullptr, nullptr) != 0)
	{
		std::cerr << "无法打开文件" << std::endl;
		return -1;
	}
	if (avformat_find_stream_info(formatContext, nullptr) < 0) {
		std::cerr << "无法找到视频流" << std::endl;
		return -1;
	}
	for (int i = 0; i < formatContext->nb_streams; i++)
	{
		enum AVMediaType type = AVMEDIA_TYPE_VIDEO;
		AVStream* st = formatContext->streams[i];
		AVCodecParameters* codecpar = st->codecpar;
		if (codecpar->codec_type == type)
		{
			
			videoStreamIndex = i;
			const AVCodec* codec = avcodec_find_decoder(codecpar->codec_id);
			codecContext = avcodec_alloc_context3(codec);
			avcodec_parameters_to_context(codecContext, codecpar);
			avcodec_open2(codecContext, codec, nullptr);
			swsContext = sws_getContext(codecContext->width, codecContext->height, codecContext->pix_fmt,
				codecContext->width, codecContext->height, AV_PIX_FMT_YUV420P,
				SWS_BILINEAR, nullptr, nullptr, nullptr);
			std::cout << "w:" << codecpar->width << std::endl;
			std::cout << "h:" << codecpar->height << std::endl;
		}
	}
	yuvFrame = av_frame_alloc();
	yuvFrame->width = codecContext->width;
	yuvFrame->height = codecContext->height;
	yuvFrame->format = AV_PIX_FMT_YUV420P;

	int yuvlen = codecContext->width * codecContext->height * 3 / 2;
	yuvBuf = new unsigned char[yuvlen];
	int ret = av_frame_get_buffer(yuvFrame, 0);
	if (ret < 0) {
		printf("分配缓冲区失败\n");
		return -1;
	}
	//sdl_init();
	std::thread th1(readFrame);
	std::thread th2(sdl_display);
	th1.join();
	th2.join();
	delete[]yuvBuf;
	return 0;
}

5.效果展示

在这里插入图片描述

6.SDL2事件响应补充

6.1 处理方式-01

起一个refresh_video线程，用于产生一个自定义更新video的事件：REFRESH_EVENT，并且每40ms更新一次。while循环中持续等待事件的到来：SDL_WaitEvent，当收到事件后更新一帧画面。同时当监测到窗口改变的事件，SDL_GetWindowSize对窗口进行调整。

int refresh_video(void *opaque){
	thread_exit=0;
	while (thread_exit==0) {
		SDL_Event event;
		event.type = REFRESH_EVENT;
		SDL_PushEvent(&event);
		SDL_Delay(40);
	}
	thread_exit=0;
	//Break
	SDL_Event event;
	event.type = BREAK_EVENT;
	SDL_PushEvent(&event);
	return 0;
}
SDL_Thread *refresh_thread = SDL_CreateThread(refresh_video,NULL,NULL);
	SDL_Event event;
	while(1){
		//Wait
		SDL_WaitEvent(&event);
		if(event.type==REFRESH_EVENT){
			if (fread(buffer, 1, pixel_w*pixel_h*bpp/8, fp) != pixel_w*pixel_h*bpp/8){
				// Loop
				fseek(fp, 0, SEEK_SET);
				fread(buffer, 1, pixel_w*pixel_h*bpp/8, fp);
			}

			SDL_UpdateTexture( sdlTexture, NULL, buffer, pixel_w);  

			//FIX: If window is resize
			sdlRect.x = 0;  
			sdlRect.y = 0;  
			sdlRect.w = screen_w;  
			sdlRect.h = screen_h;  
			
			SDL_RenderClear( sdlRenderer );   
			SDL_RenderCopy( sdlRenderer, sdlTexture, NULL, &sdlRect);  
			SDL_RenderPresent( sdlRenderer );  
			
		}else if(event.type==SDL_WINDOWEVENT){
			//If Resize
			SDL_GetWindowSize(screen,&screen_w,&screen_h);
		}else if(event.type==SDL_QUIT){
			thread_exit=1;
		}else if(event.type==BREAK_EVENT){
			break;
		}
	}
	SDL_Quit();

6.2 处理方式-02

起一个事件循环线程，SDL_PeepEvents 从事件队列中取出一个事件，然后更新事件队列，并进行绘制操作。

在这里插入图片描述

//播放控制循环
void VideoCtl::LoopThread(VideoState *cur_stream)
{
    SDL_Event event;
    double incr, pos, frac;

    m_bPlayLoop = true;

    while (m_bPlayLoop)
    {
        double x;
        refresh_loop_wait_event(cur_stream, &event);
        switch (event.type) {
        case SDL_KEYDOWN:
            switch (event.key.keysym.sym) {
            case SDLK_s: // S: Step to next frame
                step_to_next_frame(cur_stream);
                break;
            case SDLK_a:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
                break;
            case SDLK_v:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
                break;
            case SDLK_c:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_VIDEO);
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_AUDIO);
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
                break;
            case SDLK_t:
                stream_cycle_channel(cur_stream, AVMEDIA_TYPE_SUBTITLE);
                break;

            default:
                break;
            }
            break;
        case SDL_WINDOWEVENT:
            //窗口大小改变事件
            qDebug()<<"SDL_WINDOWEVENT  "<<endl;
            switch (event.window.event) {
            case SDL_WINDOWEVENT_RESIZED:
                screen_width = cur_stream->width = event.window.data1;
                screen_height = cur_stream->height = event.window.data2;
            case SDL_WINDOWEVENT_EXPOSED:
                cur_stream->force_refresh = 1;
            }
            break;
        case SDL_QUIT:
        case FF_QUIT_EVENT:
            do_exit(cur_stream);
            break;
        default:
            break;
        }
    }


    do_exit(m_CurStream);
    //m_CurStream = nullptr;

}