GStreamer第一阶段的简单总结

前言

因为涉及很多细节的GStreamer官方论坛有详细解链接: GStreamer官网，这里不做说明，以下只是涉及到个人的理解和认知，方便后续的查阅。

---

个人的总结

1)了解pipeline的使用，简单说就是通道的概念, 因为涉及到数据的复用和拓展， 里面有一个filter(tee插件), 通过tee和后续拓展
的绑定关系复用数据，很像许多Soc编解码芯片中的vpss通道的概念。 数据复用为视频编码通道， 数据jpeg通道以及对数据处理的通道。

2)重点是去了解Pad Templates 中的Availability,这个关系到link的方式，另外关注Element Properties的属性设置。

3)工作使用最多的不是作为播放器，而是将采集数据获取后做后续的图像处理，因此以下是从v4l2src插件中获取到数据的方式：
4)GStreamer复用的拓展性导致其复杂性, 还涉及到glib的学习，因此学习成本还是比较高的， 但现主流的流媒体芯片或边缘化芯片大部分都用到GStreamer，
比如NVIDIA和瑞芯微,特别是瑞芯微直接将GStreamer作为框架，增加自己的硬编解码。
5) GStreamer的许多控件非常适合拓展， 比如推流rtp/rtsp/rtmp/webrtc开发非常快，但走API这套的调试以及对API的理解和学习周期比较长。
这里只花了3天时间学习， 做一个记录。

v4l2src插件的简单使用

/**
 *			
	@brief  实现从v4l2src中拿到数据,方便后续数据的操作，比如用作重新硬编解码用

	appsink关键属性： emit-signals默认是关闭
		
 emit-signals        : Emit new-preroll and new-sample signals
                        flags: readable, writable
                        Boolean. Default: false

   
 Element Signals:
   "eos" :	void user_function (GstElement* object,
								gpointer user_data);
   "new-preroll" :	GstFlowReturn user_function (GstElement* object,
												 gpointer user_data);
   "new-sample" :  GstFlowReturn user_function (GstElement* object,
												gpointer user_data);
 
 Element Actions:
   "pull-preroll" :  GstSample * user_function (GstElement* object);
   "pull-sample" :	GstSample * user_function (GstElement* object);
   "try-pull-preroll" :  GstSample * user_function (GstElement* object,
													guint64 arg0);
   "try-pull-sample" :	GstSample * user_function (GstElement* object,
												   guint64 arg0);
 */
#include <unistd.h>
#include <gst/gst.h>
#include <pthread.h>

#include <time.h>
#include <sys/time.h>

#define		TEST_MODULE

//#define VIDEO_X_RAW

typedef struct ExtraCtrl_
{
	int s32brightness;
	int s32Contrast;
	int s32Saturation;
	int s32Hue;
	int s32Sharpness;
	
}Extra_Ctrl_S;

/*
*		@brief 设置ISP图像质量，主要是图像增强类
		@prams[in] source  - SRC plugin			extraCtrl - 外部的图像喜好
		@return  null

		@test:
			1. 通过inspect查看插件支持的图像属性有哪些
*/
void setCameraQuality(GstElement *source, const Extra_Ctrl_S *extraCtrl)
{
		
		g_object_set(source, "do-timestamp", TRUE, NULL);
#if 0
	  GstStructure* extraCtrls = gst_structure_new("logitech_controls",
	  											   "brightness", extraCtrl->s32brightness,
	  											   "contrast", G_TYPE_INT, extraCtrl->s32Contrast,
	  											   "saturation", G_TYPE_INT, extraCtrl->s32Saturation,
	  											   "hue", G_TYPE_INT, extraCtrl->s32Hue,
	  											   "sharpness", G_TYPE_INT, extraCtrl->s32Sharpness,
												   
												 //  "focus_auto", G_TYPE_BOOLEAN, FALSE,
												 //  "white_balance_temperature_auto", G_TYPE_BOOLEAN, FALSE,
												 //  "white_balance_temperature", G_TYPE_INT, 3500,
												   NULL);
#else 

	/*gst-1.0 无sharpness
		默认值 brightness 0， hue 0 contrast 36 saturation:54

	*/
	int brightness = 1, hue = 1, contrast = 1, saturation = 1;
	g_object_get(source , "brightness", &brightness, "hue", &hue, "contrast", &contrast, "saturation", &saturation, NULL);

	g_print("get: brightness:%d, hue:%d, contrast:%d, saturation:%d\n", brightness, hue, contrast, saturation);

	
#if 1
	int setBrightness = 0;
	GstStructure* extraCtrls = gst_structure_new("logitech_controls",
												"brightness", G_TYPE_INT, brightness ,
												"hue", G_TYPE_INT, hue,
                                                 "contrast", G_TYPE_INT, contrast,
                                                 "saturation", G_TYPE_INT, saturation,//160,
                                                 NULL);
  g_object_set(source, "extra-controls" , extraCtrls, NULL);
 #endif      
#endif 
	
	g_object_get(source , "brightness", &brightness, "hue", &hue, "contrast", &contrast, "saturation", &saturation, NULL);

	g_print("get: brightness:%d, hue:%d, contrast:%d, saturation:%d\n", brightness, hue, contrast, saturation);
}

TEST_MODULE void *ispCtlProc(void *arg)
{
	while(1)
	{
		GstElement * element = (GstElement*)arg;
		if(element)
		{
			static int rec = 0;
			g_print("rec:%d\n", rec++);
			Extra_Ctrl_S s;memset(&s, 0, sizeof(s));
			static int count = 0;
			s.s32brightness = count++ % 255;
			s.s32Contrast = 128;
			s.s32Saturation = 128;
			s.s32Hue = 128;
			s.s32Sharpness = 128;
			setCameraQuality(element, &s);
		}

		usleep(1000*100);
	}

	return NULL;
}

/*
		@brief  原始数据回调
*/
GstFlowReturn cb_appsink_new_sample(GstElement* appsink, gpointer user_data)

{
     // LOG_INFO_MSG ("cb_appsink_new_sample called, user data: %p", user_data);
#if 0
     SinkPipeline* sp = reinterpret_cast<SinkPipeline*> (user_data);
#endif
     GstSample* sample = NULL;
     GstBuffer* buffer = NULL;
     GstMapInfo map;
     const GstStructure* info = NULL;
     GstCaps* caps = NULL;
     GstFlowReturn ret = GST_FLOW_OK;
     int sample_width = 0;
     int sample_height = 0;
     // equals to gst_app_sink_pull_sample (GST_APP_SINK_CAST (appsink), sample);
     g_signal_emit_by_name (appsink, "pull-sample", &sample, &ret);
     if (ret != GST_FLOW_OK) {
         g_print ("can't pull GstSample.\n");
         return ret;
     }
     if (sample) 
     {
         buffer = gst_sample_get_buffer (sample);
         if ( buffer == NULL ) {
             g_print ("get buffer is null\n");
             goto exit;
         }
        gboolean mapret= gst_buffer_map (buffer, &map, GST_MAP_READ);
		if(mapret == FALSE)
		{
			goto exit;
		}
         /**/
         caps = gst_sample_get_caps (sample);
         if ( caps == NULL ) {
             g_print ("get caps is null\n");
             goto exit;
         }
         info = gst_caps_get_structure (caps, 0);
         if ( info == NULL ) {
             g_print ("get info is null\n");
             goto exit;
         }
         // -------- Read frame and convert to opencv format --------
         // convert gstreamer data to OpenCV Mat, you could actually
         // resolve height / width from caps...
         gst_structure_get_int (info, "width", &sample_width);
         gst_structure_get_int (info, "height", &sample_height);
        
         g_print("format:%s, width:%d, height:%d\n",  gst_structure_get_string(info, "format"), sample_width, sample_height);

         if(NULL == map.data)
         {
				g_print("appsink buffer data empty\n");
				goto exit;
         }	
		static struct timeval s, e;
         static int flag = 1;
         if(flag)
         {
         	
         	 gettimeofday(&s, 0); e = s;
			flag = 0;

         }
         gettimeofday(&e, 0);
         static int frameNo = 0;
         frameNo++;
         unsigned int timespan = (e.tv_sec - s.tv_sec )*1000*1000 + e.tv_usec - s.tv_usec;
         if(timespan >= 1000000)
         {
         
		 	g_print("frame siz:%d, max_siz:%d, %02d FPS\n", map.size, map.maxsize, frameNo);
			frameNo = 0;
			gettimeofday(&s, 0);
		 }
         //FILE *fp = 
#if 0
         // customized user action
         {
             // init a cv::Mat with gst buffer address: deep copy
             // sometime you may got a empty buffer
             if (map.data == NULL) {
                 LOG_ERROR_MSG("appsink buffer data empty\n");
                 return GST_FLOW_OK;
             }
             cv::Mat img (sample_height, sample_width, CV_8UC3,
                             (unsigned char*)map.data, cv::Mat::AUTO_STEP);
             img = img.clone();
             // redirection outside operation: for decoupling use
             if (sp->m_putDataFunc) {
                 sp->m_putDataFunc(std::make_shared<cv::Mat> (img),
                     sp->m_putDataArgs);
             } else {
                 goto exit;
             }
         }
 #endif
 exit:
     if (buffer) {
         gst_buffer_unmap (buffer, &map);
     }
     if (sample) {
         gst_sample_unref (sample);
     }
     return GST_FLOW_OK;
 	}

}
/*
	@brief 创造线程设置图像数据， TEST 模块
*/
TEST_MODULE int ispInit(GstElement *source)
{
	int ret = 0;
	pthread_t ispControlThread;
  	pthread_create(&ispControlThread, 0, ispCtlProc, (void *)source);
}


int main(int argc, char *argv[]) 
{
	
  GstElement *pipeline, *source;
  GstElement  *converter,*appsink;											
  
  GstBus *bus;
  GstMessage *msg;

 
 
  /* Initialize GStreamer */
  gst_init (&argc, &argv);

  /* Create the elements */
  /*src(always)*/
  source = gst_element_factory_make ("v4l2src", "v4l2src");
  
  converter= 	gst_element_factory_make("videoconvert","converter01");
  //sink
  appsink = gst_element_factory_make("appsink", "appsink");

  /* Create the empty pipeline */
  pipeline = gst_pipeline_new ("test-pipeline");

  if ( !pipeline || !source || !appsink ) {
    g_printerr ("Not all elements could be created.\n");
    return -1;
  }

//	set appsink 
g_object_set(appsink, "emit-signals", TRUE, NULL);

  g_signal_connect (appsink, "new-sample", G_CALLBACK (cb_appsink_new_sample), NULL);

 

  /* Link all elements that can be automatically linked because they have "Always" pads */
  gst_bin_add_many (GST_BIN (pipeline), source, converter, appsink, NULL);

  
//  TEST_MODULE ispInit(source); 

/*
	@note:
		1. 需要在gst_bin_add_many之后调用
		先需要知道摄像头支持的Pixel Format,通过命令可查，
		v4l2-ctl --list-formats-ext --device /dev/video0可以查看支持的格式只有YUYV和MJPG,另外可以查看到具体的分辨率和帧率
*/
#if defined(VIDEO_X_RAW)
 GstCaps *caps = gst_caps_new_simple ("video/x-raw", 
       //  "format", G_TYPE_STRING, "YV12", //<<< IF THIS IS SET TO ARGB (THE FORMAT I WANT IT FAILS ON LINKING) 
      // 	"format", G_TYPE_STRING, "NV21",
         "framerate", GST_TYPE_FRACTION, 25, 1, 
        // "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, 
         "width", G_TYPE_INT, 320, 
         "height", G_TYPE_INT, 240, 
         NULL); 
#else
		 GstCaps *caps = gst_caps_new_simple ("video/mpeg", 
      	 "mpegversion", G_TYPE_INT, 2,
      	 "systemstream", G_TYPE_BOOLEAN, TRUE,
         NULL); 

#endif 
	
	gst_element_link_filtered(source, converter, caps);
	gst_caps_unref(caps);

	

	/*这里做了转换，需要逐一连接*/
	gst_element_link(source, converter);
	gst_element_link(converter, appsink);

	

  /* Start playing the pipeline */
  gst_element_set_state (pipeline, GST_STATE_PLAYING);

  /* Wait until error or EOS */
  bus = gst_element_get_bus (pipeline);
  msg = gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE, GST_MESSAGE_ERROR | GST_MESSAGE_EOS);

if(msg != NULL)
{
	GError *err;
	gchar *debug_info;

	switch (GST_MESSAGE_TYPE (msg)) {
	case GST_MESSAGE_ERROR:
	gst_message_parse_error (msg, &err, &debug_info);
	g_printerr ("Error received from element %s: %s\n",
	GST_OBJECT_NAME (msg->src), err->message);
	g_printerr ("Debugging information: %s\n",
	debug_info ? debug_info : "none");
	g_clear_error (&err);
	g_free (debug_info);
	break;
	
	case GST_MESSAGE_EOS:
	g_print ("End-Of-Stream reached.\n");
	break;
	
	default:
	/* We should not reach here because we only asked for ERRORs and EOS */
	g_printerr ("Unexpected message received.\n");
	break;
	}
	gst_message_unref (msg);
}else
{
	g_printerr("error.");
}
	

  /* Release the request pads from the Tee, and unref them */
 
  
  /* Free resources */
  if (msg != NULL)
    gst_message_unref (msg);
  gst_object_unref (bus);
  gst_element_set_state (pipeline, GST_STATE_NULL);

  gst_object_unref (pipeline);
  return 0;
}

编译:

cmake_minimum_required(VERSION 3.0)
project(test_project)

find_package(PkgConfig REQUIRED)
pkg_search_module(GST1 REQUIRED gstreamer-1.0)#查找到gstreamer-1.0 并用GST1代表，GST1自己定义

add_executable(a.out v4l2src_appsink.c)

target_include_directories(a.out PRIVATE ${GST1_INCLUDE_DIRS})
target_link_libraries(a.out ${GST1_LIBRARIES} -lpthread)

message("GST1_LIBRARIES
${GST1_LIBRARIES}
")

运行结果：

format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
frame siz:4147200, max_siz:4147200, 07 FPS
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080
format:YUY2, width:1920, height:1080