ffmpeg 从mp4上提取H264的nalu

http://blog.youkuaiyun.com/gavinr/article/details/7183499

1.获取数据
ffmpeg读取mp4中的H264数据，并不能直接得到NALU，文件中也没有储存0x00000001的分隔符。下面这张图为packet.data中的数据


从图中可以发现，packet中的数据起始处没有分隔符(0x00000001), 也不是0x65、0x67、0x68、0x41等字节，所以可以肯定这不是标准的nalu。

其实，前4个字0x000032ce表示的是nalu的长度，从第5个字节开始才是nalu的数据。所以直接将前4个字节替换为0x00000001即可得到标准的nalu数据。


2.获取pps及sps

pps及sps不能从packet获得，而是保存在AVCodecContext的extradata数据域中。如下：




如何从extradata中解析出sps及pps呢？ffmpeg中提供了一个流过滤器"h264_mp4toannexb"完成这项工作，关键代码如下

[cpp] view plain copy print ?

1. //h264_mp4toannexb_bsf.c 
2. static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc, 
3.                                    AVCodecContext *avctx, const char *args, 
4.                                    uint8_t  **poutbuf, int *poutbuf_size, 
5.                                    const uint8_t *buf,int      buf_size, 
6.                                    int keyframe) { 
7.     H264BSFContext *ctx = bsfc->priv_data; 
8.     uint8_t unit_type; 
9.     int32_t nal_size; 
10.     uint32_t cumul_size = 0; 
11.     const uint8_t *buf_end = buf + buf_size; 
12.  
13.  
14.     /* nothing to filter */ 
15.     if (!avctx->extradata || avctx->extradata_size < 6) { 
16.         *poutbuf = (uint8_t*) buf; 
17.         *poutbuf_size = buf_size; 
18.         return 0; 
19.     } 
20.      
21.     // 
22.     //从extradata中分析出SPS、PPS 
23.     // 
24.     /* retrieve sps and pps NAL units from extradata */ 
25.     if (!ctx->extradata_parsed) { 
26.         uint16_t unit_size; 
27.         uint64_t total_size = 0; 
28.         uint8_t *out = NULL, unit_nb, sps_done = 0, sps_seen = 0, pps_seen = 0; 
29.         const uint8_t *extradata = avctx->extradata+4; //跳过前4个字节 
30.         static const uint8_t nalu_header[4] = {0, 0, 0, 1}; 
31.  
32.  
33.         /* retrieve length coded size */ 
34.         ctx->length_size = (*extradata++ & 0x3) + 1;    //用于指示表示编码数据长度所需字节数 
35.         if (ctx->length_size == 3) 
36.             return AVERROR(EINVAL); 
37.  
38.  
39.         /* retrieve sps and pps unit(s) */ 
40.         unit_nb = *extradata++ & 0x1f; /* number of sps unit(s) */ 
41.         if (!unit_nb) { 
42.             goto pps; 
43.         } else { 
44.             sps_seen = 1; 
45.         } 
46.  
47.  
48.         while (unit_nb--) { 
49.             void *tmp; 
50.  
51.  
52.             unit_size = AV_RB16(extradata); 
53.             total_size += unit_size+4; 
54.             if (total_size > INT_MAX - FF_INPUT_BUFFER_PADDING_SIZE || 
55.                 extradata+2+unit_size > avctx->extradata+avctx->extradata_size) { 
56.                 av_free(out); 
57.                 return AVERROR(EINVAL); 
58.             } 
59.             tmp = av_realloc(out, total_size + FF_INPUT_BUFFER_PADDING_SIZE); 
60.             if (!tmp) { 
61.                 av_free(out); 
62.                 return AVERROR(ENOMEM); 
63.             } 
64.             out = tmp; 
65.             memcpy(out+total_size-unit_size-4, nalu_header, 4); 
66.             memcpy(out+total_size-unit_size,   extradata+2, unit_size); 
67.             extradata += 2+unit_size; 
68. pps: 
69.             if (!unit_nb && !sps_done++) { 
70.                 unit_nb = *extradata++; /* number of pps unit(s) */ 
71.                 if (unit_nb) 
72.                     pps_seen = 1; 
73.             } 
74.         } 
75.  
76.  
77.         if(out) 
78.             memset(out + total_size, 0, FF_INPUT_BUFFER_PADDING_SIZE); 
79.  
80.  
81.         if (!sps_seen) 
82.             av_log(avctx, AV_LOG_WARNING, "Warning: SPS NALU missing or invalid. The resulting stream may not play.\n"); 
83.         if (!pps_seen) 
84.             av_log(avctx, AV_LOG_WARNING, "Warning: PPS NALU missing or invalid. The resulting stream may not play.\n"); 
85.  
86.  
87.         av_free(avctx->extradata); 
88.         avctx->extradata      = out; 
89.         avctx->extradata_size = total_size; 
90.         ctx->first_idr        = 1; 
91.         ctx->extradata_parsed = 1; 
92.     } 
93.  
94.  
95.     *poutbuf_size = 0; 
96.     *poutbuf = NULL; 
97.     do { 
98.         if (buf + ctx->length_size > buf_end) 
99.             goto fail;  //buf为NULL时，以下代码将不再执行 
100.  
101.  
102.         // 
103.         //用于保存数据长度的字节数，是在分析原extradata计算出来的 
104.         // 
105.         if (ctx->length_size == 1) { 
106.             nal_size = buf[0]; 
107.         } else if (ctx->length_size == 2) { 
108.             nal_size = AV_RB16(buf); 
109.         } else 
110.             nal_size = AV_RB32(buf); 
111.  
112.  
113.         buf += ctx->length_size; 
114.         unit_type = *buf & 0x1f; 
115.  
116.  
117.         if (buf + nal_size > buf_end || nal_size < 0) 
118.             goto fail; 
119.  
120.  
121.         /* prepend only to the first type 5 NAL unit of an IDR picture */ 
122.         if (ctx->first_idr && unit_type == 5) { 
123.             // 
124.             //copy IDR 帧时，需要将sps及pps一同拷贝 
125.             // 
126.             if (alloc_and_copy(poutbuf, poutbuf_size, 
127.                                avctx->extradata, avctx->extradata_size, 
128.                                buf, nal_size) < 0) 
129.                 goto fail; 
130.             ctx->first_idr = 0; 
131.         } else { 
132.             // 
133.             //非IDR帧，没有sps及pps 
134.             if (alloc_and_copy(poutbuf, poutbuf_size, 
135.                                NULL, 0, 
136.                                buf, nal_size) < 0) 
137.                 goto fail; 
138.             if (!ctx->first_idr && unit_type == 1) 
139.                 ctx->first_idr = 1; 
140.         } 
141.  
142.  
143.         buf += nal_size; 
144.         cumul_size += nal_size + ctx->length_size; 
145.     } while (cumul_size < buf_size); 
146.  
147.  
148.     return 1; 
149.  
150.  
151. fail: 
152.     av_freep(poutbuf); 
153.     *poutbuf_size = 0; 
154.     return AVERROR(EINVAL); 
155. } 

//h264_mp4toannexb_bsf.c static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc, AVCodecContext *avctx, const char *args, uint8_t **poutbuf, int *poutbuf_size, const uint8_t *buf, int buf_size, int keyframe) { H264BSFContext *ctx = bsfc->priv_data; uint8_t unit_type; int32_t nal_size; uint32_t cumul_size = 0; const uint8_t *buf_end = buf + buf_size; /* nothing to filter */ if (!avctx->extradata || avctx->extradata_size < 6) { *poutbuf = (uint8_t*) buf; *poutbuf_size = buf_size; return 0; } // //从extradata中分析出SPS、PPS // /* retrieve sps and pps NAL units from extradata */ if (!ctx->extradata_parsed) { uint16_t unit_size; uint64_t total_size = 0; uint8_t *out = NULL, unit_nb, sps_done = 0, sps_seen = 0, pps_seen = 0; const uint8_t *extradata = avctx->extradata+4; //跳过前4个字节 static const uint8_t nalu_header[4] = {0, 0, 0, 1}; /* retrieve length coded size */ ctx->length_size = (*extradata++ & 0x3) + 1; //用于指示表示编码数据长度所需字节数 if (ctx->length_size == 3) return AVERROR(EINVAL); /* retrieve sps and pps unit(s) */ unit_nb = *extradata++ & 0x1f; /* number of sps unit(s) */ if (!unit_nb) { goto pps; } else { sps_seen = 1; } while (unit_nb--) { void *tmp; unit_size = AV_RB16(extradata); total_size += unit_size+4; if (total_size > INT_MAX - FF_INPUT_BUFFER_PADDING_SIZE || extradata+2+unit_size > avctx->extradata+avctx->extradata_size) { av_free(out); return AVERROR(EINVAL); } tmp = av_realloc(out, total_size + FF_INPUT_BUFFER_PADDING_SIZE); if (!tmp) { av_free(out); return AVERROR(ENOMEM); } out = tmp; memcpy(out+total_size-unit_size-4, nalu_header, 4); memcpy(out+total_size-unit_size, extradata+2, unit_size); extradata += 2+unit_size; pps: if (!unit_nb && !sps_done++) { unit_nb = *extradata++; /* number of pps unit(s) */ if (unit_nb) pps_seen = 1; } } if(out) memset(out + total_size, 0, FF_INPUT_BUFFER_PADDING_SIZE); if (!sps_seen) av_log(avctx, AV_LOG_WARNING, "Warning: SPS NALU missing or invalid. The resulting stream may not play.\n"); if (!pps_seen) av_log(avctx, AV_LOG_WARNING, "Warning: PPS NALU missing or invalid. The resulting stream may not play.\n"); av_free(avctx->extradata); avctx->extradata = out; avctx->extradata_size = total_size; ctx->first_idr = 1; ctx->extradata_parsed = 1; } *poutbuf_size = 0; *poutbuf = NULL; do { if (buf + ctx->length_size > buf_end) goto fail; //buf为NULL时，以下代码将不再执行 // //用于保存数据长度的字节数，是在分析原extradata计算出来的 // if (ctx->length_size == 1) { nal_size = buf[0]; } else if (ctx->length_size == 2) { nal_size = AV_RB16(buf); } else nal_size = AV_RB32(buf); buf += ctx->length_size; unit_type = *buf & 0x1f; if (buf + nal_size > buf_end || nal_size < 0) goto fail; /* prepend only to the first type 5 NAL unit of an IDR picture */ if (ctx->first_idr && unit_type == 5) { // //copy IDR 帧时，需要将sps及pps一同拷贝 // if (alloc_and_copy(poutbuf, poutbuf_size, avctx->extradata, avctx->extradata_size, buf, nal_size) < 0) goto fail; ctx->first_idr = 0; } else { // //非IDR帧，没有sps及pps if (alloc_and_copy(poutbuf, poutbuf_size, NULL, 0, buf, nal_size) < 0) goto fail; if (!ctx->first_idr && unit_type == 1) ctx->first_idr = 1; } buf += nal_size; cumul_size += nal_size + ctx->length_size; } while (cumul_size < buf_size); return 1; fail: av_freep(poutbuf); *poutbuf_size = 0; return AVERROR(EINVAL); }
一般情况下，extradata中包含一个sps、一个pps 的nalu, 从上面的代码中容易看出extradata的数据格式。分析后的sps及pps依然储存在extradata域中，并添加了起始符。从代码中还可以看出，上面的函数会将sps、pps及packet中的数据，都copy到poutbuf指示的内存中，如果不需要copy到指定内存，直接给buf参数传入空值即可。




3.使用ffmpeg的流过滤器获取sps及pps
流过滤器"h264_mp4toannexb", 在av_register_all()函数中会被注册。用法示例如下：

[cpp] view plain copy print ?

1. int ParseH264ExtraDataInMp4(int stream_id) 
2. { 
3.     uint8_t *dummy = NULL; 
4.     int dummy_size; 
5.     AVBitStreamFilterContext* bsfc =  av_bitstream_filter_init("h264_mp4toannexb"); 
6.  
7.  
8.     if(bsfc == NULL) 
9.     { 
10.         return -1; 
11.     } 
12.  
13.  
14.     av_bitstream_filter_filter( 
15.             bsfc, format_ctx_->streams[stream_id]->codec, NULL, &dummy, &dummy_size, NULL, 0, 0); 

int ParseH264ExtraDataInMp4(int stream_id) { uint8_t *dummy = NULL; int dummy_size; AVBitStreamFilterContext* bsfc = av_bitstream_filter_init("h264_mp4toannexb"); if(bsfc == NULL) { return -1; } av_bitstream_filter_filter( bsfc, format_ctx_->streams[stream_id]->codec, NULL, &dummy, &dummy_size, NULL, 0, 0);

[cpp] view plain copy print ?

1.     av_bitstream_filter_close(bsfc); 
2.  
3.  
4.     return 0; 
5. }