原因
问题：tpline/宇视接入正常，海康设备没有画面和音频。
排查方向：
1.抓包发现海康设备mark全是false，rtp完整帧判断没有做时间戳判断单独处理
2.扩展rtp时间戳判断完整帧机制后，dump解析后的文件只有I帧，音频存在问题。因此解析ps逻辑后的流有问题。
从wreshark抓包看mark最后一帧下一个包的起始数据，然后看dump抓包文件16进制。判断地址区间是否只有一种数据，如果存在音视频都有，需要特别处理。
根据00 00 01 e0/c0判断 0000A870之前的地址是否同时存在音视频数据。那就说明rtp的一帧判断机制是包含音视频数据，需要做特别处理。
1.锁定rtp完整一帧的本地文件区间，判断区间内是否同时存在e0/c0
tpline/宇视属于rtp完整一帧只有音频or视频
回归海康设备
查看时间戳分界的最后一个包，查看二进制数据与dump完整对比。
第0个rtp完整帧（rtp时间戳相同）最后一帧 没看到同时存在音视频起始码，因为I帧过大原因
0210 ed 5a 1b 96 f3 41 04 c1 b3 23 ff 97 4c 67 f1 e9 .Z...A...#..Lg..
0220 90 63 ed 76 bf 84 54 33 b9 7f 6f a6 92 0d 30 e4 .c.v..T3..o...0.
0230 40 6c c1 f6 26 f9 02 e2 84 3f cc 59 2c d0 72 47 @l..&....?.Y,.rG
0240 8c 44 e3 5d 08 c7 88 1c ff d3 f9 5e 3b ff ff 0c .D.].......^;...
0250 b5 f4 00 00 01 bd 00 6a 8c 80 07 2f d4 e5 06 31 .......j.../...1
0260 ff f8 00 02 00 17 00 01 80 00 23 ff a2 a0 e0 f1 ..........#.....
0270 f0 50 e0 67 2e cb e6 58 60 a4 47 17 5b 46 1f d5 .P.g...X`.G.[F..
0280 cb 77 6d 25 82 e7 39 49 13 e7 c0 90 9d 9a 40 46 .wm%..9I......@F
0290 f7 de ed c8 22 2e 3c fb db be 43 94 59 d2 b5 c6 ....".<...C.Y...
02a0 cc ad ed c8 22 2e 3c fb db be 43 94 59 d2 b5 c6 ....".<...C.Y...
02b0 cc ad ff 6c b3 c1 9e 78 eb ff ea 65 d9 21 a9 09 ...l...x...e.!..
02c0 cf 87
第1个rtp完整帧（rtp时间戳相同）
第一个包
0000 00 0c 29 01 52 cc 04 bd 70 26 71 27 08 00 45 00 ..).R...p&q'..E.
0010 01 78 4f fb 40 00 3f 11 73 f8 ac 19 08 52 c0 a8 .xO.@.?.s....R..
0020 01 6e 3a d4 33 91 01 64 ca e3 80 60 00 47 00 00 .n:.3..d...`.G..
0030 0e 10 0b eb df 7c 00 00 01 c0
第二个包
0020 01 6e 3a d4 33 91 05 8c 07 d1 80 60 00 48 00 00 .n:.3......`.H..
0030 0e 10 0b eb df 7c 00 00 01 ba 7f 53 94 89 44 01 .....|.....S..D.
0040 01 36 6b fe ff ff 00 23 a2 a1 00 00 01 e0 0d 02 .6k....#........
0050 8c 80 08 2f d4 e5 22 51 ff ff f8 00 00 00 01 41 .../.."Q.......A
第2个rtp完整帧（rtp时间戳相同）
第一个包
0010 01 78 50 01 40 00 3f 11 73 f2 ac 19 08 52 c0 a8 .xP.@.?.s....R..
0020 01 6e 3a d4 33 91 01 64 7c 9b 80 60 00 4b 00 00 .n:.3..d|..`.K..
0030 1c 20 0b eb df 7c 00 00 01 c0 01 4a 8c 80 07 2f . ...|.....J.../第二个包
0000 00 0c 29 01 52 cc 04 bd 70 26 71 27 08 00 45 00 ..).R...p&q'..E.
0010 05 a0 50 03 40 00 3f 11 6f c8 ac 19 08 52 c0 a8 ..P.@.?.o....R..
0020 01 6e 3a d4 33 91 05 8c 0f 1b 80 60 00 4c 00 00 .n:.3......`.L..
0030 1c 20 0b eb df 7c 00 00 01 ba 7f 53 94 f9 c4 01 . ...|.....S....
0040 01 36 6b fe ff ff 00 23 a2 a2 00 00 01 e0 0c 22 .6k....#......."
调试过程中，发现海康ps流携带了bd，记录一下。
因此解析rtp包后，解析ps流时，需要对完整帧做音频和视频直接处理。
代码
rtp完整帧，时间戳判断
std::vector<std::unique_ptr<PacketBuffer::Packet>> PacketBuffer::FindFrames(uint64_t seq_num)
{
last_rtp_seq_num = seq_num;
std::vector<std::unique_ptr<PacketBuffer::Packet>> found_frames;
for (size_t i = 0; i < buffer_.size(); ++i)
{
size_t cur_index = seq_num % buffer_.size();
const auto &cur_entry = buffer_[cur_index];
size_t index = (seq_num-1+buffer_.size()) % buffer_.size();
const auto &entry = buffer_[index];
if (entry == nullptr || cur_entry == nullptr)
break;
if (entry->marker_bit || cur_entry->timestamp != entry->timestamp)
{
int64_t start_seq_num = seq_num - 1 ;
int start_index = index;
uint64_t last_packet_receive_time_ms = entry->receive_time_ms;
bool continuous = true;
while (true)
{
if (first_seq_num_ == start_seq_num)
break;
if (start_seq_num - 1 == last_sent_seq_num_)
break;
start_index = start_index > 0 ? start_index - 1 : buffer_.size() - 1;
if (buffer_[start_index] == nullptr)
{
uint64_t recv_time = GetPrePacketReceiveTimeMs(start_seq_num);
if (last_packet_receive_time_ms - recv_time <= kDefaultPacketTimeoutInterval)
{
continuous = false;
RT_FUNC_DEBUG_TRACE_THIS(",start_seq_num = " << start_seq_num
<<" last_packet_receive_time_ms=" << last_packet_receive_time_ms
<<" recv_time=" << recv_time);
break;
}
}
else
{
// last_packet_receive_time_ms = buffer_[start_index]->receive_time_ms;
}
--start_seq_num;
}
if (continuous)
{
const uint64_t end_seq_num = seq_num;
uint16_t num_packets = end_seq_num - start_seq_num;
found_frames.reserve(found_frames.size() + num_packets);
for (uint64_t i = start_seq_num; i != end_seq_num; ++i)
{
std::unique_ptr<Packet> &packet = buffer_[i % buffer_.size()];
if (packet == nullptr)
continue;
found_frames.push_back(std::move(packet));
}
}
else
{
break;
}
}
++seq_num;
}
return found_frames;
}
解析ps流代码示例
void GB28181StreamParser::ParserStream(rtc::CopyOnWriteBuffer payload)
{
uint8_t *buf = payload.data(); // 获取载荷数据和长度
unsigned int len = payload.size();
std::unique_ptr<MediaFrame> frame = std::make_unique<MediaFrame>();
frame->frameType_ = FRAME_TYPE_P;
frame->timestamp_ = -1;
// 解析 PS 流
uint8_t *pos = buf;
uint8_t *posLast = pos;
while (pos - buf < len)
{
int curHdrLen = 0;
int curDataLen = -1;
// 检查 PS 包头
if (0x00 == pos[0] && 0x00 == pos[1] && 0x01 == pos[2])
{
curDataLen = 0;
// 根据不同的包类型进行处理
if (0xBA == pos[3]) // 处理 PS 包头
{
uint8_t stuffingLen = (*(uint8_t *)(pos + 13)); // 跳过 9 个字节，暂时不关心它的内容，看第 10 个字节
stuffingLen &= 0x07; // 取低 3 位,长度就是是扩展内容
curHdrLen = 14 + stuffingLen;
pos += curHdrLen; // 移动指针到下一个包
}
else if (0xBB == pos[3]) // 处理系统头
{ // 处理系统头
uint16_t systemHeaderLen = ntohs(*(uint16_t *)(pos + 4)); // 两个字节的系统头长度
curHdrLen = systemHeaderLen + 6;
pos += curHdrLen; // 移动指针到下一个包
}
else if (0xBC == pos[3]) // I帧,psm头
{ // 处理节目流图，可能包含 stream_type
frame->frameType_ = FRAME_TYPE_I;
// frame->width_ = *(uint8_t *)(pos + 26) * 8;
// frame->height_ = *(uint8_t *)(pos + 27) * 8;
int offset = 8;
uint16_t ps_info_len = ntohs(*(uint16_t *)(pos + offset));
offset += (2 + ps_info_len);
uint16_t es_map_len = ntohs(*(uint16_t *)(pos + offset));
offset += 2;
for (int i = 0; i <= es_map_len - 4;) {
uint8_t stream_type = pos[offset + i];
uint8_t stream_id = pos[offset + i + 1];
if (0xE0 == stream_id) {
video_stream_type_ = stream_type;
if(0x1B != stream_type && 0x24 != stream_type)
{
RT_FUNC_WARNING_TRACE_THIS("unknow video stream_type: " << stream_type);
}
} else if (0xC0 == stream_id ) {
audio_stream_type_ = stream_type;
// 0x92 G.722.1 (not supported)
// 0x93 G.723.1 (not supported)
// 0x99 G.729 (not supported)
// 0x9B SVAC (not supported)
if(0x0F != stream_type&& 0x91 != stream_type && 0x90 != stream_type)
{
RT_FUNC_WARNING_TRACE_THIS(
"unknow audio stream_type: " << stream_type);
}
}
uint16_t es_info_len =
ntohs(*(uint16_t *)(pos + offset + i + 2));
i += (4 + es_info_len);
}
uint16_t mapHeaderLen = ntohs(*(uint16_t *)(pos + 4));
curHdrLen = mapHeaderLen + 6;
pos += curHdrLen;
}
else if (0xBD == pos[3]) // 处理私有数据
{
uint16_t pesPacketLen = ntohs(*(uint16_t *)(pos + 4));
curHdrLen = pesPacketLen + 6;
pos += curHdrLen;
}
else if (0xC0 == pos[3]) // 处理音频 PES
{
OnCompleteFrame_i(std::move(frame));
frame = std::make_unique<MediaFrame>();
frame->frameType_ = FRAME_TYPE_A; // 设置帧类型为音频帧
frame->timestamp_ = m_curFrameNormalTs;
if(0x0F == audio_stream_type_){ // AAC
frame->streamType_ = AAC_TYPE;
}
else if(0x90 == audio_stream_type_){ // G.711 A
frame->streamType_ = PCMA_TYPE;
}
else if(0x91 == audio_stream_type_) // G.711 U
{
frame->streamType_ = PCMU_TYPE;
}
else{
frame->streamType_ = OTHER_TYPE;
}
uint16_t pesPacketLen = ntohs(*(uint16_t *)(pos + 4));
uint8_t stuffingLen = (*(uint8_t *)(pos + 8));
curHdrLen = 9 + stuffingLen;
pos += curHdrLen;
curDataLen = pesPacketLen - stuffingLen - 3;
}
else if (0xE0 == pos[3]) // 处理视频 PES
{
//海康设备因根据rtp.timestamp判断完整一帧，导致视频帧和音频帧混在一起。
//因音频处理时重新设置了音频类型，因此需要重新设置。并且防止音频帧和视频帧混在一起，需直接处理完整一帧音频数据。
if(frame->frameType_ == FRAME_TYPE_A){
OnCompleteFrame_i(std::move(frame));
frame = std::make_unique<MediaFrame>();
frame->timestamp_ = m_curFrameNormalTs;
frame->frameType_ = FRAME_TYPE_P;
}
if(0x1B == video_stream_type_){
frame->streamType_ = H264_TYPE;
}else if(0x24 == video_stream_type_){
frame->streamType_ = H265_TYPE;
}else{
frame->streamType_ = OTHER_TYPE;
}
uint16_t pesPacketLen = ntohs(*(uint16_t *)(pos + 4));
uint8_t stuffingLen = (*(uint8_t *)(pos + 8));
curHdrLen = 9 + stuffingLen;
pos += curHdrLen;
curDataLen = pesPacketLen - stuffingLen - 3;
}
}
// 处理剩余数据,PES Body
int remainLen = len - (pos - buf);
if (remainLen <= 0)
break;
curDataLen = -1 == curDataLen ? 1 : curDataLen;
if (curDataLen > 0)
{
int copyLen = remainLen < curDataLen ? remainLen : curDataLen;
frame->append(pos, copyLen);
pos += copyLen;
}
if (posLast == pos)
{
pos += 1;
static int time = 0;
if (time++ % 1024 == 0)
{
RT_FUNC_WARNING_TRACE_THIS(" not found ps header");
}
}
posLast = pos;
}
// 处理完整帧
OnCompleteFrame_i(std::move(frame));
}
最后成功接入海康设备，音视频画面正常。
总结
- 海康设备发出的rtp数据是mark位字段全是false。
- 海康设备音频格式布局是0XBD + 0XC0，与前两者不同，有携带私有数据字段。
- 使用时间戳机制判断rtp完整帧，完整帧存在同时拥有音频和视频。在解析ps流时需要视频和音频要直接处理。
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
