本文分析的webrtc的版本是:m84
平台:win10
WebRTC PeerConnection Client源码分析1-main window
WebRTC PeerConnection Client源码分析2-PeerConnectionClient
WebRTC PeerConnection Client源码分析3-Conductor
注:下文中所谓的回调函数,实际上是虚函数。把虚函数说成回调函数是为了描述方便。
Conductor是PeerConnection Client的核心,现在按照Conductor实际调用顺序逐一分析。
上一篇文章中我已经给一张PeerConnection Client与信令服务器交互的时序图,现在给出一张PeerConnection的时序图:
这张时序图引自:https://blog.youkuaiyun.com/ice_ly000/article/details/103204327
1、创建Conductor对象
int PASCAL wWinMain(HINSTANCE instance, HINSTANCE prev_instance, wchar_t* cmd_line,int cmd_show)
{
...
/*创建Conductor对象*/
rtc::scoped_refptr<Conductor> conductor(new rtc::RefCountedObject<Conductor>(&client, &wnd));
...
}
Conductor::Conductor(PeerConnectionClient* client, MainWindow* main_wnd)
: peer_id_(-1), loopback_(false), client_(client), main_wnd_(main_wnd)
{
client_->RegisterObserver(this); /*注册成为PeerConnectionClient的订阅者*/
main_wnd->RegisterObserver(this); /*注册成为MainWindow的订阅者*/
}
void PeerConnectionClient::RegisterObserver(PeerConnectionClientObserver* callback)
{
RTC_DCHECK(!callback_);
callback_ = callback; /*保存订阅者*/
}
void MainWnd::RegisterObserver(MainWndCallback* callback)
{
callback_ = callback; /*保存订阅者*/
}
Conductor在构造器注册到PeerConnectionClient和MainWnd中。
struct PeerConnectionClientObserver {
/*成功登录信令服务器*/
virtual void OnSignedIn() = 0;
/*登出信令服务器*/
virtual void OnDisconnected() = 0;
/*其他客户端登录信令服务器*/
virtual void OnPeerConnected(int id, const std::string& name) = 0;
/*其他客户端登出信令服务器*/
virtual void OnPeerDisconnected(int peer_id) = 0;
/*接收到其他客户端发送过来的信息*/
virtual void OnMessageFromPeer(int peer_id, const std::string& message) = 0;
/*通知信息已发送*/
virtual void OnMessageSent(int err) = 0;
/*登录信令服务器失败*/
virtual void OnServerConnectionFailure() = 0;
protected:
virtual ~PeerConnectionClientObserver() {}
};
Conductor通过覆写上面的虚函数,用于接收PeerConnectionClient中的消息。
class MainWndCallback {
public:
/*通知登录信令服务器*/
virtual void StartLogin(const std::string& server, int port) = 0;
/*通知登出信令服务器*/
virtual void DisconnectFromServer() = 0;
/*连接对端peer*/
virtual void ConnectToPeer(int peer_id) = 0;
/*与对端断开连接*/
virtual void DisconnectFromCurrentPeer() = 0;
/*自定义消息处理函数*/
virtual void UIThreadCallback(int msg_id, void* data) = 0;
/*关闭Conductor*/
virtual void Close() = 0;
protected:
virtual ~MainWndCallback() {}
};
Conductor通过覆写上面的虚函数,用于接收MainWnd中的消息。
2、主动端主动开启通话
2.1、用户点击对端id
登录信令服务器后,会进入peer list界面。如下图:

通话双方进入peer list界面后,为了后续方便描述,后面把发起通话的一端称为主动peer,把被动发起通话的一端称为被动peer。
主动peer在peer list界面点击对方用户peer id后,会触发如下逻辑:
void MainWnd::OnDefaultAction()
{
...
else if (ui_ == LIST_PEERS) { /*peer list界面peer id被双击*/
...
if (peer_id != -1 && callback_) {
/*回调Conductor::ConnectToPeer,传入要连接peer的id。*/
callback_->ConnectToPeer(peer_id);
}
}
}
...
}
双击用户的peer id后,通过消息处理循环函数进入上述函数,接着通过回调进入Conductor::ConnectToPeer()。
void Conductor::ConnectToPeer(int peer_id) {
RTC_DCHECK(peer_id_ == -1);
RTC_DCHECK(peer_id != -1);
/*peer_connection_必须为空,返回返回。*/
if (peer_connection_.get()) {
main_wnd_->MessageBox(
"Error", "We only support connecting to one peer at a time", true);
return;
}
/*初始化PeerConnection*/
if (InitializePeerConnection()) {
peer_id_ = peer_id;
/*生成offer*/
peer_connection_->CreateOffer(
this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
} else {
main_wnd_->MessageBox("Error", "Failed to initialize PeerConnection", true);
}
}
后面的分析很多都会涉及到webrtc::PeerConnection,要好好配合文章开头给的时序以帮助理解。
bool Conductor::InitializePeerConnection()
{
RTC_DCHECK(!peer_connection_factory_);
RTC_DCHECK(!peer_connection_);
/*1、创建PeerConnectionFactory对象*/
peer_connection_factory_ = webrtc::CreatePeerConnectionFactory(
nullptr /* network_thread */, nullptr /* worker_thread */,
nullptr /* signaling_thread */, nullptr /* default_adm */,
webrtc::CreateBuiltinAudioEncoderFactory(),
webrtc::CreateBuiltinAudioDecoderFactory(),
webrtc::CreateBuiltinVideoEncoderFactory(),
webrtc::CreateBuiltinVideoDecoderFactory(), nullptr /* audio_mixer */,
nullptr /* audio_processing */);
if (!peer_connection_factory_) {
main_wnd_->MessageBox("Error", "Failed to initialize PeerConnectionFactory",
true);
DeletePeerConnection();
return false;
}
/*2、创建PeerConnection对象*/
if (!CreatePeerConnection(/*dtls=*/true)) {
main_wnd_->MessageBox("Error", "CreatePeerConnection failed", true);
DeletePeerConnection();
}
/*3、添加音视频源*/
AddTracks();
return peer_connection_ != nullptr;
}
先创建PeerConnectionFactory对象,然后通过该对象创建PeerConnection对象,最后添加音频、视频源。
bool Conductor::CreatePeerConnection(bool dtls)
{
RTC_DCHECK(peer_connection_factory_);
RTC_DCHECK(!peer_connection_);
webrtc::PeerConnectionInterface::RTCConfiguration config;
/*sdp的格式*/
config.sdp_semantics = webrtc::SdpSemantics::kUnifiedPlan;
/*是否使用dtls*/
config.enable_dtls_srtp = dtls;
/*配置stun/turn服务器*/
webrtc::PeerConnectionInterface::IceServer server;
server.uri = GetPeerConnectionString();
config.servers.push_back(server);
/*创建PeerConnection对象*/
peer_connection_ = peer_connection_factory_->CreatePeerConnection(config, nullptr, nullptr, this);
return peer_connection_ != nullptr;
}
创建PeerConnection对象
void Conductor::AddTracks() {
if (!peer_connection_->GetSenders().empty()) {
return; // Already added tracks.
}
/*创建audio track*/
rtc::scoped_refptr<webrtc::AudioTrackInterface> audio_track(peer_connection_factory_->CreateAudioTrack(kAudioLabel, peer_connection_factory_->CreateAudioSource(cricket::AudioOptions())));
/*添加audio track*/
auto result_or_error = peer_connection_->AddTrack(audio_track, {kStreamId});
if (!result_or_error.ok()) {
RTC_LOG(LS_ERROR) << "Failed to add audio track to PeerConnection: " << result_or_error.error().message();
}
/*创建video device*/
rtc::scoped_refptr<CapturerTrackSource> video_device = CapturerTrackSource::Create();
if (video_device) {
/*创建video track*/
rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track_(peer_connection_factory_->CreateVideoTrack(kVideoLabel, video_device));
/*将video track送至本地视频渲染器,用于本地视频的渲染。*/
main_wnd_->StartLocalRenderer(video_track_);
/*添加video track*/
result_or_error = peer_connection_->AddTrack(video_track_, {kStreamId});
if (!result_or_error.ok()) {
RTC_LOG(LS_ERROR) << "Failed to add video track to PeerConnection: "
<< result_or_error.error().message();
}
} else {
RTC_LOG(LS_ERROR) << "OpenVideoCaptureDevice failed";
}
/*将界面从peer list界面切换至视频显示界面*/
main_wnd_->SwitchToStreamingUI();
}
创建audio track、video track,并添加到PeerConnection中。本地视频需要通过视频渲染器显示出来。
2.2、创建offer
InitializePeerConnection()函数完成后,会调用CreateOffer()生成offer,此函数生成offer是异步的,生成的offer会通过回调函数Conductor::OnSuccess()传回。
/*offer生成成功后的回调*/
void Conductor::OnSuccess(webrtc::SessionDescriptionInterface* desc)
{
/*生成的offer需要通过SetLocalDescription设置到PeerConnection中*/
peer_connection_->SetLocalDescription(DummySetSessionDescriptionObserver::Create(), desc);
std::string sdp;
desc->ToString(&sdp); /*将offer转成string*/
...
/*将offer包装到json中*/
Json::StyledWriter writer;
Json::Value jmessage;
jmessage[kSessionDescriptionTypeName] = webrtc::SdpTypeToString(desc->GetType());
jmessage[kSessionDescriptionSdpName] = sdp;
/*发送至信令服务器*/
SendMessage(writer.write(jmessage));
}
/*offer生成失败后的回调*/
void Conductor::OnFailure(webrtc::RTCError error) {
RTC_LOG(LERROR) << ToString(error.type()) << ": " << error.message();
}
offer的生成是异步的,成功的生成offer或生成失败会调用对应的回调函数(虚函数)。
class DummySetSessionDescriptionObserver : public webrtc::SetSessionDescriptionObserver
{
public:
static DummySetSessionDescriptionObserver* Create() {
return new rtc::RefCountedObject<DummySetSessionDescriptionObserver>();
}
virtual void OnSuccess() { RTC_LOG(INFO) << __FUNCTION__; }
virtual void OnFailure(webrtc::RTCError error) {
RTC_LOG(INFO) << __FUNCTION__ << " " << ToString(error.type()) << ": " << error.message();
}
};
在调用SetLocalDescription和SetRemoteDescription时,其处理的结果是通过回调函数告知的。当成功时,会调用OnSuccess
函数,失败时会调用OnFailure
函数。
传入SetLocalDescription和SetRemoteDescription中用于反馈结果的是一个类对象,该类对象需要继承webrtc::SetSessionDescriptionObserver接口,同时覆写OnSuccess和OnFailure虚函数,这样就可形成多态,将处理的结果通过覆写的虚函数回调上来以供用户处理。
void Conductor::SendMessage(const std::string& json_object) {
std::string* msg = new std::string(json_object);
/*投递的消息类型是SEND_MESSAGE_TO_PEER*/
main_wnd_->QueueUIThreadCallback(SEND_MESSAGE_TO_PEER, msg);
}
通过回调返回的offer,并没有直接向信令服务器发送。信令的发送需要在主线程中进行,此时还在webrtc的内部线程中,所以通过上面的函数将待发送的offer信令投递给了主线程,主线程在处理这个消息时,会将对应的信令发送给信令服务器。
/*自定义的消息类型*/
enum WindowMessages {
UI_THREAD_CALLBACK = WM_APP + 1,
};
/*向Windows Application消息队列投递自定义的消息*/
void MainWnd::QueueUIThreadCallback(int msg_id, void* data)
{
/*向消息队列投递自己定义的消息*/
::PostThreadMessage(ui_thread_id_, UI_THREAD_CALLBACK, static_cast<WPARAM>(msg_id), reinterpret_cast<LPARAM>(data));
}
bool MainWnd::PreTranslateMessage(MSG* msg)
{
...
/*处理自定义类型的消息*/
} else if (msg->hwnd == NULL && msg->message == UI_THREAD_CALLBACK) {
callback_->UIThreadCallback(static_cast<int>(msg->wParam), reinterpret_cast<void*>(msg->lParam)); /*进入Conductor处理自己发出的消息*/
ret = true;
}
...
}
在MainWnd中自定义了消息类型UI_THREAD_CALLBACK
,通过MainWnd::QueueUIThreadCallback函数可以向Windows Application消息队列投递自定义的消息。在MainWnd::PreTranslateMessage中处理投递的消息。
void Conductor::UIThreadCallback(int msg_id, void* data)
{
switch (msg_id) {
case PEER_CONNECTION_CLOSED:
case SEND_MESSAGE_TO_PEER: {
case NEW_TRACK_ADDED: {
case TRACK_REMOVED: {
default:
RTC_NOTREACHED();
break;
}
}
通过MainWnd::QueueUIThreadCallback函数投递的消息,最终会在Conductor::UIThreadCallback得到处理。
void Conductor::UIThreadCallback(int msg_id, void* data)
{
switch (msg_id) {
...
case SEND_MESSAGE_TO_PEER: {
RTC_LOG(INFO) << "SEND_MESSAGE_TO_PEER";
/*获取消息*/
std::string* msg = reinterpret_cast<std::string*>(data);
if (msg) {
pending_messages_.push_back(msg); /*将消息存放到队列中,保证消息有序发送。*/
}
/*如果有消息待发送,且PeerConnectionClient可以发送信令。*/
if (!pending_messages_.empty() && !client_->IsSendingMessage()) {
msg = pending_messages_.front(); /*从队首获取一条待发送的消息*/
pending_messages_.pop_front();
/*通过PeerConnectionClient放该消息*/
if (!client_->SendToPeer(peer_id_, *msg) && peer_id_ != -1) {
RTC_LOG(LS_ERROR) << "SendToPeer failed";
DisconnectFromServer();
}
delete msg;
}
if (!peer_connection_.get())
peer_id_ = -1;
break;
}
...
}
上述在发送offer信令时,会通过上述代码发送给信令服务器。
3、被动端被动开启视频通话

被动peer登录信令服务器后,处于peer listen界面,如上图。
当被动peer收到信令服务器发送过来的offer信令后,其逻辑如下:
void PeerConnectionClient::OnHangingGetRead(rtc::AsyncSocket* socket)
{
...
} else {
/*用于处理offer、answer、candidate、bye信令*/
OnMessageFromPeer(static_cast<int>(peer_id), notification_data_.substr(pos));
}
...
}
当信令服务器发送过来offer信令时,PeerConnectionClient::OnHangingGetRead函数被触发。
void PeerConnectionClient::OnMessageFromPeer(int peer_id, const std::string& message)
{
...
} else {
/*收到的是offer、answer、candidate信令*/
callback_->OnMessageFromPeer(peer_id, message);
}
}
在PeerConnectionClient对象中,通过OnMessageFromPeer回调函数,将信令送至了Conductor::OnMessageFromPeer函数中。
void Conductor::OnMessageFromPeer(int peer_id, const std::string& message)
{
RTC_DCHECK(peer_id_ == peer_id || peer_id_ == -1);
RTC_DCHECK(!message.empty());
/*此时被动peer还没有创建PeerConnection对象*/
if (!peer_connection_.get()) {
RTC_DCHECK(peer_id_ == -1);
peer_id_ = peer_id;
/*创建PeerConnection对象*/
if (!InitializePeerConnection()) {
RTC_LOG(LS_ERROR) << "Failed to initialize our PeerConnection instance";
client_->SignOut();
return;
}
} else if (peer_id != peer_id_) {
RTC_DCHECK(peer_id_ != -1);
RTC_LOG(WARNING) << "Received a message from unknown peer while already in a "
"conversation with a different peer.";
return;
}
Json::Reader reader;
Json::Value jmessage;
/*将收到的消息解析成json对象*/
if (!reader.parse(message, jmessage)) {
RTC_LOG(WARNING) << "Received unknown message. " << message;
return;
}
std::string type_str;
std::string json_object;
/*从json消息中解析出消息的类型*/
rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionTypeName, &type_str);
if (!type_str.empty()) {
...
/*获取消息的类型*/
absl::optional<webrtc::SdpType> type_maybe = webrtc::SdpTypeFromString(type_str);
if (!type_maybe) {
RTC_LOG(LS_ERROR) << "Unknown SDP type: " << type_str;
return;
}
webrtc::SdpType type = *type_maybe;
std::string sdp;
/*从json消息中获取sdp,此处为offer。*/
if (!rtc::GetStringFromJsonObject(jmessage, kSessionDescriptionSdpName, &sdp)){
RTC_LOG(WARNING) << "Can't parse received session description message.";
return;
}
/*将offer转成webrtc可以理解的对象*/
webrtc::SdpParseError error;
std::unique_ptr<webrtc::SessionDescriptionInterface> session_description =
webrtc::CreateSessionDescription(type, sdp, &error);
if (!session_description) {
RTC_LOG(WARNING) << "Can't parse received session description message. "
"SdpParseError was: " << error.description;
return;
}
RTC_LOG(INFO) << " Received session description :" << message;
/*将offer通过SetRemoteDescription设置到PeerConnection中*/
peer_connection_->SetRemoteDescription(DummySetSessionDescriptionObserver::Create(),session_description.release());
/*收到了对端的offer,本端需要产生answer。*/
if (type == webrtc::SdpType::kOffer) {
peer_connection_->CreateAnswer(
this, webrtc::PeerConnectionInterface::RTCOfferAnswerOptions());
}
} else {
...
}
}
被动peer在收到offer时,还没有创建PeerConnection对象,所以先创建PeerConnection对象,过程同主动peer。
将收到的消息解析成json,然后从json中获取type
和sdp
对应的值,通过SetRemoteDescription设置offer,然后通过CreateAnswer生成answer,注意answer的生成也是异步的,生成后也会通过Conductor::OnSuccess回调函数传递上来,其处理方式同offer,且生成的answer会通过信令服务器发送给对端。
offer信令的格式如下:
{
"sdp" : "v=0\r\no=- 7038993275920826226 ...",
"type" : "offer"
}
4、处理candidate
生成offer和answer后,WebRTC会通过Conductor::OnIceCandidate()函数上传生成的candidate。
void Conductor::OnIceCandidate(const webrtc::IceCandidateInterface* candidate) {
RTC_LOG(INFO) << __FUNCTION__ << " " << candidate->sdp_mline_index();
Json::StyledWriter writer;
Json::Value jmessage;
/*将candidate包装到json中*/
jmessage[kCandidateSdpMidName] = candidate->sdp_mid();
jmessage[kCandidateSdpMlineIndexName] = candidate->sdp_mline_index();
std::string sdp;
if (!candidate->ToString(&sdp)) {
RTC_LOG(LS_ERROR) << "Failed to serialize candidate";
return;
}
jmessage[kCandidateSdpName] = sdp;
/*通过信令服务器发送给对端*/
SendMessage(writer.write(jmessage));
}
生成的candidate需要通过信令服务器发送给对端。
candidate的信令格式如下:
{
"candidate" : "candidate:2999745851 1 udp 2122260223 192.168.56.1 58572 typ host generation 0 ufrag Cy2E network-id 3",
"sdpMLineIndex" : 1,
"sdpMid" : "1"
}
void Conductor::OnMessageFromPeer(int peer_id, const std::string& message)
{
...
if (!type_str.empty()) {
...
} else { /*处理收到的candidate*/
std::string sdp_mid;
int sdp_mlineindex = 0;
std::string sdp;
/*从json中解析处需要的数据*/
if (!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpMidName, &sdp_mid) ||
!rtc::GetIntFromJsonObject(jmessage, kCandidateSdpMlineIndexName,
&sdp_mlineindex) ||
!rtc::GetStringFromJsonObject(jmessage, kCandidateSdpName, &sdp)) {
RTC_LOG(WARNING) << "Can't parse received message.";
return;
}
webrtc::SdpParseError error;
/*根据接收的信息生成candidate对象*/
std::unique_ptr<webrtc::IceCandidateInterface> candidate(
webrtc::CreateIceCandidate(sdp_mid, sdp_mlineindex, sdp, &error));
if (!candidate.get()) {
RTC_LOG(WARNING) << "Can't parse received candidate message. "
"SdpParseError was: " << error.description;
return;
}
/*给PeerConnection对象添加candidate*/
if (!peer_connection_->AddIceCandidate(candidate.get())) {
RTC_LOG(WARNING) << "Failed to apply the received candidate";
return;
}
RTC_LOG(INFO) << " Received candidate :" << message;
}
}
将收到的candidate通过AddIceCandidate函数添加到PeerConnection对象中。
5、收到远端视频流
void Conductor::OnAddTrack(
rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver,
const std::vector<rtc::scoped_refptr<webrtc::MediaStreamInterface>>&
streams) {
RTC_LOG(INFO) << __FUNCTION__ << " " << receiver->id();
/*向主线程发送自定义的消息*/
main_wnd_->QueueUIThreadCallback(NEW_TRACK_ADDED, receiver->track().release());
}
经过上面的步骤后,建立了音视频的连接,当收到远端的视频流时,会通过OnAddTrack()通知用户。
void Conductor::UIThreadCallback(int msg_id, void* data)
{
...
case NEW_TRACK_ADDED: {
auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
if (track->kind() == webrtc::MediaStreamTrackInterface::kVideoKind) {
/*获取远端video track*/
auto* video_track = static_cast<webrtc::VideoTrackInterface*>(track);
/*送至MainWnd处理*/
main_wnd_->StartRemoteRenderer(video_track);
}
track->Release();
break;
}
...
}
void MainWnd::StartRemoteRenderer(webrtc::VideoTrackInterface* remote_video)
{
/*生成远端视频渲染器,同时将远端视频渲染器注册到webrtc中。*/
remote_renderer_.reset(new VideoRenderer(handle(), 1, 1, remote_video));
}
在这个函数中会将MainWnd中的remote_renderer_添加到WebRTC中,用于接收并渲染远端视频帧。
void Conductor::OnRemoveTrack(
rtc::scoped_refptr<webrtc::RtpReceiverInterface> receiver) {
RTC_LOG(INFO) << __FUNCTION__ << " " << receiver->id();
main_wnd_->QueueUIThreadCallback(TRACK_REMOVED, receiver->track().release());
}
void Conductor::UIThreadCallback(int msg_id, void* data)
{
...
case TRACK_REMOVED: {
auto* track = reinterpret_cast<webrtc::MediaStreamTrackInterface*>(data);
track->Release();
break;
}
...
}
远端视频流移除的逻辑
6、远端peer关闭
当远端关闭时,会想信令服务器发送一条sign out信令,信令服务器会转发该信令。本端收到信令后的处理逻辑如下:
void PeerConnectionClient::OnHangingGetRead(rtc::AsyncSocket* socket)
{
...
if (connected) {
peers_[id] = name;
callback_->OnPeerConnected(id, name);
} else { /*收到远端peer关闭的信令。*/
peers_.erase(id);
callback_->OnPeerDisconnected(id); /*通知Conductor远端关闭*/
}
...
}
void Conductor::OnPeerDisconnected(int id)
{
RTC_LOG(INFO) << __FUNCTION__;
if (id == peer_id_) {
RTC_LOG(INFO) << "Our peer disconnected";
/*通知远端关闭*/
main_wnd_->QueueUIThreadCallback(PEER_CONNECTION_CLOSED, NULL);
} else {
/*刷新peer list界面的用户列表*/
if (main_wnd_->current_ui() == MainWindow::LIST_PEERS)
main_wnd_->SwitchToPeerList(client_->peers());
}
}
void Conductor::UIThreadCallback(int msg_id, void* data)
{
switch (msg_id) {
case PEER_CONNECTION_CLOSED:
RTC_LOG(INFO) << "PEER_CONNECTION_CLOSED";
DeletePeerConnection(); /*释放PeerConnection对象*/
if (main_wnd_->IsWindow()) { /*如果在视频渲染界面*/
if (client_->is_connected()) { /*如果处于连接状态,则将界面切换至peer list界面*/
main_wnd_->SwitchToPeerList(client_->peers());
} else {
main_wnd_->SwitchToConnectUI(); /*将界面切换至connect界面*/
}
} else {
DisconnectFromServer();
}
break;
...
}
}
7、本端关闭程序
void Conductor::Close()
{
client_->SignOut(); /*向信令服务器发送sign out信令*/
DeletePeerConnection(); /*释放PeerConnection对象*/
}
关闭本端程序,首先需要向信令服务器发送sign out信令,然后销毁PeerConnection对象。