webRTC的音频和视频引擎的使用
运行的时候命令行参数为: 0 127.0.0.1 8888#include <iostream> #include <Windows.h> #include <tchar.h> #include <WinSock.h> #include <process.h> #include "voe_base.h" #include "voe_network.h" #include "voe_codec.h" #include "voe_network.h" #include "voe_rtp_rtcp.h" #include "voe_errors.h" #include "voe_audio_processing.h" #include "vie_base.h" #include "vie_network.h" #include "vie_codec.h" #include "vie_network.h" #include "vie_rtp_rtcp.h" #include "vie_image_process.h" #include "vie_capture.h" #include "webrtc/modules/video_capture/include/video_capture_factory.h" #include "vie_render.h" #include <list> #pragma comment(lib,"ws2_32.lib") //* #pragma comment(lib,"audio_coding_module.lib") #pragma comment(lib, "audio_conference_mixer.lib") #pragma comment(lib,"audio_device.lib") #pragma comment(lib,"audio_processing.lib") #pragma comment(lib,"audio_processing_sse2.lib") #pragma comment(lib,"audioproc_debug_proto.lib") #pragma comment(lib,"bitrate_controller.lib") //#pragma comment(lib,"call.lib") #pragma comment(lib,"CNG.lib") #pragma comment(lib,"common_video.lib") #pragma comment(lib,"crnspr.lib") #pragma comment(lib,"crnss.lib") #pragma comment(lib,"directshow_baseclasses.lib") #pragma comment(lib,"expat.lib") #pragma comment(lib,"G711.lib") #pragma comment(lib,"G722.lib") #pragma comment(lib,"genperf_libs.lib") #pragma comment(lib,"gunit.lib") #pragma comment(lib,"icui18n.lib") #pragma comment(lib,"icuuc.lib") #pragma comment(lib,"iLBC.lib") #pragma comment(lib,"iSAC.lib") #pragma comment(lib,"iSACFix.lib") #pragma comment(lib,"jsoncpp.lib") #pragma comment(lib,"libjingle.lib") #pragma comment(lib,"libjingle_media.lib") #pragma comment(lib,"libjingle_p2p.lib") #pragma comment(lib,"libjingle_peerconnection.lib") #pragma comment(lib,"libjingle_sound.lib") //#pragma comment(lib,"libjingle_unittest_main.lib") #pragma comment(lib,"libjingle_xmpphelp.lib") #pragma comment(lib,"libjpeg.lib") #pragma comment(lib,"libsrtp.lib") #pragma comment(lib,"libvpx.lib") #pragma comment(lib,"libvpx_asm_offsets.lib") #pragma comment(lib,"libvpx_asm_offsets_vp9.lib") #pragma comment(lib,"libvpx_intrinsics.lib") #pragma comment(lib,"libyuv.lib") #pragma comment(lib,"media_file.lib") #pragma comment(lib,"NetEq.lib") #pragma comment(lib,"nss_static.lib") #pragma comment(lib,"opus.lib") #pragma comment(lib,"paced_sender.lib") #pragma comment(lib,"PCM16B.lib") //#pragma comment(lib,"peerconnection_client.lib") #pragma comment(lib,"protobuf_full_do_not_use.lib") #pragma comment(lib,"protobuf_lite.lib") #pragma comment(lib,"remote_bitrate_estimator.lib") #pragma comment(lib,"resampler.lib") #pragma comment(lib,"rtp_rtcp.lib") #pragma comment(lib,"signal_processing.lib") #pragma comment(lib,"sqlite3.lib") #pragma comment(lib,"system_wrappers.lib") #pragma comment(lib,"udp_transport.lib") #pragma comment(lib,"vad.lib") #pragma comment(lib,"video_capture_module.lib") #pragma comment(lib,"video_coding_utility.lib") #pragma comment(lib,"video_engine_core.lib") #pragma comment(lib,"video_processing.lib") #pragma comment(lib,"video_processing_sse2.lib") #pragma comment(lib,"video_render_module.lib") #pragma comment(lib,"voice_engine_core.lib") #pragma comment(lib,"webrtc_i420.lib") #pragma comment(lib,"webrtc_opus.lib") #pragma comment(lib,"webrtc_utility.lib") #pragma comment(lib,"webrtc_video_coding.lib") #pragma comment(lib,"webrtc_vp8.lib") //*/ using namespace webrtc; int ViECreateWindow(HWND &hwndMain, int xPos, int yPos, int width, int height, TCHAR* className); LRESULT CALLBACK ViEAutoTestWinProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) ; unsigned __stdcall ServiceRun(void* trans); unsigned __stdcall ServiceRtcpRun(void* trans); class my_transportation : public Transport { public : my_transportation(VoENetwork* vo):netw(vo) { } int SendPacket(int channel,const void *data,int len); int SendRTCPPacket(int channel, const void *data, int len); private : VoENetwork* netw ; }; int my_transportation::SendPacket(int channel,const void *data,int len) { netw->ReceivedRTPPacket(channel, data, len); return len; } int my_transportation::SendRTCPPacket(int channel, const void *data, int len) { netw->ReceivedRTCPPacket(channel, data, len); return len; } class MyObserver : public VoiceEngineObserver { public: virtual void CallbackOnError(const int channel, const int err_code); }; void MyObserver::CallbackOnError(const int channel, const int err_code) { // Add printf for other error codes here if (err_code == VE_TYPING_NOISE_WARNING) { printf(" TYPING NOISE DETECTED \n"); } else if (err_code == VE_RECEIVE_PACKET_TIMEOUT) { printf(" RECEIVE PACKET TIMEOUT \n"); } else if (err_code == VE_PACKET_RECEIPT_RESTARTED) { printf(" PACKET RECEIPT RESTARTED \n"); } else if (err_code == VE_RUNTIME_PLAY_WARNING) { printf(" RUNTIME PLAY WARNING \n"); } else if (err_code == VE_RUNTIME_REC_WARNING) { printf(" RUNTIME RECORD WARNING \n"); } else if (err_code == VE_SATURATION_WARNING) { printf(" SATURATION WARNING \n"); } else if (err_code == VE_RUNTIME_PLAY_ERROR) { printf(" RUNTIME PLAY ERROR \n"); } else if (err_code == VE_RUNTIME_REC_ERROR) { printf(" RUNTIME RECORD ERROR \n"); } else if (err_code == VE_REC_DEVICE_REMOVED) { printf(" RECORD DEVICE REMOVED \n"); } } int VoiceTest() { VoiceEngine* _voiceEngine; VoEBase* _veBase; VoENetwork* _veNetwork; VoECodec* _veCodec; VoERTP_RTCP* _veRTCP; VoEAudioProcessing* _veApmPtr; _voiceEngine = VoiceEngine::Create(); _veBase = VoEBase::GetInterface(_voiceEngine); _veNetwork = VoENetwork::GetInterface(_voiceEngine); _veCodec = VoECodec::GetInterface(_voiceEngine); _veRTCP = VoERTP_RTCP::GetInterface(_voiceEngine); _veApmPtr = VoEAudioProcessing::GetInterface(_voiceEngine); my_transportation my_transport(_veNetwork); //_vieBase->SetVoiceEngine(_voiceEngine); //编码器选择,编码的配置参数可以配置CodecInst: // Each codec supported can be described by this structure. /******** struct CodecInst { int pltype; char plname[32]; int plfreq; int pacsize; int channels; int rate; };********/ CodecInst voiceCodec; int numOfVeCodecs = _veCodec->NumOfCodecs(); for(int i=0; i<numOfVeCodecs;++i) { if(_veCodec->GetCodec(i,voiceCodec)!=-1) { if(strncmp(voiceCodec.plname,"ISAC",4)==0) break; } } // define iSAC codec parameters strcpy(voiceCodec.plname, "ISAC"); voiceCodec.plfreq = 16000; // iSAC宽带模式 voiceCodec.pltype = 103; // 默认动态负载类型 voiceCodec.pacsize = 480; // 480kbps,即使用30ms的packet size voiceCodec.channels = 1; // 单声道 voiceCodec.rate = -1; // 信道自适应模式,单位bps int ret = 0; ret = _veBase->Init(); if (ret<0) { printf("ERR _veBase->Init() \n"); return -1; } MyObserver my_observer; //网络传输应用 int _audioChannel = _veBase->CreateChannel(); ret = _veNetwork->RegisterExternalTransport(_audioChannel, my_transport); //ret = _veNetwork->RegisterExternalTransport(_audioChannel_paly, my_transport); //ret = _veBase->RegisterVoiceEngineObserver(my_observer); ret = _veRTCP->SetRTCPStatus(_audioChannel, true); if (ret<0) { printf("ERR SetRTCPStatus \n"); return -1; } ret = _veCodec->SetSendCodec(_audioChannel, voiceCodec); if (ret<0) { printf( "ERR _veCodec->SetSendCodec\n"); return -1; } ret = _veBase->StartPlayout(_audioChannel); if (ret<0) { printf( "ERR __veBase->StartPlayout\n"); return -1; } //音频和视频绑定 //_vieBase->ConnectAudioChannel(_channelId,_audioChannel); //网络发送接收配置,远程端口:remotePort 目的IP:IP /* int remotePort = atoi(argvc[2]); char* remoteIP = argvc[1]; int localPort = atoi(argvc[3]); printf("remoteIP %s remotePort %d localRecvPort %d \n", remoteIP, remotePort, localPort);*/ /*ret = _veBase->SetSendDestination(_audioChannel, remotePort, remoteIP); if (ret<0) { DWORD err = _veBase->LastError(); printf( "ERR _veBase->SetSendDestination\n"); return -1; }*/ //本地接收 /*ret=_veBase->SetLocalReceiver(_audioChannel, localPort); if (ret<0) { printf( "ERR _veBase->SetLocalReceiver\n"); return -1; }*/ ret = _veBase->StartPlayout(_audioChannel); if (ret<0) { printf( "ERR _veBase->StartPlayout\n"); return -1; } ret = _veBase->StartSend(_audioChannel); if (ret<0) { printf( "ERR _veBase->StartSend\n"); return -1; } ret = _veBase->StartReceive(_audioChannel); if (ret<0) { printf( "ERR _veBase->StartReceive\n"); return -1; } NsModes mode(kNsDefault);// ret = _veApmPtr->SetRxNsStatus(_audioChannel, true, mode); if (ret<0) { printf( "ERR _veBase->SetRxNsStatus\n"); return -1; } AgcModes agcmode(kAgcDefault); ret = _veApmPtr->SetRxAgcStatus(_audioChannel, true, agcmode); if (ret<0) { printf( "ERR _veApmptr->SetRxAgcStatus\n"); return -1; } ret = _veApmPtr->SetEcStatus(true, kEcAec); if (ret<0) { printf( "ERR _veApmptr->SetRxAgcStatus\n"); return -1; } /*ret = _veApmPtr->EnableHighPassFilter(true); if (ret<0) { printf( "ERR _veApmPtr->EnableHighPassFilter\n"); return -1; } */ system("pause"); _veBase->StopReceive(_audioChannel); _veBase->StopSend(_audioChannel); //结束,释放资源 if (_voiceEngine) { _veBase->DeleteChannel(_audioChannel); _veBase->Release(); _veNetwork->Release(); _veCodec->Release(); _veRTCP->Release(); _veApmPtr->Release(); VoiceEngine::Delete(_voiceEngine); } return 0; } struct Node { int len; void* data; }; class my_transportationVideo : public Transport { public : my_transportationVideo(ViENetwork* vo, std::string ip="127.0.0.1", int port=8888):netw(vo) { socket=::socket(AF_INET, SOCK_DGRAM, IPPROTO_IP); addr.sin_addr.S_un.S_addr=inet_addr(ip.c_str()); addr.sin_family=AF_INET; addr.sin_port=ntohs(port); addr_rtcp.sin_addr.S_un.S_addr=inet_addr(ip.c_str()); addr_rtcp.sin_family=AF_INET; addr_rtcp.sin_port=ntohs(port+1); } int SendPacket(int channel,const void *data,int len); int SendRTCPPacket(int channel, const void *data, int len); int SendRtpData(const void *data, int len) { int err = netw->ReceivedRTPPacket(channel_, data, len); if (err < 0) { printf(" netw->ReceivedRTPPacket err \n"); } return err; } int SendRtcpData(const void *data, int len) { int err = netw->ReceivedRTCPPacket(channel_rtcp, data, len); if (err < 0) { printf(" netw->ReceivedRTCPPacket err \n"); } return err; } private : ViENetwork* netw ; SOCKET socket; sockaddr_in addr; sockaddr_in addr_rtcp; int channel_; int channel_rtcp; }; int my_transportationVideo::SendPacket(int channel,const void *data,int len) { //printf("..Recv.....len %d\n", len); /*int err = netw->ReceivedRTPPacket(channel, data, len); if (err < 0) { printf(" netw->ReceivedRTPPacket err \n"); }*/ printf("senrtp pack len %d\n", len); channel_ = channel; int slen=sizeof(addr); int ret = sendto(socket,(char*)data,len, 0,(SOCKADDR*)&addr,slen);//用回原来的地址结构 if (ret < 0) { printf("send tcp err\n"); } return len; } int my_transportationVideo::SendRTCPPacket(int channel, const void *data, int len) { //printf("..recv RTCP.....len %d\n", len); //int err = netw->ReceivedRTCPPacket(channel, data, len); //if (err < 0) //{ // printf(" netw->ReceivedRTCPPacket err \n"); //} channel_rtcp = channel; int slen=sizeof(addr); int ret = sendto(socket,(char*)data,len, 0,(SOCKADDR*)&addr_rtcp,slen);//用回原来的地址结构 if (ret < 0) { printf("send rtcp err\n"); } return len; } void CaptureTest(std::string remoteip, int port) { VideoEngine* _videoEngine; ViEBase* _veBase; ViENetwork* _veNetwork; ViECodec* _veCodec; ViERTP_RTCP* _veRTCP; ViEImageProcess* _veApmPtr; ViECapture* capture; ViERender* render; int captureId =-1; int ret = 0; HWND randerwindow_capture = NULL; HWND randerwindow_remote = NULL; _videoEngine = VideoEngine::Create(); _veBase = ViEBase::GetInterface(_videoEngine); _veNetwork = ViENetwork::GetInterface(_videoEngine); _veCodec = ViECodec::GetInterface(_videoEngine); _veRTCP = ViERTP_RTCP::GetInterface(_videoEngine); _veApmPtr = ViEImageProcess::GetInterface(_videoEngine); capture = ViECapture::GetInterface(_videoEngine); render = ViERender::GetInterface(_videoEngine); ret = _veBase->Init(); if (ret < 0) { printf("_veBase->Init err\n"); return; } int _videoChannel = -1; ret = _veBase->CreateChannel(_videoChannel); if (ret < 0) { printf("_veBase->CreateChannel err\n"); return; } const unsigned int KMaxDeviceNameLength = 128; const unsigned int KMaxUniqueIdLength = 256; char deviceName[KMaxDeviceNameLength]; memset(deviceName, 0, KMaxDeviceNameLength); char uniqueId[KMaxUniqueIdLength]; memset(uniqueId, 0, KMaxUniqueIdLength); webrtc::VideoCaptureModule* vcpm_; webrtc::VideoCaptureModule::DeviceInfo* devInfo = webrtc::VideoCaptureFactory::CreateDeviceInfo(0); for (size_t captureIdx = 0; captureIdx < devInfo->NumberOfDevices(); captureIdx++) { ret = devInfo->GetDeviceName(captureIdx, deviceName, KMaxDeviceNameLength, uniqueId, KMaxUniqueIdLength); if (ret < 0) { continue; } vcpm_ = webrtc::VideoCaptureFactory::Create( captureIdx, uniqueId); if (vcpm_ == NULL) // Failed to open this device. Try next. { continue; } vcpm_->AddRef(); int error = capture->AllocateCaptureDevice(*vcpm_, captureId); if (error == 0) { printf("get CaptureDevice OK....\n"); break; } } delete devInfo; ret = capture->StartCapture(captureId); if (ret < 0) { printf("capture->StartCapture err\n"); return; } ret = ViECreateWindow(randerwindow_capture, 0, 0, 1024, 768, _T("ViE Capture")); if (ret < 0) { printf("ViECreateWindow err\n"); return; } ret = render->AddRenderer(captureId, randerwindow_capture, 0, 0.0, 0.0, 1.0, 1.0); if (ret < 0) { printf("render->AddRenderer err\n"); return; } ret = ViECreateWindow(randerwindow_remote, 0, 0, 1024, 768, _T("ViE Remote")); if (ret < 0) { printf("ViECreateWindow err\n"); return; } ret = render->AddRenderer(_videoChannel, randerwindow_remote, 1, 0.0, 0.0, 1.0, 1.0); ret = render->StartRender(captureId); if (ret < 0) { printf("render->StartRender 1 err\n"); return; } ret = render->StartRender(_videoChannel); if (ret < 0) { printf("render->StartRender 2 err\n"); return; } VideoCodec videoCodec; int numOfVeCodecs = _veCodec->NumberOfCodecs(); for(int i=0; i<numOfVeCodecs;++i) { if(_veCodec->GetCodec(i,videoCodec)!=-1) { if(videoCodec.codecType == kVideoCodecVP8) break; /*if (videoCodec.codecType == kVideoCodecI420) { break; }*/ } } ret = capture->ConnectCaptureDevice(captureId, _videoChannel); if (ret < 0) { printf("capture->ConnectCaptureDevice err\n"); return; } ret = _veRTCP->SetRTCPStatus(_videoChannel, webrtc::kRtcpCompound_RFC4585); if (ret < 0) { printf("_veRTCP->SetRTCPStatu err\n"); return; } ret = _veRTCP->SetKeyFrameRequestMethod(_videoChannel, webrtc::kViEKeyFrameRequestPliRtcp); if (ret < 0) { printf("_veRTCP->SetKeyFrameRequestMethod err\n"); return; } ret = _veRTCP->SetTMMBRStatus(_videoChannel, true); if (ret < 0) { printf("__veRTCP->SetTMMBRStatus err\n"); return; } my_transportationVideo* my_transport =new my_transportationVideo(_veNetwork, remoteip, port); ret = _veNetwork->RegisterSendTransport(_videoChannel, *my_transport); if (ret < 0) { printf("_veNetwork->RegisterSendTransport err\n"); return; } ret = _veCodec->SetSendCodec(_videoChannel, videoCodec); if (ret<0) { printf( "ERR _veCodec->SetSendCodec\n"); return; } ret = _veCodec->SetReceiveCodec(_videoChannel, videoCodec); if (ret<0) { printf( "ERR _veCodec->SetReceiveCodec\n"); return; } //const char* ip_address = "127.0.0.1"; //const uint16_t rtp_port = 6000; //ret = _veNetwork->SetLocalReceiver(_videoChannel, rtp_port); //if (ret < 0) //{ // printf("_veNetwork->SetLocalReceiver \n"); // return; //} //ret = _veNetwork->SetSendDestination(_videoChannel, ip_address, rtp_port); //if (ret < 0) //{ // printf("_veNetwork->SetSendDestination \n"); // return; //} ret = _veBase->StartSend(_videoChannel); if (ret<0) { printf( "ERR _veBase->StartSend\n"); return; } ret = _veBase->StartReceive(_videoChannel); if (ret<0) { printf( "ERR _veBase->StartSend\n"); return; } //here create thread HANDLE hThread; unsigned threadID; hThread = (HANDLE) _beginthreadex(NULL, 0, ServiceRun, my_transport, 0, &threadID); hThread = (HANDLE) _beginthreadex(NULL, 0, ServiceRtcpRun, my_transport, 0, &threadID); MSG msg; while (true) { if (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) { TranslateMessage(&msg); DispatchMessage(&msg); } Sleep(10); //printf("running.....\n"); } system("pause"); _veBase->StopReceive(_videoChannel); _veBase->StopSend(_videoChannel); //结束,释放资源 if (_videoEngine) { capture->DisconnectCaptureDevice(_videoChannel); capture->StopCapture(captureId); capture->ReleaseCaptureDevice(captureId); if (vcpm_) vcpm_->Release(); _veBase->DeleteChannel(_videoChannel); _veBase->Release(); _veNetwork->Release(); _veCodec->Release(); _veRTCP->Release(); VideoEngine::Delete(_videoEngine); } } unsigned __stdcall voiceRun(void* ) { VoiceTest(); return 0; } int main(int argc, char* argvc[]) { int type = atoi(argvc[1]); if (type == 1) { VoiceTest(); }else if (type == 0) { HANDLE hThread; unsigned threadID; hThread = (HANDLE) _beginthreadex(NULL, 0, voiceRun, NULL, 0, &threadID); CaptureTest(std::string(argvc[2]), atoi(argvc[3])); } return 0; } int ViECreateWindow(HWND &hwndMain, int xPos, int yPos, int width, int height, TCHAR* className) { HINSTANCE hinst = GetModuleHandle(0); WNDCLASSEX wcx; wcx.hInstance = hinst; wcx.lpszClassName = className; wcx.lpfnWndProc = (WNDPROC) ViEAutoTestWinProc; wcx.style = CS_DBLCLKS; wcx.hIcon = LoadIcon(NULL, IDI_APPLICATION); wcx.hIconSm = LoadIcon(NULL, IDI_APPLICATION); wcx.hCursor = LoadCursor(NULL, IDC_ARROW); wcx.lpszMenuName = NULL; wcx.cbSize = sizeof(WNDCLASSEX); wcx.cbClsExtra = 0; wcx.cbWndExtra = 0; wcx.hbrBackground = GetSysColorBrush(COLOR_3DFACE); RegisterClassEx(&wcx); // Create the main window. hwndMain = CreateWindowEx(0, // no extended styles className, // class name className, // window name WS_OVERLAPPED | WS_THICKFRAME, // overlapped window xPos, // horizontal position yPos, // vertical position width, // width height, // height (HWND) NULL, // no parent or owner window (HMENU) NULL, // class menu used hinst, // instance handle NULL); // no window creation data if (!hwndMain) return -1; // Show the window using the flag specified by the program // that started the application, and send the application // a WM_PAINT message. ShowWindow(hwndMain, SW_SHOWDEFAULT); UpdateWindow(hwndMain); ::SetWindowPos(hwndMain, HWND_TOP, xPos, yPos, width, height, SWP_FRAMECHANGED); return 0; } LRESULT CALLBACK ViEAutoTestWinProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { switch (uMsg) { case WM_DESTROY: PostQuitMessage( WM_QUIT); break; case WM_COMMAND: break; } return DefWindowProc(hWnd, uMsg, wParam, lParam); } unsigned __stdcall ServiceRun(void* trans) //server { SOCKET localrecv=::socket(AF_INET,SOCK_DGRAM,IPPROTO_IP); sockaddr_in udpadd; udpadd.sin_addr.S_un.S_addr=inet_addr("127.0.0.1"); udpadd.sin_family=AF_INET; udpadd.sin_port=ntohs(8888); int rs = bind(localrecv,(sockaddr*)&udpadd,sizeof(udpadd)); if (rs < 0) { printf("bind err\n"); return 1; } char buf[1500]="\0"; sockaddr_in addrcl; int len=sizeof(sockaddr); my_transportationVideo* tran = (my_transportationVideo*) trans; while(true) { int ret = recvfrom(localrecv, buf, 1500, 0,(sockaddr*)&addrcl,&len);//用新建的地址结构保存客户端的信息 //printf("i recv rtp data len %d\n", ret); if (ret < 0) break; tran->SendRtpData(buf, ret); } closesocket(localrecv); WSACleanup(); printf("exit\n"); return 0; } unsigned __stdcall ServiceRtcpRun(void* trans) //server { SOCKET localrecv=::socket(AF_INET,SOCK_DGRAM,IPPROTO_IP); sockaddr_in udpadd; udpadd.sin_addr.S_un.S_addr=inet_addr("127.0.0.1"); udpadd.sin_family=AF_INET; udpadd.sin_port=ntohs(8889); int rs = bind(localrecv,(sockaddr*)&udpadd,sizeof(udpadd)); if (rs < 0) { printf("bind err\n"); return 1; } char buf[1500]="\0"; sockaddr_in addrcl; int len=sizeof(sockaddr); my_transportationVideo* tran = (my_transportationVideo*) trans; while(true) { int ret = recvfrom(localrecv, buf, 1500, 0,(sockaddr*)&addrcl,&len);//用新建的地址结构保存客户端的信息 printf("i recv rtcp data len %d\n", ret); if (ret < 0) break; tran->SendRtcpData(buf, ret); } closesocket(localrecv); WSACleanup(); printf("exit\n"); return 0; }