目录
一、前言
终于轮到ffmpeg的内容了。通过网络传输一部不是传裸数据,而是要经过编码后再进行传输,一方面节省带宽,另一方面一定程度提高通话可靠性。
ffmpeg是c库,需要通过jni接口进行调用。这里还是先将ffmpeg的接口封装成AudioEncoder和AudioDecoder,分别利用ffmpeg进行编解码。(之后有机会在将编码后的数据打包成rtp格式和从rtp中解析,以便进行可靠传输)
整体流程是:AudioRecoder采集到PCM数据后,将数据通过jni接口送给AudioEncoder进行编码,编码之后通过UdpIo进行发送;UdpIo监听到数据之后将数据通过jni接口送给AudioDecoder进行解码,解码之后得到的PCM数据送给AudioPlayer进行播放。
二、语音编码
AudioEncoder.h:
//
// Created by 13658 on 2024/10/24.
//
#ifndef FFMPEGTESTPRJ_AUDIOENCODER_H
#define FFMPEGTESTPRJ_AUDIOENCODER_H
extern "C"{//必须要添加该声明
#include "libavcodec/ac3_parser.h"
#include "libavcodec/adts_parser.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/avdct.h"
#include "libavcodec/avfft.h"
#include "libavcodec/bsf.h"
#include "libavcodec/codec_desc.h"
#include "libavcodec/codec_id.h"
#include "libavcodec/codec_par.h"
#include "libavcodec/codec.h"
//#include "libavcodec/d3d11va.h"
#include "libavcodec/defs.h"
#include "libavcodec/dirac.h"
#include "libavcodec/dv_profile.h"
//#include "libavcodec/dxva2.h"
#include "libavcodec/jni.h"
#include "libavcodec/mediacodec.h"
#include "libavcodec/packet.h"
//#include "libavcodec/qsv.h"
//#include "libavcodec/vdpau.h"
#include "libavcodec/version_major.h"
#include "libavcodec/version.h"
//#include "libavcodec/videotoolbox.h"
#include "libavcodec/vorbis_parser.h"
#include "libavdevice/avdevice.h"
#include "libavdevice/version_major.h"
#include "libavdevice/version.h"
#include "libavfilter/avfilter.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/version_major.h"
#include "libavfilter/version.h"
#include "libavformat/avformat.h"
#include "libavformat/avio.h"
#include "libavformat/version.h"
#include "libavformat/version_major.h"
#include "libavutil/adler32.h"
#include "libavutil/aes_ctr.h"
#include "libavutil/aes.h"
#include "libavutil/ambient_viewing_environment.h"
#include "libavutil/attributes.h"
#include "libavutil/audio_fifo.h"
#include "libavutil/avassert.h"
#include "libavutil/avconfig.h"
#include "libavutil/avstring.h"
#include "libavutil/avutil.h"
#include "libavutil/base64.h"
#include "libavutil/blowfish.h"
#include "libavutil/bprint.h"
#include "libavutil/bswap.h"
#include "libavutil/buffer.h"
#include "libavutil/camellia.h"
#include "libavutil/cast5.h"
#include "libavutil/channel_layout.h"
#include "libavutil/common.h"
#include "libavutil/cpu.h"
#include "libavutil/crc.h"
#include "libavutil/csp.h"
#include "libavutil/des.h"
#include "libavutil/detection_bbox.h"
#include "libavutil/dict.h"
#include "libavutil/display.h"
#include "libavutil/dovi_meta.h"
#include "libavutil/downmix_info.h"
#include "libavutil/encryption_info.h"
#include "libavutil/error.h"
#include "libavutil/eval.h"
#include "libavutil/executor.h"
#include "libavutil/ffversion.h"
#include "libavutil/fifo.h"
#include "libavutil/file.h"
#include "libavutil/film_grain_params.h"
#include "libavutil/frame.h"
#include "libavutil/hash.h"
#include "libavutil/hdr_dynamic_metadata.h"
#include "libavutil/hdr_dynamic_vivid_metadata.h"
#include "libavutil/hmac.h"
//#include "libavutil/hwcontext_cuda.h"
//#include "libavutil/hwcontext_d3d11va.h"
//#include "libavutil/hwcontext_d3d12va.h"
#include "libavutil/hwcontext_drm.h"
//#include "libavutil/hwcontext_dxva2.h"
#include "libavutil/hwcontext_mediacodec.h"
//#include "libavutil/hwcontext_opencl.h"
//#include "libavutil/hwcontext_qsv.h"
//#include "libavutil/hwcontext_vaapi.h"
//#include "libavutil/hwcontext_vdpau.h"
//#include "libavutil/hwcontext_videotoolbox.h"
#include "libavutil/hwcontext_vulkan.h"
#include "libavutil/hwcontext.h"
#include "libavutil/iamf.h"
#include "libavutil/imgutils.h"
#include "libavutil/intfloat.h"
#include "libavutil/intreadwrite.h"
#include "libavutil/lfg.h"
#include "libavutil/log.h"
#include "libavutil/lzo.h"
#include "libavutil/macros.h"
#include "libavutil/mastering_display_metadata.h"
#include "libavutil/mathematics.h"
#include "libavutil/md5.h"
#include "libavutil/mem.h"
#include "libavutil/motion_vector.h"
#include "libavutil/murmur3.h"
#include "libavutil/opt.h"
#include "libavutil/parseutils.h"
#include "libavutil/pixdesc.h"
#include "libavutil/pixelutils.h"
#include "libavutil/pixfmt.h"
#include "libavutil/random_seed.h"
#include "libavutil/rational.h"
#include "libavutil/rc4.h"
#include "libavutil/replaygain.h"
#include "libavutil/ripemd.h"
#include "libavutil/samplefmt.h"
#include "libavutil/sha.h"
#include "libavutil/sha512.h"
#include "libavutil/spherical.h"
#include "libavutil/stereo3d.h"
#include "libavutil/tea.h"
#include "libavutil/threadmessage.h"
#include "libavutil/time.h"
#include "libavutil/timecode.h"
#include "libavutil/timestamp.h"
#include "libavutil/tree.h"
#include "libavutil/twofish.h"
#include "libavutil/tx.h"
#include "libavutil/uuid.h"
#include "libavutil/version.h"
#include "libavutil/video_enc_params.h"
#include "libavutil/video_hint.h"
#include "libavutil/xtea.h"
#include "libpostproc/postprocess.h"
#include "libpostproc/version_major.h"
#include "libpostproc/version.h"
#include "libswresample/swresample.h"
#include "libswresample/version_major.h"
#include "libswresample/version.h"
#include "libswscale/swscale.h"
#include "libswscale/version_major.h"
#include "libswscale/version.h"
}
#include"AudioResampler.h"
namespace xpg {
class AudioEncoder {
enum class ae_return_code{
AE_OK = 0,
AE_FAIL
};
public:
AudioEncoder();
~AudioEncoder();
ae_return_code Init();
int Encode(uint8_t* dataIn, const int& bytesIn, std::function<void(uint8_t*, int)> cb);
private:
int Sint16ToFltp32(uint8_t* dataIn, int samplesIn, int nb_channels_in,
uint8_t* dataOut, int samplesOut,int nb_channels_out);
void Uninit();
private:
bool inited_ = false;
//AVFormatContext* format_context_ = nullptr;
const AVCodec* codec_ = nullptr;
AVCodecContext* codec_context_ = nullptr;
AVPacket* packet_ = nullptr;
AVFrame* frame_ = nullptr;
AudioResampler* resampler_ = nullptr;
FILE* input_file_ = nullptr;
FILE* output_file_ = nullptr;
};
} // xpg
#endif //FFMPEGTESTPRJ_AUDIOENCODER_H
AudioEncoder.cpp:
//
// Created by 13658 on 2024/10/24.
//
#include "AudioEncoder.h"
#include "../ffmpeg_utils/codec_checker.h"
namespace xpg {
static void get_adts_header(AVCodecContext *ctx, uint8_t *adts_header, int aac_length)
{
uint8_t freq_idx = 0; //0: 96000 Hz 3: 48000 Hz 4: 44100 Hz
switch (ctx->sample_rate) {
case 96000: freq_idx = 0; break;
case 88200: freq_idx = 1; break;
case 64000: freq_idx = 2; break;
case 48000: freq_idx = 3; break;
case 44100: freq_idx = 4; break;
case 32000: freq_idx = 5; break;
case 24000: freq_idx = 6; break;
case 22050: freq_idx = 7; break;
case 16000: freq_idx = 8; break;
case 12000: freq_idx = 9; break;
case 11025: freq_idx = 10; break;
case 8000: freq_idx = 11; break;
case 7350: freq_idx = 12; break;
default: freq_idx = 4; break;
}
uint8_t chanCfg = ctx->ch_layout.nb_channels;
uint32_t frame_length = aac_length + 7;
adts_header[0] = 0xFF;
adts_header[1] = 0xF1;
adts_header[2] = ((ctx->profile) << 6) + (freq_idx << 2) + (chanCfg >> 2);
adts_header[3] = (((chanCfg & 3) << 6) + (frame_length >> 11));
adts_header[4] = ((frame_length & 0x7FF) >> 3);
adts_header[5] = (((frame_length & 7) << 5) + 0x1F);
adts_header[6] = 0xFC;
}
AudioEncoder::AudioEncoder(){
}
AudioEncoder::~AudioEncoder(){
Uninit();
}
AudioEncoder::ae_return_code AudioEncoder::Init(){
codec_ = avcodec_find_encoder(AV_CODEC_ID_AAC);
if(!codec_){
return ae_return_code::AE_FAIL;
}
codec_context_ = avcodec_alloc_context3(codec_);
if(!codec_context_){
return ae_return_code::AE_FAIL;
}
// 设置编码参数
codec_context_->codec_type = AVMEDIA_TYPE_AUDIO;
codec_context_->bit_rate = 64000;
codec_context_->sample_fmt = AV_SAMPLE_FMT_FLTP;
codec_context_->sample_rate = 44100;
av_channel_layout_default(&codec_context_->ch_layout, 1);
//codec_context_->ch_layout.nb_channels = 2;
if (!check_sample_fmt(codec_, codec_context_->sample_fmt)) {
fprintf(stderr, "Encoder does not support sample format %s",
av_get_sample_fmt_name(codec_context_->sample_fmt));
return ae_return_code::AE_FAIL;
}
/* select other audio parameters supported by the encoder */
codec_context_->sample_rate = select_sample_rate(codec_);
int ret = select_channel_layout(codec_, &codec_context_->ch_layout);
if (ret < 0)
return ae_return_code::AE_FAIL;
av_opt_set(codec_context_->priv_data, "tune","zerolatency", 0);
ret = avcodec_open2(codec_context_, codec_, NULL);
if(0 != ret){
return ae_return_code::AE_FAIL;
}
av_opt_set(codec_context_->priv_data, "tune", "zerolatency", 0);
// 创建输入音频帧
frame_ = av_frame_alloc();
if(!frame_){
return ae_return_code::AE_FAIL;
}
frame_->nb_samples = codec_context_->frame_size;
frame_->format = codec_context_->sample_fmt;
frame_->ch_layout = codec_context_->ch_layout;
frame_->linesize[0] = frame_->nb_samples * frame_->ch_layout.nb_channels * av_get_bytes_per_sample(codec_context_->sample_fmt);
ret = av_frame_get_buffer(frame_, 0);
if (ret < 0) {
printf("Could not allocate audio data buffers\n");
return ae_return_code::AE_FAIL;
}
// 创建输出音频包
packet_ = av_packet_alloc();
if(!packet_){
return ae_return_code::AE_FAIL;
}
resampler_ = new AudioResampler();
xpg::AudioResampler::ResamplerParam param;
param.in_sample_rate = 44100;
param.out_sample_rate = 48000;
param.in_channels = 2;
param.out_channels = 1;
param.in_format = AV_SAMPLE_FMT_S16;
param.out_format = AV_SAMPLE_FMT_FLTP;
ret = resampler_->Init(param);
if(ret < 0){
return ae_return_code::AE_FAIL;
}
/*input_file_ = fopen("/storage/emulated/0/Download/inputencodePcm.pcm","wb");
if(!input_file_){
return ae_return_code::AE_FAIL;
}
output_file_ = fopen("/storage/emulated/0/Download/encodeAAC.aac","wb");
if(!output_file_){
return ae_return_code::AE_FAIL;
}*/
return ae_return_code::AE_OK;
}
int AudioEncoder::Encode(uint8_t* data_in, const int& bytes_in, std::function<void(uint8_t*, int)> cb){
if(!inited_){
if(AudioEncoder::ae_return_code::AE_OK != Init()){
Uninit();
return -1;
}
inited_ = true;
}
int ret = av_frame_make_writable(frame_);
if (ret < 0){
return -1;
}else{
//int outlen = 0;
//重采样需要变更采样率,变更采样率采样点会发生变化,需要重新组帧,这里暂时不做重采样,而是直接进行格式转换
//resampler_->Process(data_in, bytes_in,frame_->data[0],frame_->linesize[0]);
if(0 != Sint16ToFltp32(data_in, 1024, 1, frame_->data[0], frame_->linesize[0], 1)){
return -1;
}
}
//fwrite(data_in, 1, bytes_in, input_file_);
// 发送音频帧数据给编码器
ret = avcodec_send_frame(codec_context_, frame_);
if (ret < 0) {
fprintf(stderr, "Failed to send frame for encoding\n");
return -1;
}
// 接收编码后的音频帧
while (ret >= 0) {
ret = avcodec_receive_packet(codec_context_, packet_);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return 0;
else if (ret < 0) {
printf("Error encoding audio frame\n");
return -1;
}
// 现在,frame中存储了解码后的音频帧,可以进行后续处理或播放
// 添加adts头,想直接在手机听音可以使用该功能
/*memmove(packet_->data+7, packet_->data, packet_->size);
get_adts_header(codec_context_, packet_->data,packet_->size);
packet_->size += 7;*/
uint8_t tmpbuf[1500] = {0};
xpg::get_adts_header(codec_context_, tmpbuf, packet_->size);
memcpy(tmpbuf+7, packet_->data, packet_->size);
//fwrite(tmpbuf, 1, packet_->size + 7, output_file_);
//输出
cb(packet_->data, packet_->size);
av_packet_unref(packet_);
}
return 0;
}
int AudioEncoder::Sint16ToFltp32(uint8_t* dataIn, int samplesIn, int nb_channels_in,
uint8_t* dataOut, int samplesOut,int nb_channels_out){
if(samplesIn != samplesOut){
//return -1;
}
int16_t* dataInS16 = (int16_t*)dataIn;
float_t* dataOutf32 = (float*)dataOut;
for(int i = 0; i < samplesIn; i++){
dataOutf32[i] = (float_t)dataInS16[2*i] / 32768.0f;
//dataOutf32[samplesIn + i] = (int16_t)dataInS16[2*i + 1];
}
return 0;
}
void AudioEncoder::Uninit(){
// 释放资源
if(frame_) av_frame_free(&frame_);
if(packet_) av_packet_free(&packet_);
if(codec_context_) avcodec_free_context(&codec_context_);
//if(format_context_ && format_context_->pb) avio_close(format_context_->pb);
//if(format_context_) avformat_free_context(format_context_);
if(resampler_){
delete resampler_;
resampler_ = nullptr;
}
if(output_file_){
fflush(output_file_);
fclose(output_file_);
output_file_ = nullptr;
}
if(input_file_){
fflush(input_file_);
fclose(input_file_);
input_file_ = nullptr;
}
inited_ = false;
}
} // xpg
三、语音解码
AudioDecoer.h:
//
// Created by 13658 on 2024/10/24.
//
#ifndef FFMPEGTESTPRJ_AUDIODECODER_H
#define FFMPEGTESTPRJ_AUDIODECODER_H
extern "C"{//必须要添加该声明
#include "libavcodec/ac3_parser.h"
#include "libavcodec/adts_parser.h"
#include "libavcodec/avcodec.h"
#include "libavcodec/avdct.h"
#include "libavcodec/avfft.h"
#include "libavcodec/bsf.h"
#include "libavcodec/codec_desc.h"
#include "libavcodec/codec_id.h"
#include "libavcodec/codec_par.h"
#include "libavcodec/codec.h"
//#include "libavcodec/d3d11va.h"
#include "libavcodec/defs.h"
#include "libavcodec/dirac.h"
#include "libavcodec/dv_profile.h"
//#include "libavcodec/dxva2.h"
#include "libavcodec/jni.h"
#include "libavcodec/mediacodec.h"
#include "libavcodec/packet.h"
//#include "libavcodec/qsv.h"
//#include "libavcodec/vdpau.h"
#include "libavcodec/version_major.h"
#include "libavcodec/version.h"
//#include "libavcodec/videotoolbox.h"
#include "libavcodec/vorbis_parser.h"
#include "libavdevice/avdevice.h"
#include "libavdevice/version_major.h"
#include "libavdevice/version.h"
#include "libavfilter/avfilter.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include "libavfilter/version_major.h"
#include "libavfilter/version.h"
#include "libavformat/avformat.h"
#include "libavformat/avio.h"
#include "libavformat/version.h"
#include "libavformat/version_major.h"
#include "libavutil/adler32.h"
#include "libavutil/aes_ctr.h"
#include "libavutil/aes.h"
#include "libavutil/ambient_viewing_environment.h"
#include "libavutil/attributes.h"
#include "libavutil/audio_fifo.h"
#include "libavutil/avassert.h"
#include "libavutil/avconfig.h"
#include "libavutil/avstring.h"
#include "libavutil/avutil.h"
#include "libavutil/base64.h"
#include "libavutil/blowfish.h"
#include "libavutil/bprint.h"
#include "libavutil/bswap.h"
#include "libavutil/buffer.h"
#include "libavutil/camellia.h"
#include "libavutil/cast5.h"
#include "libavutil/channel_layout.h"
#include "libavutil/common.h"
#include "libavutil/cpu.h"
#include "libavutil/crc.h"
#include "libavutil/csp.h"
#include "libavutil/des.h"
#include "libavutil/detection_bbox.h"
#include "libavutil/dict.h"
#include "libavutil/display.h"
#include "libavutil/dovi_meta.h"
#include "libavutil/downmix_info.h"
#include "libavutil/encryption_info.h"
#include "libavutil/error.h"
#include "libavutil/eval.h"
#include "libavutil/executor.h"
#include "libavutil/ffversion.h"
#include "libavutil/fifo.h"
#include "libavutil/file.h"
#include "libavutil/film_grain_params.h"
#include "libavutil/frame.h"
#include "libavutil/hash.h"
#include "libavutil/hdr_dynamic_metadata.h"
#include "libavutil/hdr_dynamic_vivid_metadata.h"
#include "libavutil/hmac.h"
//#include "libavutil/hwcontext_cuda.h"
//#include "libavutil/hwcontext_d3d11va.h"
//#include "libavutil/hwcontext_d3d12va.h"
#include "libavutil/hwcontext_drm.h"
//#include "libavutil/hwcontext_dxva2.h"
#include "libavutil/hwcontext_mediacodec.h"
//#include "libavutil/hwcontext_opencl.h"
//#include "libavutil/hwcontext_qsv.h"
//#include "libavutil/hwcontext_vaapi.h"
//#include "libavutil/hwcontext_vdpau.h"
//#include "libavutil/hwcontext_videotoolbox.h"
#include "libavutil/hwcontext_vulkan.h"
#include "libavutil/hwcontext.h"
#include "libavutil/iamf.h"
#include "libavutil/imgutils.h"
#include "libavutil/intfloat.h"
#include "libavutil/intreadwrite.h"
#include "libavutil/lfg.h"
#include "libavutil/log.h"
#include "libavutil/lzo.h"
#include "libavutil/macros.h"
#include "libavutil/mastering_display_metadata.h"
#include "libavutil/mathematics.h"
#include "libavutil/md5.h"
#include "libavutil/mem.h"
#include "libavutil/motion_vector.h"
#include "libavutil/murmur3.h"
#include "libavutil/opt.h"
#include "libavutil/parseutils.h"
#include "libavutil/pixdesc.h"
#include "libavutil/pixelutils.h"
#include "libavutil/pixfmt.h"
#include "libavutil/random_seed.h"
#include "libavutil/rational.h"
#include "libavutil/rc4.h"
#include "libavutil/replaygain.h"
#include "libavutil/ripemd.h"
#include "libavutil/samplefmt.h"
#include "libavutil/sha.h"
#include "libavutil/sha512.h"
#include "libavutil/spherical.h"
#include "libavutil/stereo3d.h"
#include "libavutil/tea.h"
#include "libavutil/threadmessage.h"
#include "libavutil/time.h"
#include "libavutil/timecode.h"
#include "libavutil/timestamp.h"
#include "libavutil/tree.h"
#include "libavutil/twofish.h"
#include "libavutil/tx.h"
#include "libavutil/uuid.h"
#include "libavutil/version.h"
#include "libavutil/video_enc_params.h"
#include "libavutil/video_hint.h"
#include "libavutil/xtea.h"
#include "libpostproc/postprocess.h"
#include "libpostproc/version_major.h"
#include "libpostproc/version.h"
#include "libswresample/swresample.h"
#include "libswresample/version_major.h"
#include "libswresample/version.h"
#include "libswscale/swscale.h"
#include "libswscale/version_major.h"
#include "libswscale/version.h"
}
#include"AudioResampler.h"
namespace xpg {
class AudioDecoder {
enum class ad_return_code{
AE_OK = 0,
AE_FAIL
};
public:
AudioDecoder();
~AudioDecoder();
ad_return_code Init();
int Decode(uint8_t* dataIn, const int& bytesIn, std::function<void(uint8_t*, int)> cb);
private:
int Fltp32ToSint16(uint8_t* dataIn, int samplesIn, int nb_channels_in,
uint8_t* dataOut, int samplesOut,int nb_channels_out);
void Uninit();
private:
bool inited_ = false;
//AVFormatContext* format_context_ = nullptr;
const AVCodec* codec_ = nullptr;
AVCodecContext* codec_context_ = nullptr;
AVCodecParserContext* parser_ = nullptr;
AVPacket* packet_ = nullptr;
AVFrame* frame_ = nullptr;
AudioResampler* resampler_ = nullptr;
FILE* decodeInFile_ = nullptr;
FILE* decodeOutFile_ = nullptr;
};
} // xpg
#endif //FFMPEGTESTPRJ_AUDIODECODER_H
AudioDecoder.cpp:
//
// Created by 13658 on 2024/10/24.
//
#include "AudioDecoder.h"
#include"../ffmpeg_utils/codec_checker.h"
namespace xpg {
static void get_adts_header(AVCodecContext *ctx, uint8_t *adts_header, int aac_length)
{
uint8_t freq_idx = 0; //0: 96000 Hz 3: 48000 Hz 4: 44100 Hz
switch (ctx->sample_rate) {
case 96000: freq_idx = 0; break;
case 88200: freq_idx = 1; break;
case 64000: freq_idx = 2; break;
case 48000: freq_idx = 3; break;
case 44100: freq_idx = 4; break;
case 32000: freq_idx = 5; break;
case 24000: freq_idx = 6; break;
case 22050: freq_idx = 7; break;
case 16000: freq_idx = 8; break;
case 12000: freq_idx = 9; break;
case 11025: freq_idx = 10; break;
case 8000: freq_idx = 11; break;
case 7350: freq_idx = 12; break;
default: freq_idx = 4; break;
}
uint8_t chanCfg = ctx->ch_layout.nb_channels;
uint32_t frame_length = aac_length + 7;
adts_header[0] = 0xFF;
adts_header[1] = 0xF1;
adts_header[2] = ((ctx->profile) << 6) + (freq_idx << 2) + (chanCfg >> 2);
adts_header[3] = (((chanCfg & 3) << 6) + (frame_length >> 11));
adts_header[4] = ((frame_length & 0x7FF) >> 3);
adts_header[5] = (((frame_length & 7) << 5) + 0x1F);
adts_header[6] = 0xFC;
}
AudioDecoder::AudioDecoder(){
}
AudioDecoder::~AudioDecoder(){
Uninit();
}
AudioDecoder::ad_return_code AudioDecoder::Init(){
codec_ = avcodec_find_decoder(AV_CODEC_ID_AAC);
if(!codec_){
return ad_return_code::AE_FAIL;
}
parser_ = av_parser_init(codec_->id);
if (!parser_) {
fprintf(stderr, "Parser not found\n");
return ad_return_code::AE_FAIL;;
}
codec_context_ = avcodec_alloc_context3(codec_);
if(!codec_context_){
return ad_return_code::AE_FAIL;
}
// 设置编码参数
codec_context_->codec_type = AVMEDIA_TYPE_AUDIO;
codec_context_->bit_rate = 64000;
codec_context_->sample_fmt = AV_SAMPLE_FMT_FLTP;
codec_context_->sample_rate = 44100;
av_channel_layout_default(&codec_context_->ch_layout, 1);
//codec_context_->ch_layout.nb_channels = 2;
if (!check_sample_fmt(codec_, codec_context_->sample_fmt)) {
fprintf(stderr, "Encoder does not support sample format %s",
av_get_sample_fmt_name(codec_context_->sample_fmt));
return ad_return_code::AE_FAIL;
}
/* select other audio parameters supported by the encoder */
codec_context_->sample_rate = 44100;//select_sample_rate(codec_);
/*int ret = select_channel_layout(codec_, &codec_context_->ch_layout);
if (ret < 0)
return ad_return_code::AE_FAIL;*/
//av_opt_set(codec_context_->priv_data, "tune", "zerolatency", 0);
int ret = avcodec_open2(codec_context_, codec_, NULL);
if(0 != ret){
Uninit();
return ad_return_code::AE_FAIL;
}
//av_opt_set(codec_context_->priv_data, "tune", "zerolatency", 0);
// 创建输入音频帧
packet_ = av_packet_alloc();
if(!packet_){
return ad_return_code::AE_FAIL;
}
// 创建AVFrame对象
frame_ = av_frame_alloc();
if(!frame_){
return ad_return_code::AE_FAIL;
}
/*decodeInFile_ = fopen("/storage/emulated/0/Download/decodeAAC.aac", "wb");
if(!decodeInFile_){
return ad_return_code::AE_FAIL;
}
decodeOutFile_ = fopen("/storage/emulated/0/Download/decodeOutPcm.pcm", "wb");
if(!decodeOutFile_){
return ad_return_code::AE_FAIL;
}*/
return ad_return_code::AE_OK;
}
int AudioDecoder::Decode(uint8_t* data_in, const int& bytes_in, std::function<void(uint8_t*, int)> cb){
if(!inited_){
if(AudioDecoder::ad_return_code::AE_OK != Init()){
Uninit();
return -1;
}
inited_ = true;
}
uint8_t tmpbuf[1500] = {0};
xpg::get_adts_header(codec_context_, tmpbuf, bytes_in);
memcpy(tmpbuf+7, data_in, bytes_in);
//fwrite(tmpbuf, 1, bytes_in + 7, decodeInFile_);
int ret = av_parser_parse2(parser_, codec_context_, &packet_->data, &packet_->size,
data_in, bytes_in,
AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0);
if (ret < 0) {
fprintf(stderr, "Error while parsing\n");
return -1;
}
if(!packet_->size){
return 0;
}
/* send the packet with the compressed data to the decoder */
ret = avcodec_send_packet(codec_context_, packet_);
if (ret < 0) {
fprintf(stderr, "Error submitting the packet to the decoder\n");
return -1;
}
/* read all the output frames (in general there may be any number of them */
while (ret >= 0) {
ret = avcodec_receive_frame(codec_context_, frame_);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
return 0;
else if (ret < 0) {
return 0;
}
int data_size = av_get_bytes_per_sample(codec_context_->sample_fmt);
if (data_size < 0) {
/* This should not occur, checking just for paranoia */
fprintf(stderr, "Failed to calculate data size\n");
return -1;
}
// 现在,frame中存储了解码后的音频帧,可以进行后续处理或播放
//调用回调函数将数据输出
int16_t tmp_buf_len = frame_->linesize[0]/2;
uint8_t* tmpbuf = new uint8_t[tmp_buf_len];
Fltp32ToSint16(frame_->data[0],frame_->linesize[0]/4, 1, tmpbuf, frame_->linesize[0]/2, 1);
//fwrite(frame_->data[0], 1, frame_->linesize[0]/2, decodeOutFile_);
//fwrite(tmpbuf, 1, tmp_buf_len, decodeOutFile_);
cb(tmpbuf, tmp_buf_len);
/*for (i = 0; i < frame->nb_samples; i++)
for (ch = 0; ch < dec_ctx->ch_layout.nb_channels; ch++)
fwrite(frame->data[ch] + data_size*i, 1, data_size, outfile);*/
delete[] tmpbuf;
tmpbuf = nullptr;
}
return 0;
}
int AudioDecoder::Fltp32ToSint16(uint8_t* dataIn, int samplesIn, int nb_channels_in,
uint8_t* dataOut, int samplesOut,int nb_channels_out){
if(samplesIn != samplesOut){
//return -1;
}
float_t* dataOutf32 = (float*)dataIn;
int16_t* dataInS16 = (int16_t*)dataOut;
for(int i = 0; i < samplesIn; i++){
dataInS16[2*i] = (int16_t)(dataOutf32[i] * 32768.0f);
dataInS16[2*i+1] = (int16_t)(dataOutf32[i] * 32768.0f);
}
return 0;
}
void AudioDecoder::Uninit(){
// 释放资源
if(parser_)av_parser_close(parser_);
if(frame_) av_frame_free(&frame_);
if(packet_) av_packet_free(&packet_);
if(codec_context_) avcodec_free_context(&codec_context_);
if(decodeInFile_){
fflush(decodeInFile_);
fclose(decodeInFile_);
}
if(decodeOutFile_){
fflush(decodeOutFile_);
fclose(decodeOutFile_);
}
//if(format_context_ && format_context_->pb) avio_close(format_context_->pb);
//if(format_context_) avformat_free_context(format_context_);
}
} // xpg
四、jni接口
#include <jni.h>
#include <string>
#include "codec/AudioDecoder.h"
#include "codec/AudioEncoder.h"
#include "codec/AudioResampler.h"
#include <functional>
#include <vector>
#include "demo/audio_encode_demo.h"
#include "demo/audio_decode_demo.h"
using namespace xpg;
class FileHandle{
public:
FileHandle(std::string file_name):file_name_(file_name){
}
~FileHandle(){
if(file_){
fflush(file_);
fclose(file_);
file_ = nullptr;
}
}
void writePcm(uint8_t* data, int len){
if(!file_){
file_ = fopen(std::string("/storage/emulated/0/Download/" + file_name_).c_str(),"wb");
}
fwrite(data, 1, len, file_);
}
int readPcm(uint8_t* data, int len){
if(!file_){
std::string fn = "/storage/emulated/0/Download/" + file_name_;
file_ = fopen(fn.c_str(), "rb");
if(!file_){
return 0;
}
}
if(feof(file_)){
return 0;
}
return fread(data, 1, len, file_);
}
private:
FILE* file_ = nullptr;
std::string file_name_;
};
AudioEncoder audioEncoder;
AudioDecoder audioDecoder;
FileHandle g_fileIn{"nativePcmIn.pcm"};
FileHandle g_fileOut{"nativePcmOut.pcm"};
FileHandle g_fileRead{"nativePcmIn.pcm"};
extern "C" JNIEXPORT jbyteArray JNICALL
Java_com_example_ffmpegtestprj_MainActivity_encodeAudio(JNIEnv* env,
jobject instance,
jbyteArray javaArrayIn,
jint arrayLenIn
) {
jbyteArray result = nullptr;
jbyte *bytesIn = (*env).GetByteArrayElements(javaArrayIn, NULL);
//g_fileIn.writePcm((uint8_t*)bytesIn, arrayLenIn);
if(0 == audioEncoder.Encode((uint8_t*)bytesIn, arrayLenIn, [&result, &env](uint8_t* dataOut, int dataLen){
result = (*env).NewByteArray(dataLen);
jbyte *bytes = (*env).GetByteArrayElements(result, NULL);
memcpy(bytes, dataOut, dataLen);
(*env).ReleaseByteArrayElements(result, bytes, 0);
})){
return result;
}else{
return nullptr;
}
}
extern "C" JNIEXPORT jbyteArray JNICALL
Java_com_example_ffmpegtestprj_MainActivity_decodeAudio(JNIEnv* env,
jobject instance,
jbyteArray javaArrayIn,
jint arrayLenIn
) {
jbyteArray result = nullptr;
jbyte *bytesIn = (*env).GetByteArrayElements(javaArrayIn, NULL);
if(0 == audioDecoder.Decode((uint8_t*)bytesIn, arrayLenIn, [&result, &env](uint8_t* dataOut, int dataLen){
result = (*env).NewByteArray(dataLen);
jbyte *bytes = (*env).GetByteArrayElements(result, NULL);
//g_fileOut.writePcm((uint8_t*)dataOut, dataLen);
memcpy(bytes, dataOut, dataLen);
(*env).ReleaseByteArrayElements(result, bytes, 0);
})){
return result;
}else{
return nullptr;
}
/*jbyteArray result = (*env).NewByteArray(4096);
jbyte *bytes = (*env).GetByteArrayElements(result, NULL);
int read_len = g_fileRead.readPcm((uint8_t*)bytes, 4096);
if(read_len < 4096){
return nullptr;
}
(*env).ReleaseByteArrayElements(result, bytes, 0);
return result;*/
}
extern "C" JNIEXPORT jint JNICALL
Java_com_example_ffmpegtestprj_MainActivity_encodeAudioTest(JNIEnv* env,
jobject instance
){
/*const char *dirPath = env->GetStringUTFChars("/sdcard/Android/data/ywd/encodeOut.aac", nullptr);
int fd = open(dirPath, flags, 0666);
//若所有欲核查的权限都通过了检查则返回 0值,表示成功,
//只要有一个权限被禁止则返回-1。
if (fd == -1){
return -1;
}*/
std::vector<std::string> param;
param.push_back("/sdcard/Android/data/ywd/encodeBin");
param.push_back("/storage/emulated/0/Download/encodeOut.aac");
return audio_encode_demo::encode_demo_main(param);
}
extern "C" JNIEXPORT jint JNICALL
Java_com_example_ffmpegtestprj_MainActivity_decodeAudioTest(JNIEnv* env,
jobject instance
){
std::vector<std::string> param;
param.push_back("decodeBin");
param.push_back("/storage/emulated/0/Download/encodeOut.aac");
param.push_back("/storage/emulated/0/Download/decodeOut.pcm");
return audio_decode_demo::decode_demo_main(param);
}
五、修改MainActivity
这里主要贴MainActivity,AudioRecorder、AudioPlayer等也要做适当修改。
MainActivity:
package com.example.ffmpegtestprj;
import androidx.appcompat.app.AlertDialog;
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioTrack;
import android.content.pm.PackageManager;
import android.os.Bundle;
import android.os.Environment;
import android.widget.Button;
import android.widget.EditText;
import android.widget.TextView;
import android.view.View;
import com.example.ffmpegtestprj.databinding.ActivityMainBinding;
import java.io.File;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.sql.Struct;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.locks.ReentrantLock;
public class MainActivity extends AppCompatActivity implements View.OnClickListener{
// Used to load the 'ffmpegtestprj' library on application startup.
static {
System.loadLibrary("ffmpegtestprj");
}
/**
* A native method that is implemented by the 'ffmpegtestprj' native library,
* which is packaged with this application.
*/
public native byte[] encodeAudio(byte[] dataIn, int bytesSizeIn);
public native byte[] decodeAudio(byte[] dataIn, int bytesSizeIn);
public native int encodeAudioTest();
public native int decodeAudioTest();
private ActivityMainBinding binding;
private AudioRecorder audioRecorder;
private AudioPlayer audioPlayer;
private UdpIo udpIo;
private boolean calling = false;
EditText ipText;
private String remoteIp = "127.0.0.1";
private ReentrantLock queueLock = new ReentrantLock();
private Queue<byte[]> pcmQueue = new LinkedList<>();
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
binding = ActivityMainBinding.inflate(getLayoutInflater());
setContentView(binding.getRoot());
initDevice();
initView();
}
private void initDevice(){
audioRecorder = new AudioRecorder();
audioPlayer = new AudioPlayer();
udpIo = new UdpIo();
}
private void initView(){
//初始化控件
Button btnDecode = (Button) findViewById(R.id.btnGetPcm);
Button btnPlay = (Button) findViewById(R.id.btnPlayPcm);
Button btnCall = (Button) findViewById(R.id.btnCall);
Button btnHangUp = (Button) findViewById(R.id.btnHangUp);
Button btnEncodeTest = (Button) findViewById(R.id.btnEncode);
Button btnDecodeTest = (Button) findViewById(R.id.btnDecode);
//注册按钮事件监听器
btnDecode.setOnClickListener(this);
btnPlay.setOnClickListener(this);
btnCall.setOnClickListener(this);
btnHangUp.setOnClickListener(this);
btnEncodeTest.setOnClickListener(this);
btnDecodeTest.setOnClickListener(this);
}
@Override
public void onClick(View view) {
if(view.getId() == R.id.btnCall){
ipText = findViewById(R.id.editText);
remoteIp = ipText.getText().toString();
startCall(remoteIp, 9999);
}
if(view.getId() == R.id.btnHangUp){
stopCall();
}
if(view.getId() == R.id.btnEncode){
/*String directory = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS).toString();
File f = new File("/storage/emulated/0/Download/testywd.txt");
if(!f.exists()){
f.mkdirs();
}*/
encodeTest();
}
if(view.getId() == R.id.btnDecode){
decodeTest();
}
}
private void startCall(String ip, int port){
if(calling == true) {
return;
}
calling = true;
audioPlayer.start();
udpIo.startSending(ip, port);
udpIo.startRecving(port);
audioRecorder.start();
new Thread(new Runnable() {
@Override
public void run() {
final int bufferSize = AudioRecorder.BUFFER_SIZE;
try {
while (calling) {
//1、采集
byte[] pcmRecord = audioRecorder.getFromRecorderBuffer();
byte[] audioDataEncoded = null;
if(pcmRecord != null){
//2、编码
audioDataEncoded = encodeAudio(pcmRecord, pcmRecord.length);
}
//3、获取编码数据
if(null != audioDataEncoded){
//4、发送
udpIo.writeToSendQueue(audioDataEncoded);
}
Thread.sleep(23);//44100hz, 1024个采样点的时间
}
} catch (Exception e) {
e.printStackTrace();
}
}
}).start();
new Thread(new Runnable() {
@Override
public void run() {
final int bufferSize = AudioRecorder.BUFFER_SIZE;
try {
while (calling) {
//1、接收
byte[] dataRecv = udpIo.getFromRecvQueue();
byte[] pcmData = null;
if(dataRecv != null){
//2、解码
pcmData = decodeAudio(dataRecv, dataRecv.length);
}
//3、获取解码数据
if(pcmData != null){
//4、播放
audioPlayer.writeToRecorderBuffer(pcmData);
}
Thread.sleep(23);//44100hz, 1024个采样点的时间
}
} catch (Exception e) {
e.printStackTrace();
}
}
}).start();
}
private void stopCall(){
if(calling == true) {
calling = false;
audioRecorder.stop();
udpIo.stopRecving();
udpIo.stopSending();
audioPlayer.stop();
}
}
private void encodeTest(){
int ret = encodeAudioTest();
showDialog(Integer.toString(ret));
}
private void decodeTest(){
int ret = decodeAudioTest();
showDialog(Integer.toString(ret));
}
private void showDialog(String message){
AlertDialog.Builder dialog = new AlertDialog.Builder(MainActivity.this);
dialog.setTitle("HINT!!!");
dialog.setMessage(message);
dialog.show();
}
}
AudioRecorder:
package com.example.ffmpegtestprj;
import android.content.pm.PackageManager;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaRecorder;
import android.util.Log;
import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import java.io.IOException;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.locks.ReentrantLock;
public class AudioRecorder {
private static final String TAG = "AudioCapture";
private static final int PERMISSION_REQUEST_CODE = 200;
private static final int SAMPLE_RATE = 44100;
private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
private static final int AUDIO_FORMAT = AudioFormat.ENCODING_PCM_16BIT;
public static final int BUFFER_SIZE = AudioRecord.getMinBufferSize(
SAMPLE_RATE,
CHANNEL_CONFIG,
AUDIO_FORMAT);
private ReentrantLock pcmQueueLock = new ReentrantLock();
private Queue<byte[]> pcmQueue = new LinkedList<>();
private boolean recording = false;
private AudioRecord audioRecord;
byte[] recordBuffer = null;
int curBufferSize = 0;
private int BytesPerSample(int format){
switch (format){
case AudioFormat.ENCODING_PCM_16BIT:
return 2;
default:
return 2;
}
}
private int Channels(int channelType){
switch (channelType){
case AudioFormat.CHANNEL_IN_STEREO:
return 2;
default:
return 2;
}
}
private void initRecording(boolean isAAC) {
int sampleNum = 20 * SAMPLE_RATE / 1000;
if(isAAC){
sampleNum = 1024;
}
recordBuffer = new byte[sampleNum * BytesPerSample(AUDIO_FORMAT) * Channels(AUDIO_FORMAT)];
audioRecord = new AudioRecord(
MediaRecorder.AudioSource.MIC,
SAMPLE_RATE,
CHANNEL_CONFIG,
AUDIO_FORMAT,
BUFFER_SIZE);
if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) {
Log.e(TAG, "音频录制器初始化失败");
return;
}
}
private void startRecording() {
audioRecord.startRecording();
}
public void start(){
if(!recording){
recording = true;
}else{
return;
}
initRecording(true);
startRecording();
new Thread(new Runnable() {
@Override
public void run() {
//final int bufferSize = AudioRecorder.BUFFER_SIZE;
try {
while (recording) {
int bytesRead = audioRecord.read(recordBuffer, curBufferSize, recordBuffer.length - curBufferSize);
if(bytesRead > 0){
if(bytesRead == recordBuffer.length - curBufferSize){
//将数据写到缓存
writeToRecorderBuffer(recordBuffer, recordBuffer.length);
curBufferSize = 0;
}else{
curBufferSize += bytesRead;
}
}
//睡眠10ms
//Thread.sleep(10);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}).start();
}
public void stop() {
if(recording){
recording = false;
audioRecord.stop();
audioRecord.release();
pcmQueue.clear();
}
}
private void writeToRecorderBuffer(byte[] buffer, int size){
/*byte[] pcmData = new byte[size];
System.arraycopy(buffer, 0, pcmData, 0, size);*/
//锁
pcmQueueLock.lock();
pcmQueue.offer(buffer);
pcmQueueLock.unlock();
}
public byte[] getFromRecorderBuffer() {
//锁
pcmQueueLock.lock();
if(!pcmQueue.isEmpty()){
byte[] pcmData = pcmQueue.poll();
pcmQueueLock.unlock();
return pcmData;
}
pcmQueueLock.unlock();
return null;
}
}
AudioPlayer:
package com.example.ffmpegtestprj;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioTrack;
import android.media.MediaRecorder;
import java.io.IOException;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.util.LinkedList;
import java.util.Queue;
import java.util.concurrent.locks.ReentrantLock;
public class AudioPlayer {
private boolean inited = false;
private boolean playing = false;
private ReentrantLock pcmQueueLock = new ReentrantLock();
private Queue<byte[]> pcmQueue = new LinkedList<>();
private int sampleRateInHz = 44100;
private int channelConfig = AudioFormat.CHANNEL_OUT_STEREO;
private int audioFormat = AudioFormat.ENCODING_PCM_16BIT;
private int bufferSizeInBytes = AudioTrack.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat);
private AudioTrack audioTrack;
private void init() {
audioTrack = new AudioTrack(
AudioManager.STREAM_MUSIC, // 指定音频流类型
sampleRateInHz, // 采样率,例如44100Hz
channelConfig, // 声道配置,例如AudioFormat.CHANNEL_OUT_MONO
audioFormat, // 数据格式,例如AudioFormat.ENCODING_PCM_16BIT
bufferSizeInBytes, // 缓冲区大小,例如1024字节
AudioTrack.MODE_STREAM // 播放模式,使用流模式**
);
}
public void start() {
if(!inited){
inited = true;
init();
}
if (audioTrack.getState() != AudioTrack.STATE_UNINITIALIZED) {
audioTrack.play();
}
if(!playing){
playing = true;
}else{
return;
}
new Thread(new Runnable() {
@Override
public void run() {
while (playing) {
try {
byte[] pcmData = getFromRecorderBuffer();
if (pcmData != null) {
play(pcmData, pcmData.length);
}
//Thread.sleep(10);//睡眠10ms
} catch (Exception e) {
e.printStackTrace();
}
}
}
}).start();
}
public void stop() {
playing = false;
try {
if (audioTrack.getState() != AudioTrack.STATE_UNINITIALIZED) {
audioTrack.stop();
audioTrack.release();
pcmQueue.clear();
inited = false;
}
} catch (IllegalStateException e) {
e.printStackTrace();
}
}
public void writeToRecorderBuffer(byte[] buffer){
/*byte[] pcmData = new byte[size];
System.arraycopy(buffer, 0, pcmData, 0, size);*/
//锁
pcmQueueLock.lock();
pcmQueue.offer(buffer);
pcmQueueLock.unlock();
}
private byte[] getFromRecorderBuffer() {
//锁
pcmQueueLock.lock();
if(!pcmQueue.isEmpty()){
byte[] pcmData = pcmQueue.poll();
pcmQueueLock.unlock();
return pcmData;
}
pcmQueueLock.unlock();
return null;
}
private void play(byte[] buffer, int size){
if(size > 0 ) {
audioTrack.write(buffer, 0, size);
}
}
}