IJKPLAYER源码分析-音视频同步

1 前言

    本文来介绍下IJKPLAYER的音视频同步策略及方法。IJKPLAYER所支持的同步策略，源自FFPLAY，因此有以下3种：

• 视频同步音频：这是缺省的同步策略，以audio为主时钟-参考时钟，采取video同步audio的方式；
• 音频同步视频：这是较视频同步音频更为麻烦的同步策略，以video为主时钟-参考时钟，采取audio同步video的方法；
• 同步外部时钟：音视频均以外部时钟，为参考时钟；

2 视频同步音频

    这是IJKPLAYER所支持的缺省同步策略，以audio为主时钟-参考时钟，采取以video同步audio的方法进行。简单起见，此处不考虑不同serial的frame，下文仅讨论同1个serial的frame的音视频同步。

2.1 显示主流程

    video_refresh_thread实现： 

static int video_refresh_thread(void *arg)
{
    FFPlayer *ffp = arg;
    VideoState *is = ffp->is;
    double remaining_time = 0.0;
    while (!is->abort_request) {
        if (remaining_time > 0.0)
            av_usleep((uint)(uint64_t)(remaining_time * 1000000.0));
        remaining_time = REFRESH_RATE;
        if (ffp->cover_after_prepared && !ffp->first_video_frame_rendered) {
            is->force_refresh = true;
        }
#if ANDROID
        if (is->paused) {
            SDL_Delay(1000/24);
            is->force_refresh = true;
        }
#endif
        if (is->show_mode != SHOW_MODE_NONE && (!is->paused || is->force_refresh))
            video_refresh(ffp, &remaining_time);
    }
    if (ffp->vout)
        SDL_VoutFreeContext(ffp->vout);
    return 0;
}

    video_refresh方法保留显示、同步等主要逻辑： 

/* called to display each frame */
static void video_refresh(FFPlayer *opaque, double *remaining_time)
{
    FFPlayer *ffp = opaque;
    VideoState *is = ffp->is;
    double time;

    Frame *sp, *sp2;

    if (!is->paused && get_master_sync_type(is) == AV_SYNC_EXTERNAL_CLOCK && is->realtime)
        check_external_clock_speed(is);

    if (!ffp->display_disable && is->show_mode != SHOW_MODE_VIDEO && is->audio_st) {
        time = av_gettime_relative() / 1000000.0;
        if (is->force_refresh || is->last_vis_time + ffp->rdftspeed < time) {
            video_display2(ffp);
            is->last_vis_time = time;
        }
        *remaining_time = FFMIN(*remaining_time, is->last_vis_time + ffp->rdftspeed - time);
    }

    if (is->video_st) {
retry:
        if (frame_queue_nb_remaining(&is->pictq) == 0) {
            // nothing to do, no picture to display in the queue
        } else {
            double last_duration, duration, delay;
            Frame *vp, *lastvp;

            /* dequeue the picture */
            lastvp = frame_queue_peek_last(&is->pictq);
            vp = frame_queue_peek(&is->pictq);

            if (vp->serial != is->videoq.serial) {
                frame_queue_next(&is->pictq);
                goto retry;
            }

            if (lastvp->serial != vp->serial)
                is->frame_timer = av_gettime_relative() / 1000000.0;

            if (is->paused && (ffp->first_video_frame_rendered || !ffp->cover_after_prepared))
                goto display;

            /* compute nominal last_duration */
            last_duration = vp_duration(is, lastvp, vp);
            delay = compute_target_delay(ffp, last_duration, is);

            int64_t time_us = av_gettime_relative();
            if (!is->audio_st)
                ffp_clock_msg_notify_cycle(ffp, time_us / 1000);
            time = time_us / 1000000.0;
            if (isnan(is->frame_timer) || time < is->frame_timer)
                is->frame_timer = time;
            if (time < is->frame_timer + delay) {
                *remaining_time = FFMIN(is->frame_timer + delay - time, *remaining_time);
                goto display;
            }

            is->frame_timer += delay;
            if (delay > 0 && time - is->frame_timer > AV_SYNC_THRESHOLD_MAX)
                is->frame_timer = time;

            SDL_LockMutex(is->pictq.mutex);
            if (!isnan(vp->pts))
                update_video_pts(is, vp->pts, vp->pos, vp->serial);
            SDL_UnlockMutex(is->pictq.mutex);

            if (frame_queue_nb_remaining(&is->pictq) > 1) {
                Frame *nextvp = frame_queue_peek_next(&is->pictq);
                duration = vp_duration(is, vp, nextvp);
                if(!is->step && (ffp->framedrop > 0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) && time > is->frame_timer + duration) {
                    frame_queue_next(&is->pictq);
                    goto retry;
                }
            }

            frame_queue_next(&is->pictq);
            is->force_refresh = 1;

            SDL_LockMutex(ffp->is->play_mutex);
            if (is->step) {
                is->step = 0;
                if (!is->paused)