Video的thumbnail抽取算法

How to generate video thumbnail：

注意生成thumbnail时，传入getFrameAtTime的时间是－1

frameworks/base/media/java/android/media/ThumbnailUtils.java

    158     public static Bitmap createVideoThumbnail(String filePath, int kind) {
    159         Bitmap bitmap = null;
    160         MediaMetadataRetriever retriever = new MediaMetadataRetriever();
    161         try {
    162             retriever.setDataSource(filePath);
    163             bitmap = retriever.getFrameAtTime(-1);
    164         } catch (IllegalArgumentException ex) {
    165             // Assume this is a corrupt video file
    166         } catch (RuntimeException ex) {
    167             // Assume this is a corrupt video file.
    168         } finally {
    169             try {
    170                 retriever.release();
    171             } catch (RuntimeException ex) {
    172                 // Ignore failures while cleaning up.
    173             }
    174         }

frameworks / base / media / java / android / media / MediaMetadataRetriever.java

    270     public Bitmap getFrameAtTime(long timeUs) {
    271         return getFrameAtTime(timeUs, OPTION_CLOSEST_SYNC);

frameworks/av/media/libmedia/mediametadataretriever.cpp

sp<IMemory> MediaMetadataRetriever::getFrameAtTime(int64_t timeUs, int option)
{
    ALOGV("getFrameAtTime: time(%lld us) option(%d)", timeUs, option);
    Mutex::Autolock _l(mLock);
    if (mRetriever == 0) {
        ALOGE("retriever is not initialized");
        return NULL;
    }
    return mRetriever->getFrameAtTime(timeUs, option);
}

frameworks/av/media/libmediaplayerservice/MetadataRetrieverClient.cpp

sp<IMemory> MetadataRetrieverClient::getFrameAtTime(int64_t timeUs, int option)
{
    ALOGV("getFrameAtTime: time(%lld us) option(%d)", timeUs, option);
    Mutex::Autolock lock(mLock);
    mThumbnail.clear();
    if (mRetriever == NULL) {
        ALOGE("retriever is not initialized");
        return NULL;
    }
    VideoFrame *frame = mRetriever->getFrameAtTime(timeUs, option);

media/libstagefright/StagefrightMetadataRetriever.cpp

VideoFrame *StagefrightMetadataRetriever::getFrameAtTime(
        int64_t timeUs, int option) {...

    if (mExtractor.get() == NULL) {
        ALOGV("no extractor.");
        return NULL;
    }

    sp<MetaData> fileMeta = mExtractor->getMetaData();

...

    size_t n = mExtractor->countTracks();
    size_t i;
    for (i = 0; i < n; ++i) {
        sp<MetaData> meta = mExtractor->getTrackMetaData(i);  // 这里将赋值给kKeyThumbnailTime

        const char *mime;
        CHECK(meta->findCString(kKeyMIMEType, &mime));

        if (!strncasecmp(mime, "video/", 6)) {
            break;
        }
    }

...

    if (fileMeta->findData(kKeyAlbumArt, &type, &data, &dataSize)
            && mAlbumArt == NULL) {
        mAlbumArt = new MediaAlbumArt;
        mAlbumArt->mSize = dataSize;
        mAlbumArt->mData = new uint8_t[dataSize];
        memcpy(mAlbumArt->mData, data, dataSize);
    }

    VideoFrame *frame =
        extractVideoFrameWithCodecFlags(                    ／／抽取thumbnail
                &mClient, trackMeta, source, OMXCodec::kSoftwareCodecsOnly,
                timeUs, option);

    ...}

下面函数具体抽取thumbnail：

    static VideoFrame *extractVideoFrameWithCodecFlags(

        OMXClient *client,
        const sp<MetaData> &trackMeta,
        const sp<MediaSource> &source,
        uint32_t flags,
        int64_t frameTimeUs,
        int seekMode) {...

    int64_t thumbNailTime;
    if (frameTimeUs < 0) { ／／如果传入的时间为负数
        if (!trackMeta->findInt64(kKeyThumbnailTime, &thumbNailTime)
                || thumbNailTime < 0) { ／／查看kKeyThumbnailTime是否存在
            thumbNailTime = 0; ／／如不存在取第0帧
        }
        options.setSeekTo(thumbNailTime, mode);
    } else {
        thumbNailTime = -1;
        options.setSeekTo(frameTimeUs, mode);
    }

    MediaBuffer *buffer = NULL;
    do {
        if (buffer != NULL) {
            buffer->release();
            buffer = NULL;
        }
        err = decoder->read(&buffer, &options);
        options.clearSeekTo();
    } while (err == INFO_FORMAT_CHANGED
             || (buffer != NULL && buffer->range_length() == 0));

之前在MPEG4Extractor中读取TrackMetadata时，已经对取哪一帧作为thumbnail进行了计算，并存入到kKeyThumbnailTime中。具体代码见下面的函数getTrackMetaData

注意仅当存在movie fragments ('moof' box）时，将把1/4duration处的帧作为thumbnail，否则将取前20个sample内具有最大值的sample作为thumbnail

media/libstagefright/MPEG4Extractor.cpp

sp<MetaData> MPEG4Extractor::getTrackMetaData(
        size_t index, uint32_t flags) {...

    if ((flags & kIncludeExtensiveMetaData)
            && !track->includes_expensive_metadata) {
        track->includes_expensive_metadata = true;    ／／置位

        const char *mime;
        CHECK(track->meta->findCString(kKeyMIMEType, &mime));
        if (!strncasecmp("video/", mime, 6)) {   ／／如果是video
            if (mMoofOffset > 0) { // if has moof box ,why ??
                int64_t duration;
                if (track->meta->findInt64(kKeyDuration, &duration)) {
                    // nothing fancy, just pick a frame near 1/4th of the duration
                    track->meta->setInt64(
                            kKeyThumbnailTime, duration / 4);
                }
            } else {
                uint32_t sampleIndex;
                uint32_t sampleTime;
                if (track->sampleTable->findThumbnailSample(&sampleIndex) == OK
                        && track->sampleTable->getMetaDataForSample(
                            sampleIndex, NULL /* offset */, NULL /* size */,
                            &sampleTime) == OK) {
                    track->meta->setInt64( ／／将选定sample处的时间作为thumbnail对应时间
                            kKeyThumbnailTime,
                            ((int64_t)sampleTime * 1000000) / track->timescale);
                }
            }
        }

具体sample的挑选算法：

media/libstagefright/SampleTable.cpp

status_t SampleTable::findThumbnailSample(uint32_t *sample_index) {
    Mutex::Autolock autoLock(mLock);

    uint32_t bestSampleIndex = 0;
    size_t maxSampleSize = 0;

    static const size_t kMaxNumSyncSamplesToScan = 20;  ／／取前20个sample

    // Consider the first kMaxNumSyncSamplesToScan sync samples and
    // pick the one with the largest (compressed) size as the thumbnail.

    size_t numSamplesToScan = mNumSyncSamples;
    if (numSamplesToScan > kMaxNumSyncSamplesToScan) {
        numSamplesToScan = kMaxNumSyncSamplesToScan;
    }
    for (size_t i = 0; i < numSamplesToScan; ++i) {
        uint32_t x = mSyncSamples[i];

        // Now x is a sample index.
        size_t sampleSize;
        status_t err = getSampleSize_l(x, &sampleSize);
        if (err != OK) {
            return err;
        }

        if (i == 0 || sampleSize > maxSampleSize) {
            bestSampleIndex = x;
            maxSampleSize = sampleSize;
        }

    }
    *sample_index = bestSampleIndex;

    return OK;
}