项目记录: HEVC单帧码流硬解渲染

HEVC单帧码流硬解渲染

项目里面需要使用读取每一帧单帧码流进行渲染.特此记录一下.

硬解码

• 硬解码使用的MediaCodec . 一般它与 MediaExtractor 配合使用.
• MediaExtractor 从MP4等格式中抽取出 码流数据 送给MediaCodec解码器.

坑

• H.264码流主要分Annex-B和AVCC两种格式，H.265码流主要分为Annex-B和HVCC格式。AnnexB与AVCC/HVCC的区别在于参数集与帧格式，AnnexB的参数集sps、pps以NAL的形式存在码流中（带内传输），以startcode分割NAL。
• 而AVCC/HVCC 的参数集存储在extradata中（带外传输），使用NALU长度（固定字节，通常为4字节，从extradata中解析）分隔NAL，通常MP4、MKV使用AVCC格式来存储。
• Android的硬解只接受Annex-B格式的码流，所以在解码MP4 Demux出的视频流时，需要解析extradata，取出sps、pps，通过CSD(Codec-Specific Data)来初始化解码器；并且将AVCC码流转换为Annex-B，在ffmpeg中使用h264_mp4toannexb_filter或hevc_mp4toannexb做转换。

方法:

• 一般编码出来的是Annex-B 格式码流,手动去解析出 CSD 来初始化解码器.
• 源码链接

package jp.yohhoy.hevcdec;

import android.app.Activity;
import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.os.Bundle;
import android.util.Log;
import android.util.Size;
import android.view.Surface;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.ViewGroup;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;

public class MainActivity extends Activity {
    private static final String TAG = "MainActivity";

    private static byte[] loadRawResource(Context ctx, int resId) {
        int size = (int) ctx.getResources().openRawResourceFd(resId).getLength();
        try {
            byte data[] = new byte[size];
            InputStream is = ctx.getResources().openRawResource(resId);
            is.read(data);
            return data;
        } catch (IOException ex) {
            return null;
        }
    }

    private static MediaCodec findHevcDecoder() {
        // "video/hevc" may select hardware decoder on the device.
        // "OMX.google.hevc.decoder" is software decoder.
        final String[] codecNames = {"video/hevc", "OMX.google.hevc.decoder"};

        for (String name : codecNames) {
            try {
                MediaCodec codec = MediaCodec.createByCodecName(name);
                Log.i(TAG, "codec \"" + name + "\" is available");
                return codec;
            } catch (IOException | IllegalArgumentException ex) {
                Log.d(TAG, "codec \"" + name + "\" not found");
            }
        }
        Log.w(TAG, "HEVC decoder is not available");
        return null;
    }

    private static ByteBuffer extractHevcParamSets(byte[] bitstream) {
        final byte[] startCode = {0x00, 0x00, 0x00, 0x01};
        int nalBeginPos = 0, nalEndPos = 0;
        int nalUnitType = -1;
        int nlz = 0;
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        for (int pos = 0; pos < bitstream.length; pos++) {
            if (2 <= nlz && bitstream[pos] == 0x01) {
                nalEndPos = pos - nlz;
                if (nalUnitType == 32 || nalUnitType == 33 || nalUnitType == 34) {
                    // extract VPS(32), SPS(33), PPS(34)
                    Log.d(TAG, "NUT=" + nalUnitType + " range={" + nalBeginPos + "," + nalEndPos + "}");
                    try {
                        baos.write(startCode);
                        baos.write(bitstream, nalBeginPos, nalEndPos - nalBeginPos);
                    } catch (IOException ex) {
                        Log.e(TAG, "extractHevcParamSets", ex);
                        return null;
                    }
                }
                nalBeginPos = ++pos;
                nalUnitType = (bitstream[pos] >> 1) & 0x2f;
                if (0 <= nalUnitType && nalUnitType <= 31) {
                    break;  // VCL NAL; no more VPS/SPS/PPS
                }
            }
            nlz = (bitstream[pos] != 0x00) ? 0 : nlz + 1;
        }
        return ByteBuffer.wrap(baos.toByteArray());
    }

    private static Size calcImageSize(MediaFormat format) {
        int cropLeft = format.getInteger("crop-left");
        int cropRight = format.getInteger("crop-right");
        int cropTop = format.getInteger("crop-top");
        int cropBottom = format.getInteger("crop-bottom");
        int width = cropRight + 1 - cropLeft;
        int height = cropBottom + 1 - cropTop;
        return new Size(width, height);
    }

    private static Size renderHevcImage(byte[] bitstream, Surface surface) {
        MediaCodec decoder = findHevcDecoder();
        if (decoder == null) {
            return null;
        }

        // configure HEVC decoder
        MediaFormat inputFormat = MediaFormat.createVideoFormat("video/hevc", 640, 480);
        inputFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, bitstream.length);
        inputFormat.setByteBuffer("csd-0", extractHevcParamSets(bitstream));
        Log.d(TAG, "input-format=" + inputFormat);
        decoder.configure(inputFormat, surface, null, 0);
        MediaFormat outputFormat = decoder.getOutputFormat();
        Log.d(TAG, "output-format=" + outputFormat);
        Size imageSize = calcImageSize(outputFormat);
        decoder.start();

        // set bitstream to decoder
        int inputBufferId = decoder.dequeueInputBuffer(-1);
        if (inputBufferId < 0) {
            Log.e(TAG, "dequeueInputBuffer return " + inputBufferId);
            return null;
        }
        ByteBuffer inBuffer = decoder.getInputBuffer(inputBufferId);
        inBuffer.put(bitstream);
        decoder.queueInputBuffer(inputBufferId, 0, bitstream.length, 0, 0);

        // notify end of stream
        inputBufferId = decoder.dequeueInputBuffer(-1);
        if (inputBufferId < 0) {
            Log.e(TAG, "dequeueInputBuffer return " + inputBufferId);
            return null;
        }
        decoder.queueInputBuffer(inputBufferId, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);

        // get decoded image
        MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
        while (true) {
            int outputBufferId = decoder.dequeueOutputBuffer(bufferInfo, -1);
            if (outputBufferId >= 0) {
                decoder.releaseOutputBuffer(outputBufferId, true);
                break;
            } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                outputFormat = decoder.getOutputFormat();
                Log.d(TAG, "output-format=" + outputFormat);
                imageSize = calcImageSize(outputFormat);
            } else {
                Log.d(TAG, "dequeueOutputBuffer return " + outputBufferId);
            }
        }

        decoder.flush();
        decoder.stop();
        decoder.release();
        return imageSize;
    }

    private SurfaceView mSurfaceView;

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        mSurfaceView = new SurfaceView(this);
        setContentView(mSurfaceView);

        // load HEVC bitstream (Annex.B format)
        final byte[] bitstream = loadRawResource(this, R.raw.lena_std);
        Log.d(TAG, "length=" + bitstream.length);

        mSurfaceView.getHolder().addCallback(new SurfaceHolder.Callback() {
            @Override
            public void surfaceCreated(SurfaceHolder holder) {
            }

            @Override
            public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
                Log.d(TAG, "surfaceChanged format=" + format + " size=" + width + "x" + height);
                Size sz = renderHevcImage(bitstream, holder.getSurface());
                if (sz != null) {
                    // fit SurfaceView to decoded image
                    ViewGroup.LayoutParams lp = mSurfaceView.getLayoutParams();
                    lp.width = sz.getWidth();
                    lp.height = sz.getHeight();
                    mSurfaceView.setLayoutParams(lp);
                }
            }

            @Override
            public void surfaceDestroyed(SurfaceHolder holder) {
            }
        });
    }
}

参考文献

如何渲染编码的裸数据
码流格式
Android转码SDK实战