从IPC摄像机读取视频帧解码并转化为YUV数据到转化为Bitmap

原创已于 2024-09-18 14:35:13 修改 · 1.2k 阅读

11 ·

CC 4.0 BY-SA版权

文章标签：

#音视频

于 2024-09-18 14:25:49 首次发布

音视频专栏收录该内容

6 篇文章

订阅专栏

前言

本文主要介绍根据IPC的RTSP视频流地址，连接摄像机，并持续读取相机视频流，进一步进行播放实时画面，或者处理视频帧，将每一帧数据转化为安卓相机同格式数据，并保存为bitmap。

示例

val rtspClientListener = object: RtspClient.RtspClientListener {
    override fun onRtspConnecting() {}
    override fun onRtspConnected(sdpInfo: SdpInfo) {}
    override fun onRtspVideoNalUnitReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {
        // 发送原始H264/H265 NAL单元到解码器
    }
    override fun onRtspAudioSampleReceived(data: ByteArray, offset: Int, length: Int, timestamp: Long) {
        // 发送原始音频到解码器
    }
    override fun onRtspDisconnected() {}
    override fun onRtspFailedUnauthorized() {
        Log.e(TAG, "RTSP failed unauthorized");
    }
    override fun onRtspFailed(message: String?) {
        Log.e(TAG, "RTSP failed with message '$message'")
    }
}

val uri = Uri.parse("rtsp://192.168.43.23:554/ch01.264?dev=1")
val username = "admin"
val password = ""
val stopped = new AtomicBoolean(false)
val socket: Socket = NetUtils.createSocketAndConnect(uri.host.toString(), port, 5000)

val rtspClient = RtspClient.Builder(socket, uri.toString(), stopped, rtspClientListener)
    .requestVideo(true)
    .requestAudio(true)
    .withDebug(false)
    .withUserAgent("RTSP client")
    .withCredentials(username, password)
    .build()
// Blocking call until stopped variable is true or connection failed
rtspClient.execute()

NetUtils.closeSocket(sslSocket)

其中 mViewBind.rtspView1 为封装的SurfaceView 用来播放摄像机的实时画面。

用到的第三方库为：Lightweight RTSP client library for Android

rtspClientListener 接口包含在 RtspSurfaceView 中，通过 onRtspVideoNalUnitReceived 方法可以拿到原始视频流数据。

注意：目前该库只支持H264编码格式视频，现在摄像机大都默认H265编码，如遇到不可播放问题，请检查摄像机视频流编码格式！

解码

当我们通过 onRtspVideoNalUnitReceived 拿到原始视频流数据后，对该数据进行解码处理，转化为YUV格式数据。

    fun decode(data: ByteArray, offset: Int, length: Int,decodeCallback:DecodeCallback) {
        val inputBuffers = codec?.inputBuffers
        val outputBuffers = codec?.outputBuffers
        val inputBufferIndex = codec?.dequeueInputBuffer(10000) ?: -1

        if (inputBufferIndex >= 0) {
            val inputBuffer = inputBuffers?.get(inputBufferIndex)
            inputBuffer?.clear()
            inputBuffer?.put(data, offset, length)
            codec?.queueInputBuffer(inputBufferIndex, 0, length, 0, 0)
        }

        val bufferInfo = MediaCodec.BufferInfo()
        var outputBufferIndex = codec?.dequeueOutputBuffer(bufferInfo, 10000) ?: -1
        while (outputBufferIndex >= 0) {
            val outputBuffer = outputBuffers?.get(outputBufferIndex)
            // 处理解码后的YUV数据
            processYUVData(outputBuffer, bufferInfo,decodeCallback)
            codec?.releaseOutputBuffer(outputBufferIndex, false)
            outputBufferIndex = codec?.dequeueOutputBuffer(bufferInfo, 0) ?: -1
        }
    }

    private fun processYUVData(outputBuffer: ByteBuffer?, bufferInfo: MediaCodec.BufferInfo,decodeCallback:DecodeCallback) {
        val yuvData = ByteArray(bufferInfo.size)
        outputBuffer?.get(yuvData)//默认NV21
        decodeCallback.invoke(yuvData)
    }

此处的yuvData 就是我们常见的安卓相机回调的YUV格式数据。当然，摄像机并不是安卓相机还是有区别的，如果相机视频流使用 H.264 编码，采用 YUV 420 格式，那么此时的 yuvData数据转为bitmap就会出现红蓝颠倒问题，就是红色的东西变成蓝色，蓝色的变成红色，这是因为在标准 YUV 420 中，U 和 V 分量可能是分开存储的，或者以不同的顺序排列。某些实现中，U 和 V 的顺序可能是 U 在前。

解决方法就是将UV位置互换一下👇

    /**
     * NV21转YUV420SP 解决红蓝色颠倒的问题
     */
    private fun nv21ToYuv420sp(width: Int, height: Int, inArray: ByteArray) {
        val pixels = width * height
        val count = pixels / 2
        for (i in 0 until count step 2) {
            val s = inArray[pixels + i]
            inArray[pixels + i] = inArray[pixels + i + 1]
            inArray[pixels + i + 1] = s
        }
    }

此时你就可以拿到正常的YUV格式数据了。

如果你想要进一步得到 bitmap 则可以使用以下代码处理

     fun yuvToBitmap(yuvData: ByteArray): Bitmap? {
        val yuvImage = YuvImage(yuvData, ImageFormat.NV21, width, height, null)
        val out = ByteArrayOutputStream()
        yuvImage.compressToJpeg(android.graphics.Rect(0, 0, width, height), 100, out)
        val imageBytes = out.toByteArray()
        return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
    }

转换过程中，YuvImage 会根据给定的 YUV 数据进行适当的解析和处理。只需确保输入的 YUV 数据符合 YuvImage 所需的格式和排列即可。

解码相关完整代码如下：

typealias DecodeCallback = (result: ByteArray) -> Unit
class H264Decoder {
    private val mimeType = "video/avc"//h264
    private var codec: MediaCodec? = null
     var width = 0
     var height = 0

    fun init(videoWidth: Int, videoHeight: Int) {
        width = videoWidth
        height = videoHeight
        codec = MediaCodec.createDecoderByType(mimeType)
        val format = MediaFormat.createVideoFormat(mimeType, width, height) // 设定视频分辨率
        codec?.configure(format, null, null, 0)
        codec?.start()
    }

    fun decode(data: ByteArray, offset: Int, length: Int,decodeCallback:DecodeCallback) {
        val inputBuffers = codec?.inputBuffers
        val outputBuffers = codec?.outputBuffers
        val inputBufferIndex = codec?.dequeueInputBuffer(10000) ?: -1

        if (inputBufferIndex >= 0) {
            val inputBuffer = inputBuffers?.get(inputBufferIndex)
            inputBuffer?.clear()
            inputBuffer?.put(data, offset, length)
            codec?.queueInputBuffer(inputBufferIndex, 0, length, 0, 0)
        }

        val bufferInfo = MediaCodec.BufferInfo()
        var outputBufferIndex = codec?.dequeueOutputBuffer(bufferInfo, 10000) ?: -1
        while (outputBufferIndex >= 0) {
            val outputBuffer = outputBuffers?.get(outputBufferIndex)
            // 处理解码后的YUV数据
            processYUVData(outputBuffer, bufferInfo,decodeCallback)
            codec?.releaseOutputBuffer(outputBufferIndex, false)
            outputBufferIndex = codec?.dequeueOutputBuffer(bufferInfo, 0) ?: -1
        }
    }

    private fun processYUVData(outputBuffer: ByteBuffer?, bufferInfo: MediaCodec.BufferInfo,decodeCallback:DecodeCallback) {
        val yuvData = ByteArray(bufferInfo.size)
        outputBuffer?.get(yuvData)//默认NV21
        nv21ToYuv420sp(width, height, yuvData)
        decodeCallback.invoke(yuvData)
    }

     fun yuvToBitmap(yuvData: ByteArray): Bitmap? {
        val yuvImage = YuvImage(yuvData, ImageFormat.NV21, width, height, null)
        val out = ByteArrayOutputStream()
        yuvImage.compressToJpeg(android.graphics.Rect(0, 0, width, height), 100, out)
        val imageBytes = out.toByteArray()
        return BitmapFactory.decodeByteArray(imageBytes, 0, imageBytes.size)
    }

    /**
     * NV21转YUV420SP 解决红蓝色颠倒的问题
     */
    private fun nv21ToYuv420sp(width: Int, height: Int, inArray: ByteArray) {
        val pixels = width * height
        val count = pixels / 2
        for (i in 0 until count step 2) {
            val s = inArray[pixels + i]
            inArray[pixels + i] = inArray[pixels + i + 1]
            inArray[pixels + i + 1] = s
        }
    }

    fun release() {
        codec?.stop()
        codec?.release()
    }
}

调用方式如下


//初始化h264Decoder，主要是初始化宽和高，这里的宽高可以从摄像机后台获取，或者通过onvif协议搜索摄像机，获取相关参数
h264DecoderA = H264Decoder().apply {
                    init(ipc1_width, ipc1_height)
                }


mViewBind.rtspView1.videoNalCallback = { data, offset, length, timestamp ->
                    h264DecoderA?.decode(data, offset, length) { yuvData ->
                   
                    }
                }