Combining Image Tracking with 3D Rendering——Android AR实现

这篇文章介绍OpenCV&OpenGL开发AR的介绍。主要涉及到的内容看下图我打开的包的内容。首先和第四章中对比一下，发现多了ARCubeRenderer类，adapters包，ARFilter、NoneARFilter接口类。

我们还是从主类看起：

/**
         * 这里使用Framelayout布局管理器，然后添加用于显示实时获取的视频帧的CameraView类，
         * 再添加用于渲染三维虚拟模型的GLSurfaceView类
         */
        FrameLayout layout = new FrameLayout(this);
        layout.setLayoutParams(new FrameLayout.LayoutParams(
                FrameLayout.LayoutParams.MATCH_PARENT,
                FrameLayout.LayoutParams.MATCH_PARENT));
        setContentView(layout);
        
        mCameraView = new NativeCameraView(this, mCameraIndex);
        mCameraView.setCvCameraViewListener(this);
        mCameraView.setLayoutParams(new FrameLayout.LayoutParams(
                FrameLayout.LayoutParams.MATCH_PARENT,
                FrameLayout.LayoutParams.MATCH_PARENT));
        layout.addView(mCameraView);
        
        GLSurfaceView glSurfaceView = new GLSurfaceView(this);
        glSurfaceView.getHolder().setFormat(
                PixelFormat.TRANSPARENT);
        glSurfaceView.setEGLConfigChooser(8, 8, 8, 8, 0, 0);
        glSurfaceView.setZOrderOnTop(true);
        glSurfaceView.setLayoutParams(new FrameLayout.LayoutParams(
                FrameLayout.LayoutParams.MATCH_PARENT,
                FrameLayout.LayoutParams.MATCH_PARENT));
        layout.addView(glSurfaceView);
        
        /**
         * 关于计算机图形学中摄像机模型的一些内容 包括摄像机内部固有参数，OpenGL视锥模型 
         * 可以参考计算机图形中的有关知识以及小孔成像模型
         */
        mCameraProjectionAdapter = new CameraProjectionAdapter();
        
        mARRenderer = new ARCubeRenderer();
        mARRenderer.cameraProjectionAdapter =
                mCameraProjectionAdapter;
        glSurfaceView.setRenderer(mARRenderer);

再来看CameraProjectionAdapter类：

/**
 * 关于计算机图形学中摄像机模型的一些内容
 * 包括摄像机内部固有参数，OpenGL视锥模型
 * 可以参考计算机图形中的有关知识以及小孔成像模型
 * @author scy
 *
 */
public class CameraProjectionAdapter {
    
    float mFOVY = 43.6f; // 30mm equivalent
    float mFOVX = 65.4f; // 30mm equivalent
    int mHeightPx = 640;
    int mWidthPx = 480;
    float mNear = 1f;
    float mFar = 10000f;
    
    final float[] mProjectionGL = new float[16];
    boolean mProjectionDirtyGL = true;
    
    MatOfDouble mProjectionCV;
    boolean mProjectionDirtyCV = true;
    
    // 关于摄像头参数的获取
    // 可以通过摄像头标定，也可以像这里通过AndroidSDK获取
    // 如果能够实时对摄像头进行标定最好，如果不能实时的话，离线标定就需要对不同的移动终端进行标定，这样效率会比较低，而且怎么用于商业应用呢？
    // 还是这种比较理想，不知道高通是怎么做的？有知道朋友可以告诉我一下，谢谢！    
    public void setCameraParameters(Parameters parameters) {
        mFOVY = parameters.getVerticalViewAngle();
        mFOVX = parameters.getHorizontalViewAngle();
        
        Size pictureSize = parameters.getPictureSize();
        mHeightPx = pictureSize.height;
        mWidthPx = pictureSize.width;
        
        mProjectionDirtyGL = true;
        mProjectionDirtyCV = true;
    }
    
    public void setClipDistances(float near, float far) {
        mNear = near;
        mFar = far;
        mProjectionDirtyGL = true;
    }
    
    // http://blog.youkuaiyun.com/lyx2007825/article/details/8792475
    // FOV为视野角，这里有水平和垂直两种，见附图所示：
    public float[] getProjectionGL() {
        if (mProjectionDirtyGL) {
            // 
        	final float top =
                    (float)Math.tan(mFOVY * Math.PI / 360f) * mNear;
            final float right =
                    (float)Math.tan(mFOVX * Math.PI / 360f) * mNear;
            // 跟据设备屏幕的几何特征创建投影矩阵
            Matrix.frustumM(mProjectionGL, 0,
                    -right, right, -top, top, mNear, mFar);
            mProjectionDirtyGL = false;
        }
        return mProjectionGL;
    }
    
    // 获取计算机视觉坐标中的投影矩阵
    // 跟摄像头的内部固有参数    
    public MatOfDouble getProjectionCV() {
        if (mProjectionDirtyCV) {
            if (mProjectionCV == null) {
                mProjectionCV = new MatOfDouble();
                mProjectionCV.create(3, 3, CvType.CV_64FC1);
            }
            
            double diagonalPx = Math.sqrt(
                    (Math.pow(mWidthPx, 2.0) +
                    Math.pow(mHeightPx, 2.0)));
            double diagonalFOV = Math.sqrt(
                    (Math.pow(mFOVX, 2.0) +
                    Math.pow(mFOVY, 2.0)));
            double focalLengthPx = diagonalPx /
                    (2.0 * Math.tan(0.5 * diagonalFOV));
            
            mProjectionCV.put(0, 0, focalLengthPx);
            mProjectionCV.put(0, 1, 0.0);
            mProjectionCV.put(0, 2, 0.5 * mWidthPx);
            mProjectionCV.put(1, 0, 0.0);
            mProjectionCV.put(1, 1, focalLengthPx);
            mProjectionCV.put(1, 2, 0.5 * mHeightPx);
            mProjectionCV.put(2, 0, 0.0);
            mProjectionCV.put(2, 1, 0.0);
            mProjectionCV.put(2, 2, 0.0);
        }
        return mProjectionCV;
    }
}

然后是ImageDetectionFilter类：

public class ImageDetectionFilter implements ARFilter {
    
    private final Mat mReferenceImage;
    private final MatOfKeyPoint mReferenceKeypoints =
            new MatOfKeyPoint();
    private final Mat mReferenceDescriptors = new Mat();
    // CVType defines the color depth, number of channels, and
    // channel layout in the image.
    private final Mat mReferenceCorners =
            new Mat(4, 1, CvType.CV_32FC2);
    
    private final MatOfKeyPoint mSceneKeypoints =
            new MatOfKeyPoint();
    private final Mat mSceneDescriptors = new Mat();
    
    private final Mat mGraySrc = new Mat();
    private final MatOfDMatch mMatches = new MatOfDMatch();
    
    private final FeatureDetector mFeatureDetector =
            FeatureDetector.create(FeatureDetector.STAR);
    private final DescriptorExtractor mDescriptorExtractor =
            DescriptorExtractor.create(DescriptorExtractor.FREAK);
    private final DescriptorMatcher mDescriptorMatcher =
            DescriptorMatcher.create(
                    DescriptorMatcher.BRUTEFORCE_HAMMING);
    
    private final MatOfDouble mDistCoeffs = new MatOfDouble(
            0.0, 0.0, 0.0, 0.0);
    
    private final CameraProjectionAdapter mCameraProjectionAdapter;
    private final MatOfDouble mRVec = new MatOfDouble();
    private final MatOfDouble mTVec = new MatOfDouble();
    private final MatOfDouble mRotation = new MatOfDouble();
    private final float[] mGLPose = new float[16];
    
    private boolean mTargetFound = false;
    
    /**
     *  构造方法，功能依然是初始化，以及对参考图像进行特征点的检测和描述
     *  但是相对第四章中的内容，多了一个cameraProjectionAdapter的对象
     * @param context
     * @param referenceImageResourceID
     * @param cameraProjectionAdapter
     * @throws IOException
     */
    public ImageDetectionFilter(final Context context,
            final int referenceImageResourceID,
            final CameraProjectionAdapter cameraProjectionAdapter)
                    throws IOException {
        // 获取参考图像帧，可以修改这里设置自己的标
        mReferenceImage = Utils.loadResource(context,
                referenceImageResourceID,
                Highgui.CV_LOAD_IMAGE_COLOR);
        
        final Mat referenceImageGray = new Mat();
        Imgproc.cvtColor(mReferenceImage, referenceImageGray,
                Imgproc.COLOR_BGR2GRAY);
        Imgproc.cvtColor(mReferenceImage, mReferenceImage,
                Imgproc.COLOR_BGR2RGBA);
        
        mReferenceCorners.put(0, 0,
                new double[] {0.0, 0.0});
        mReferenceCorners.put(1, 0,
                new double[] {referenceImageGray.cols(), 0.0});
        mReferenceCorners.put(2, 0,
                new double[] {referenceImageGray.cols(),
                        referenceImageGray.rows()});
        mReferenceCorners.put(3, 0,
                new double[] {0.0, referenceImageGray.rows()});
        
        mFeatureDetector.detect(referenceImageGray,
                mReferenceKeypoints);
        mDescriptorExtractor.compute(referenceImageGray,
                mReferenceKeypoints, mReferenceDescriptors);
        
        mCameraProjectionAdapter = cameraProjectionAdapter;
    }
    
    // 复写这个接口方法
    // 根据是否有标志（Target）获取mGLPose
    @Override
    public float[] getGLPose() {
        return (mTargetFound ? mGLPose : null);
    }
    
    /**
     *  同样对实时获取的视频帧进行特征点的检测描述和匹配
     *  多了一个findPose（）方法
     */
    @Override
    public void apply(final Mat src, final Mat dst) {
        Imgproc.cvtColor(src, mGraySrc, Imgproc.COLOR_RGBA2GRAY);
        
        mFeatureDetector.detect(mGraySrc, mSceneKeypoints);
        mDescriptorExtractor.compute(mGraySrc, mSceneKeypoints,
                mSceneDescriptors);
        mDescriptorMatcher.match(mSceneDescriptors,
                mReferenceDescriptors, mMatches);
        
        findPose();
        draw(src, dst);
    }
    
    /**
     * 从方法名可以看出，这个就是核心算法了，估算出摄像头位姿
     */
    private void findPose() {
        
        List<DMatch> matchesList = mMatches.toList();
        if (matchesList.size() < 4) {
            // There are too few matches to find the pose.
            return;
        }
        
        List<KeyPoint> referenceKeypointsList =
                mReferenceKeypoints.toList();
        List<KeyPoint> sceneKeypointsList =
                mSceneKeypoints.toList();
        
        // Calculate the max and min distances between keypoints.
        double maxDist = 0.0;
        double minDist = Double.MAX_VALUE;
        for(DMatch match : matchesList) {
            double dist = match.distance;
            if (dist < minDist) {
                minDist = dist;
            }
            if (dist > maxDist) {
                maxDist = dist;
            }
        }
        
        // The thresholds for minDist are chosen subjectively
        // based on testing. The unit is not related to pixel
        // distances; it is related to the number of failed tests
        // for similarity between the matched descriptors.
        if (minDist > 50.0) {
            // The target is completely lost.
            mTargetFound = false;
            return;
        } else if (minDist > 25.0) {
            // The target is lost but maybe it is still close.
            // Keep using any previously found pose.
            return;
        }
        
        // Identify "good" keypoints based on match distance.
        List<Point3> goodReferencePointsList =
                new ArrayList<Point3>();
        ArrayList<Point> goodScenePointsList =
                new ArrayList<Point>();
        double maxGoodMatchDist = 1.75 * minDist;
        for(DMatch match : matchesList) {
            if (match.distance < maxGoodMatchDist) {
                Point point =
                        referenceKeypointsList.get(match.trainIdx).pt;
                Point3 point3 = new Point3(point.x, point.y, 0.0);
                goodReferencePointsList.add(point3);
                goodScenePointsList.add(
                        sceneKeypointsList.get(match.queryIdx).pt);
            }
        }
        
        if (goodReferencePointsList.size() < 4 ||
                goodScenePointsList.size() < 4) {
            // There are too few good points to find the pose.
            return;
        }
        
        MatOfPoint3f goodReferencePoints = new MatOfPoint3f();
        goodReferencePoints.fromList(goodReferencePointsList);
        
        MatOfPoint2f goodScenePoints = new MatOfPoint2f();
        goodScenePoints.fromList(goodScenePointsList);
        
        /**
         * 前面代码实现和第四章中相同，goodReferencePoints的类型为MatOfPoint3f，之前的是MatOfPoint2f
         * 为什么呢？
         * 主要是因为solvePnP方法，这个方法是使用PNP算法从3D-2D点之间的对应关系计算位姿矩阵
         * 在这里，3D就是参考图像帧，2D就是视频帧，projection为摄像机内部参数，可通过标定计算，
         * 也可以像本文中使用SDK检测并计算，mDistCoeffs为摄像头畸变，本文不考虑畸变情况
         * 以上是输入参数，以下两个是输出参数：mRVec, mTVec
         * mRVec为旋转矩阵
         * mTVec为平移矩阵，这两个参数也是我们最终需要的
         */
        MatOfDouble projection =
                mCameraProjectionAdapter.getProjectionCV();
        // 使用PNP算法计算位姿矩阵
        Calib3d.solvePnP(goodReferencePoints, goodScenePoints,
                projection, mDistCoeffs, mRVec, mTVec);
        
        double[] rVecArray = mRVec.toArray();
        rVecArray[1] *= -1.0;
        rVecArray[2] *= -1.0;
        mRVec.fromArray(rVecArray);
        
        // 将旋转矩阵转换成旋转向量，
        // 关于罗德里格斯变换，可以参阅这篇文章：http://blog.sina.com.cn/s/blog_5fb3f125010100hp.html
        Calib3d.Rodrigues(mRVec, mRotation);
        // 
        double[] tVecArray = mTVec.toArray();
        
        // 将计算得出的结果转换为4*4的位姿矩阵，即摄像头位姿，就是我们最终需要的结果。
        mGLPose[0]  =  (float)mRotation.get(0, 0)[0];
        mGLPose[1]  =  (float)mRotation.get(1, 0)[0];
        mGLPose[2]  =  (float)mRotation.get(2, 0)[0];
        mGLPose[3]  =  0f;
        mGLPose[4]  =  (float)mRotation.get(0, 1)[0];
        mGLPose[5]  =  (float)mRotation.get(1, 1)[0];
        mGLPose[6]  =  (float)mRotation.get(2, 1)[0];
        mGLPose[7]  =  0f;
        mGLPose[8]  =  (float)mRotation.get(0, 2)[0];
        mGLPose[9]  =  (float)mRotation.get(1, 2)[0];
        mGLPose[10] =  (float)mRotation.get(2, 2)[0];
        mGLPose[11] =  0f;
        mGLPose[12] =  (float)tVecArray[0];
        mGLPose[13] = -(float)tVecArray[1];
        mGLPose[14] = -(float)tVecArray[2];
        mGLPose[15] =  1f;
        
        mTargetFound = true;
    }
    
    // 当发现标志时，这里不再绘制边框，只在没有发现标的时候，在左上角绘制标志图片的缩略图
    // 提示需要检测这样的图像（这个功能还蛮不错的）
    protected void draw(Mat src, Mat dst) {
        
        if (dst != src) {
            src.copyTo(dst);
        }
        
        if (!mTargetFound) {
            // The target has not been found.
            
            // Draw a thumbnail of the target in the upper-left
            // corner so that the user knows what it is.
            
            int height = mReferenceImage.height();
            int width = mReferenceImage.width();
            int maxDimension = Math.min(dst.width(),
                    dst.height()) / 2;
            double aspectRatio = width / (double)height;
            if (height > width) {
                height = maxDimension;
                width = (int)(height * aspectRatio);
            } else {
                width = maxDimension;
                height = (int)(width / aspectRatio);
            }
            Mat dstROI = dst.submat(0, height, 0, width);
            Imgproc.resize(mReferenceImage, dstROI, dstROI.size(),
                    0.0, 0.0, Imgproc.INTER_AREA);
        }
    }
}

最后是ARCubeRenderer类，即OpenGL渲染类：

/**
 * OpenGL的渲染类
 * @author scy
 *
 */
public class ARCubeRenderer implements GLSurfaceView.Renderer {
    
    public ARFilter filter;
    public CameraProjectionAdapter cameraProjectionAdapter;
    public float scale = 100f;
    
    private static final ByteBuffer VERTICES;
    private static final ByteBuffer COLORS;
    private static final ByteBuffer TRIANGLE_FAN_0;
    private static final ByteBuffer TRIANGLE_FAN_1;
    
    static {
        VERTICES = ByteBuffer.allocateDirect(96);
        VERTICES.order(ByteOrder.nativeOrder());
        VERTICES.asFloatBuffer().put(new float[] {
                -1f,  1f,  1f,
                 1f,  1f,  1f,
                 1f, -1f,  1f,
                -1f, -1f,  1f,
                
                -1f,  1f, -1f,
                 1f,  1f, -1f,
                 1f, -1f, -1f,
                -1f, -1f, -1f
        });
        VERTICES.position(0);
        
        COLORS = ByteBuffer.allocateDirect(32);
        COLORS.put(new byte[] {
                Byte.MAX_VALUE, Byte.MAX_VALUE, 0, Byte.MAX_VALUE, // yellow
                0, Byte.MAX_VALUE, Byte.MAX_VALUE, Byte.MAX_VALUE, // cyan
                0, 0, 0, Byte.MAX_VALUE, // black
                Byte.MAX_VALUE, 0, Byte.MAX_VALUE, Byte.MAX_VALUE, // magenta
                
                Byte.MAX_VALUE, 0, 0, Byte.MAX_VALUE, // red
                0, Byte.MAX_VALUE, 0, Byte.MAX_VALUE, // green
                0, 0, Byte.MAX_VALUE, Byte.MAX_VALUE, // blue
                0, 0, 0, Byte.MAX_VALUE // black
        });
        COLORS.position(0);
        
        TRIANGLE_FAN_0 = ByteBuffer.allocate(18);
        TRIANGLE_FAN_0.put(new byte[] {
                1, 0, 3,
                1, 3, 2,
                1, 2, 6,
                1, 6, 5,
                1, 5, 4,
                1, 4, 0
        });
        TRIANGLE_FAN_0.position(0);
        
        TRIANGLE_FAN_1 = ByteBuffer.allocate(18);
        TRIANGLE_FAN_1.put(new byte[] {
                7, 4, 5,
                7, 5, 6,
                7, 6, 2,
                7, 2, 3,
                7, 3, 0,
                7, 0, 4
        });
        TRIANGLE_FAN_1.position(0);
    }
    
    @Override
    public void onDrawFrame(final GL10 gl) {
        
        gl.glClear(GL10.GL_COLOR_BUFFER_BIT |
                GL10.GL_DEPTH_BUFFER_BIT);
        gl.glClearColor(0f, 0f, 0f, 0f); // transparent
        
        if (filter == null) {
            return;
        }
        
        if (cameraProjectionAdapter == null) {
            return;
        }
        // 获取摄像头位姿矩阵
        float[] pose = filter.getGLPose();
        if (pose == null) {
            return;
        }
        
        /**
         * 这里有两个作用，一是设置投影矩阵
         * 二是模型试图矩阵，注意这两个顺序不能乱，
         * 然后绘制三维模型
         */
        gl.glMatrixMode(GL10.GL_PROJECTION);
        float[] projection =
                cameraProjectionAdapter.getProjectionGL();
        gl.glLoadMatrixf(projection, 0);
        
        gl.glMatrixMode(GL10.GL_MODELVIEW);
        gl.glLoadMatrixf(pose, 0);
        gl.glTranslatef(0f, 0f, 1f);
        gl.glScalef(scale, scale, scale);
        
        // 开始绘制三维虚拟模型，这里是OpenGL的内容，网上有很多介绍
        // Android OpenGL开发的教程，有兴趣可以去看看，这里我就在不再赘述了。
        gl.glEnableClientState(GL10.GL_VERTEX_ARRAY);
        gl.glEnableClientState(GL10.GL_COLOR_ARRAY);
        
        gl.glVertexPointer(3, GL11.GL_FLOAT, 0, VERTICES);
        gl.glColorPointer(4, GL11.GL_UNSIGNED_BYTE, 0, COLORS);
        
        gl.glDrawElements(GL10.GL_TRIANGLE_FAN, 18,
                GL10.GL_UNSIGNED_BYTE, TRIANGLE_FAN_0);
        gl.glDrawElements(GL10.GL_TRIANGLE_FAN, 18,
                GL10.GL_UNSIGNED_BYTE, TRIANGLE_FAN_1);
    }
    
    @Override
    public void onSurfaceChanged(final GL10 gl, final int width,
            final int height) {
    }
    
    @Override
    public void onSurfaceCreated(final GL10 arg0,
            final EGLConfig config) {
    }
}

附图：

^_^本团队专业从事移动增强现实应用开发以及解决方案，有合作请私信联系！^_^