Opencv图片识别，问题及结果

前言：

最近单位要实现图片比对一个功能，突然感觉好高大上，对于从来没接触过的我来说“两眼一抹黑”，宝宝不懂啊。从度娘那里找opencv的demo，可是马丹的demo多数都不能用，
  上opencv官方网站下载opencv for android sdk，（OpenCV-3.1.0-android-sdk）
  
  （又学习了ndk和jni，从搭建环境到生成可用的so库，挺烦人的）

  opencv是调用的C和C++生成的静态库.so，然后根据demo和网上的前人经验，打开as配置ndk，编译jni中的代码，编译缺少nonfree库  擦

下面是问题和解决办法：

问题1：编译缺少nonfree库！
        有问题就找解决办法，找nonfree库，可是官方已经不提供这个库，找到nonfree的相关代码 （图1）

                

把nonfree包 放进sdk\native\jni\include\opencv2中，然后执行ndk-build，还是报错。郁闷啊

又尝试了几次还是不行，问问大牛？   http://johnhany.net/ 根据这个上边编译nonfree库，看着好复杂，宝宝不想弄费劲

继续找，在github上找opencv nonfree，github还是比较强大的，别人开源的代码好多，找到了这个库demo
https://github.com/superzoro168/OpenCVNonFree 下载看看，as跑不起来，改吧改吧，跑起来发现这个可以用，happy了，感谢 superzoro168 这哥们，  呵呵……

问题2：实现的效果不是自己想要的！
        看实现的代码：

static {
        System.loadLibrary("opencv_java");
        System.loadLibrary("nonfree");
    }
public void sift() {
        inputImage = BitmapFactory.decodeResource(getResources(), R.mipmap.m1);

        Mat rgba = new Mat();
        Utils.bitmapToMat(inputImage, rgba);
        MatOfKeyPoint keyPoints = new MatOfKeyPoint();
        Imgproc.cvtColor(rgba, rgba, Imgproc.COLOR_RGBA2GRAY);
        detector.detect(rgba, keyPoints);
        Features2d.drawKeypoints(rgba, keyPoints, rgba);
        Utils.matToBitmap(rgba, inputImage);
        imageView.setImageBitmap(inputImage);
    }

展示结果：图片上画出关键点drawKeypoints   ，没有实现匹配，所以白扯还得找实现的方法  ( ˇ?ˇ ))


ios同事给了两个实现匹配的c++代码：

获取图像中绘制特征点


#include "opencv2/core/core.hpp"
#include "highgui.h"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/features2d/features2d.hpp"
#include "opencv2/nonfree/nonfree.hpp"
using namespace cv;
//using namespace std;
int main(int argc, char** argv){
Mat img = imread("box_in_scene.png");
SIFT sift; //实例化 SIFT 类
vector<KeyPoint> key_points; //特征点
// descriptors 为描述符， mascara 为掩码矩阵
Mat descriptors, mascara;
Mat output_img; //输出图像矩阵
sift(img,mascara,key_points,descriptors); //执行 SIFT 运算
//在输出图像中绘制特征点
drawKeypoints(img, //输入图像
key_points, //特征点矢量
output_img, //输出图像
Scalar::all(‐1), //绘制特征点的颜色，为随机
//以特征点为中心画圆，圆的半径表示特征点的大小，直线表示特征点的方向
DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
namedWindow("SIFT");
imshow("SIFT", output_img);
waitKey(0);
return 0;
}

匹配图片

#include "opencv2/core/core.hpp"
#include "highgui.h"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/features2d/features2d.hpp"
#include "opencv2/nonfree/nonfree.hpp"
#include "opencv2/legacy/legacy.hpp"
using namespace cv;
using namespace std;
int main(int argc, char** argv){
//待匹配的两幅图像，其中 img1 包括 img2，也就是要从 img1 中识别出 img2
Mat img1 = imread("box_in_scene.png");
Mat img2 = imread("box.png");
SIFT sift1, sift2;
vector<KeyPoint> key_points1, key_points2;
Mat descriptors1, descriptors2, mascara;
sift1(img1,mascara,key_points1,descriptors1);
sift2(img2,mascara,key_points2,descriptors2);
//实例化暴力匹配器——BruteForceMatcher
BruteForceMatcher<L2<float>> matcher;
//定义匹配器算子
vector<DMatch>matches;
//实现描述符之间的匹配，得到算子 matches
matcher.match(descriptors1,descriptors2,matches);
//提取出前 30 个最佳匹配结果
std::nth_element(matches.begin(), //匹配器算子的初始位置
matches.begin()+29, // 排序的数量
matches.end()); // 结束位置
//剔除掉其余的匹配结果
matches.erase(matches.begin()+30, matches.end());
Mat img_matches;
//在输出图像中绘制匹配结果
drawMatches(img1,key_points1, //第一幅图像和它的特征点
img2,key_points2, //第二幅图像和它的特征点
matches, //匹配器算子
img_matches, //匹配输出图像
Scalar(255,255,255)); //用白色直线连接两幅图像中的特征点
imshow("SIFT_matches",img_matches);
return 0;
}

但是这是C代码，因为是小白，所以不明白怎样用opencv的java sdk 封装的类写成上述的方法


找解决办法：http://www.tuicool.com/articles/FrmInyy   不行有问题，找
http://www.cnblogs.com/dawnminghuang/p/4248449.html   下载demo，实现匹配，而且这两个文章写得很详细

eg： public class MainActivity extends Activity implements CvCameraViewListener2 {
private Bitmap testimg;
private Bitmap matchbitmap;
private CameraBridgeViewBase mOpenCvCameraView;
private Mat mRgba;
private Mat mGray;
private Mat mByte;
private Scalar CONTOUR_COLOR;
private boolean isProcess = false;
private String filepath = "/sdcard/test.jpg";
private static final String TAG = "Dawn";
private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {
@Override
public void onManagerConnected(int status) {
switch (status) {
case LoaderCallbackInterface.SUCCESS: {
Log.i(TAG, "OpenCV loaded successfully");
mOpenCvCameraView.enableView();
}
break;
default: {
super.onManagerConnected(status);
}
break;
}
}
};


@Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
setContentView(R.layout.activity_main);
mOpenCvCameraView = (CameraBridgeViewBase) findViewById(R.id.objectMatch);
mOpenCvCameraView.setCvCameraViewListener(this);
final ImageView showimg = (ImageView) findViewById(R.id.ImgPhoto);
final ImageView matchimg = (ImageView) findViewById(R.id.Imgmatch);
Button showButton = (Button) findViewById(R.id.button_show);
Button matchButton = (Button) findViewById(R.id.button_match);
Button showmatchButton = (Button) findViewById(R.id.button_showmatch);
showButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
// show img in the Imageview
File file = new File(filepath);
if (file.exists()) {
testimg = BitmapFactory.decodeFile(filepath);
// 将图片显示到ImageView中
showimg.setImageBitmap(testimg);
}
}
});
matchButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
// show img in the Imageview
isProcess = !isProcess;
}
});
showmatchButton.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
// show img in the Imageview
matchimg.setImageBitmap(matchbitmap);
}
});
}


@Override
protected void onPause() {
super.onPause();
if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();


}


public void onResume() {
super.onResume();
OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_10, this,
mLoaderCallback);
}


@Override
protected void onDestroy() {
Log.e("onDestroy", "INITIATED");
super.onDestroy();


if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();


}


public void onCameraViewStarted(int width, int height) {
mRgba = new Mat(height, width, CvType.CV_8UC3);
mByte = new Mat(height, width, CvType.CV_8UC1);
       


}


public void onCameraViewStopped() { // Explicitly deallocate Mats
mRgba.release();
}


public Mat onCameraFrame(CvCameraViewFrame inputFrame) {

        Bitmap s_testimg;
Mat testimage = new Mat();
Mat grayimage=new Mat();
mRgba = inputFrame.rgba();
mGray = inputFrame.gray();
CONTOUR_COLOR = new Scalar(255);
MatOfDMatch matches = new MatOfDMatch();
MatOfKeyPoint keypoint_train = new MatOfKeyPoint();
MatOfKeyPoint keypoint_test = new MatOfKeyPoint();
  KeyPoint kpoint = new KeyPoint();
Mat mask = Mat.zeros(mGray.size(), CvType.CV_8UC1);
Mat output = new Mat(); // Mat train=new Mat(); Mat
Mat test = new Mat();
Mat train = new Mat();
if (isProcess) {
FeatureDetector detector_train = FeatureDetector
.create(FeatureDetector.ORB);
detector_train.detect(mGray, keypoint_train);
// Features2d.drawKeypoints(mGray, keypoint_train, output, new Scalar(
// 2, 254, 255), Features2d.DRAW_RICH_KEYPOINTS);


DescriptorExtractor descriptor_train = DescriptorExtractor
.create(DescriptorExtractor.ORB);
descriptor_train.compute(mGray, keypoint_train, train);
s_testimg = Bitmap.createScaledBitmap(testimg, mGray.width(), mGray.height(), false);

Utils.bitmapToMat(s_testimg, testimage);
Imgproc.cvtColor(testimage, grayimage, Imgproc.COLOR_RGB2GRAY);

FeatureDetector detector_test = FeatureDetector
.create(FeatureDetector.ORB);
detector_test.detect(grayimage, keypoint_test);


// Features2d.drawKeypoints(testimage, keypoint_test, output,
// new Scalar(2, 254, 255), Features2d.DRAW_RICH_KEYPOINTS);
DescriptorExtractor descriptor_test = DescriptorExtractor
.create(DescriptorExtractor.ORB);
descriptor_test.compute(grayimage, keypoint_test, test);
DescriptorMatcher descriptormatcher = DescriptorMatcher
.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
descriptormatcher.match(test, train, matches);
Features2d.drawMatches(grayimage,keypoint_test,mGray, keypoint_train, matches, output);
matchbitmap=Bitmap.createScaledBitmap(testimg, output.width(),  output.height(), false);
Utils.matToBitmap(output, matchbitmap);

return mRgba;
}


return mRgba;
}


}

问题3：上边的代码只是实现了匹配后的图片drawMatches，怎样算是匹配成功呢？
    解决方案，取出matches对应的DMatch中的distance确定是否匹配：

    List<DMatch> list = matches.toList().subList(0,29);
            float minDistance = 100;
            for(int i=0;i<list.size();i++){
                minDistance = Math.min(minDistance,list.get(i).distance);
            }
            Log.e("dmatch",minDistance+"");

取出前30条，最小的一个，ok实现匹配，完美。

问题4：opencv匹配的时候，图像是旋转的？

http://blog.youkuaiyun.com/qq_17121501/article/details/52142666 实现opencv摄像头旋转问题

问题5：过程中，需要安装opencvManager 支持库，比较恶心？
    
   public void onResume() {
super.onResume();

需要安装manager
//        OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_10, this,
//                mLoaderCallback);

不需要安装manager
mLoaderCallback.onManagerConnected( LoaderCallbackInterface.SUCCESS);
}

问题6：onCameraFrame是一帧一帧拿完就比对匹配的，所以比较卡，效果很差？

解决方案，写一个定时器1秒或几秒拿图片比对一次（让onCameraFrame一只取着图片）

//定时器，发送匹配的消息
TimerTask task = new TimerTask() {


@Override
public void run() {
Message message = new Message();
message.what = 0x110;
handler.sendMessage(message);
}
};
private void stopTimer(){
if (timer != null) {
timer.cancel();
timer = null;
}
if (task != null) {
task.cancel();
task = null;
}
}
public Handler handler = new Handler(){
@Override
public void handleMessage(Message msg) {


switch (msg.what){
case 0x110:
matchBitmap();
break;
case 0x111:
float distance= (float)msg.obj;
Toast.makeText(mContext,"匹配成功  "+distance,Toast.LENGTH_LONG).show();
finish();
break;
}
super.handleMessage(msg);
}
};



我的最终代码：
/**
* ClassName:PreviewImage
* Description TODO 比对图片相似度
* created by 漠天
* Data 2016/11/21 16:07
*/


public class PreviewImage extends Activity implements CameraBridgeViewBase.CvCameraViewListener2 {
static {
System.loadLibrary("opencv_java");
System.loadLibrary("nonfree");
}


private JavaCameraView mOpenCvCameraView;
private Mat mRgba;
private Mat mGray;
private Mat mByte;
private boolean isProcess = false;


private Bitmap mTestimg;
private Bitmap takeBitmap;
private Bitmap matchbitmap;


private Context mContext = null;
private Timer timer = new Timer();


@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.preview_image_layout);

mContext = this;
        mOpenCvCameraView = (JavaCameraView) findViewById(R.id.objectMatch);
        mOpenCvCameraView.setCvCameraViewListener(this);
        matchButton = (Button) findViewById(R.id.button_match);
        previewImage = (ImageView)findViewById(R.id.previewImage);
        showMatchText = (TextView)findViewById(R.id.showMatchText);
        Bitmap bitmap =  FileOperateUtil.getimage(com.motian.showcv.Utils.Utils.FileRoatePath);
        previewImage.setImageBitmap(bitmap);


        //设置边缘图片
        Mat img = new Mat();
        Utils.bitmapToMat(bitmap, img);
        Size dsize = new Size(img.width()*scale,img.height()*scale);
        Mat img2 = new Mat(dsize,CvType.CV_8SC1);
        Mat img3 = new Mat();
        img.convertTo(img2, CvType.CV_8SC1);
        Imgproc.Canny(img, img3, 123, 250);
        Utils.matToBitmap(img3,bitmap);
        previewImage.setImageBitmap(bitmap);
        previewImage.setAlpha(0.3f);


        matchButton.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                // show img in the Imageview
                isProcess = !isProcess;
                if(timer==null && CLICK_FLAG){
                    timer = new Timer();
                    timer.schedule(task,1000,1000);// 1s后执行task,经过1s再次执行
                    CLICK_FLAG = false;
                }
            }
        });

}


@Override
public void onCameraViewStarted(int width, int height) {
mRgba = new Mat(height, width, CvType.CV_8UC3);
mByte = new Mat(height, width, CvType.CV_8UC1);
}


@Override
public void onCameraViewStopped() {
mRgba.release();
mByte.release();
mGray.release();
mTestimg=null;
takeBitmap=null;
matchbitmap=null;
}


@Override
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
//一直在取图片，然后给定时器取出处理
mRgba = inputFrame.rgba();
mGray = inputFrame.gray();
return mRgba;
}


//定时器，发送匹配的消息
TimerTask task = new TimerTask() {


@Override
public void run() {
Message message = new Message();
message.what = 0x110;
handler.sendMessage(message);
}
};
private void stopTimer(){
if (timer != null) {
timer.cancel();
timer = null;
}
if (task != null) {
task.cancel();
task = null;
}
}
public Handler handler = new Handler(){
@Override
public void handleMessage(Message msg) {


switch (msg.what){
case 0x110:
matchBitmap();
break;
case 0x111:
float distance= (float)msg.obj;
Toast.makeText(mContext,"匹配成功  "+distance,Toast.LENGTH_LONG).show();
finish();
break;
}
super.handleMessage(msg);
}
};


/**
* 执行匹配
*/
public void matchBitmap(){
Mat testimage = new Mat();
Mat grayimage=new Mat();
MatOfDMatch matches = new MatOfDMatch();
MatOfKeyPoint keypoint_train = new MatOfKeyPoint();
MatOfKeyPoint keypoint_test = new MatOfKeyPoint();
KeyPoint kpoint = new KeyPoint();
Mat mask = Mat.zeros(mGray.size(), CvType.CV_8UC1);
Mat output = new Mat(); // Mat train=new Mat(); Mat
Mat test = new Mat();
Mat train = new Mat();
FeatureDetector detector_train = FeatureDetector.create(FeatureDetector.ORB);
detector_train.detect(mGray, keypoint_train);
// Features2d.drawKeypoints(mGray, keypoint_train, output, new Scalar(
// 2, 254, 255), Features2d.DRAW_RICH_KEYPOINTS);


DescriptorExtractor descriptor_train = DescriptorExtractor.create(DescriptorExtractor.ORB);
descriptor_train.compute(mGray, keypoint_train, train);
takeBitmap =  FileOperateUtil.getimage(com.motian.showcv.Utils.Utils.FileRoatePath);
try {
mTestimg = Bitmap.createScaledBitmap(takeBitmap, mGray.width(), mGray.height(), false);
}catch (Exception e){
Log.e("e","width or height error");
return;
}
mTestimg = Bitmap.createScaledBitmap(takeBitmap, mGray.width(), mGray.height(), false);
Utils.bitmapToMat(mTestimg, testimage);
Imgproc.cvtColor(testimage, grayimage, Imgproc.COLOR_RGB2GRAY);


FeatureDetector detector_test = FeatureDetector.create(FeatureDetector.ORB);
detector_test.detect(grayimage, keypoint_test);


// Features2d.drawKeypoints(testimage, keypoint_test, output,
// new Scalar(2, 254, 255), Features2d.DRAW_RICH_KEYPOINTS);
DescriptorExtractor descriptor_test = DescriptorExtractor.create(DescriptorExtractor.ORB);
descriptor_test.compute(grayimage, keypoint_test, test);
DescriptorMatcher descriptormatcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
try {
descriptormatcher.match(test, train, matches);
}catch (Exception e){
Log.e("e","match error");
return;
}
Features2d.drawMatches(grayimage,keypoint_test,mGray, keypoint_train, matches, output);
matchbitmap=Bitmap.createScaledBitmap(takeBitmap, output.width(),  output.height(), false);
Utils.matToBitmap(output, matchbitmap);


List<DMatch> list = matches.toList().subList(0,29);
float minDistance = 100;
for(int i=0;i<list.size();i++){
minDistance = Math.min(minDistance,list.get(i).distance);
}
Log.e("dmatch",minDistance+"");


if(minDistance<30.0){
Message message = new Message();
message.what = 0x111;
message.obj = minDistance;
handler.sendMessage(message);


stopTimer();
}
}
private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {
@Override
public void onManagerConnected(int status) {
switch (status) {
case LoaderCallbackInterface.SUCCESS: {
mOpenCvCameraView.enableView();
}
break;
default: {
super.onManagerConnected(status);
}
break;
}
}
};
@Override
protected void onPause() {
super.onPause();
if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();


}


public void onResume() {
super.onResume();
//        OpenCVLoader.initAsync(OpenCVLoader.OPENCV_VERSION_2_4_10, this,
//                mLoaderCallback);
mLoaderCallback.onManagerConnected( LoaderCallbackInterface.SUCCESS);
}


@Override
protected void onDestroy() {
super.onDestroy();
mRgba.release();
mByte.release();
mGray.release();
mTestimg=null;
takeBitmap=null;
matchbitmap=null;
if (mOpenCvCameraView != null)
mOpenCvCameraView.disableView();


}


}

（第一次写文章分享自己的东西，不喜勿喷！）

源码demo

补充：

再次使用，比对崩溃，error log：

descriptormatcher.match(test, train, matches);

try{}catch{} 捕获不到

解决：

if(train.empty()){
    //异常的原因是：预览图片是空白或者纯色
    Log.e("match","match error");
    return;
}
descriptormatcher.match(test, train, matches);