opencv学习-智能视觉作业(二)

这是智能视觉采集大作业的主要程序

• 目的:从视频拍摄中得到全景图,包括离线的和在线的.
• 过程:特征点提取匹配-图片射影变换-图片合成
  (1)特征点提取匹配:ORB算法
  (2)图片合成的方法分为两种,第一种是离线的,第二种是在线的(个人提出的拙劣的算法).

1.离线合成

离线部分和在线部分有一个不同的是图片合成算法.

• 对于在线合成,速度时比较重要的,所以这里对于前后两帧的图片采用
  I m g t ( i , j ) = m a x { n e w I m g t ( i , j ) , I m g t − 1 ( i , j ) } Img_t(i,j)=max\{newImg_t(i,j),Img_{t-1}(i,j)\} Imgt​(i,j)=max{newImgt​(i,j),Imgt−1​(i,j)}
  采用这种算法的原因是,新的图像经过射影变换后,在图片上会留有黑色的空位.需要通过依旧求得的全景图的部分对其进行填充.
• 对于离线合成,精确度是比较重要的,所以这里采用下述算法
  I m g t ( i , j ) = { m a x { n e w I m g t ( i , j ) , I m g t − 1 ( i , j ) } , n e w I m g t ( i , j ) < k α n e w I m g t ( i , j ) + β I m g t − 1 ( i , j ) , n e w I m g t ( i , j ) > = k Img_t(i,j)=\begin{cases}max\{newImg_t(i,j),Img_{t-1}(i,j)\},&newImg_t(i,j)<k \\ \alpha newImg_t(i,j)+\beta Img_{t-1}(i,j),&newImg_t(i,j)>=k \end{cases} Imgt​(i,j)={max{newImgt​(i,j),Imgt−1​(i,j)},αnewImgt​(i,j)+βImgt−1​(i,j),​newImgt​(i,j)<knewImgt​(i,j)>=k​
  其中$\alpha +\beta =1$
  由于图片经过变换后可能会产生部分没有对齐的情况,如果仍然采用离线的算法,将会导致黑色的物体在图像中渐渐的被掩盖掉(最典型的就是头发在全景图的生成过程中渐渐的消失).所以一般来说,在全景图的生成过程中都会使用新图和旧图乘一个系数合成的方法.同时我们仍然需要考虑到,新图经过射影变换后留有黑色空位的情况.所以,在该算法中,需要对新图的某个像素是黑色空位还是其他的情况进行区分,即设一个阈值k,用新图的像素与阈值进行比较.小于阈值,则进行加系数融合;如果大于阈值,则取最大值,消去黑色空位.

离线部分代码

#include <iostream>
#include <sstream>
#include <string>
#include <ctime>
#include <cstdio>
#include <vector>
#include <opencv2/opencv.hpp>
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>

#include <opencv2/video/background_segm.hpp>

using namespace cv;
using namespace std;
int main(){
    VideoCapture input=VideoCapture("input5.mp4");
    if(input.isOpened()){
        cout<<"input sucess"<<endl;
    }
    else{
        cout<<"fail input"<<endl;
        return 0;
    }
    Mat view; 
    int cnt=0;
    vector<Mat> calibrate_images;
    vector<Mat> calibrate_descriptors;
    vector<vector<KeyPoint> > calibrate_keypoints;
    Mat dst(720+100,1280*6, CV_8UC3);dst.setTo(0);
    Mat save_homo;
    while(1){
        if(!input.read(view)) break;
        //cout<<view.size<<endl;
        std::vector<KeyPoint> keypoints;
        Mat descriptors;
        Ptr<FeatureDetector> detector = ORB::create();
        Ptr<DescriptorExtractor> descriptor = ORB::create();
    //Ptr<DescriptorMatcher> matcher  = DescriptorMatcher::create ( "BruteForce-Hamming" );
        detector->detect ( view,keypoints );
        descriptor->compute ( view, keypoints, descriptors );
        Mat outimg;
        calibrate_keypoints.push_back(keypoints);
        calibrate_descriptors.push_back(descriptors);
        drawKeypoints( view, keypoints, outimg, Scalar::all(-1), DrawMatchesFlags::DEFAULT );
        //imshow("ORB特征点",outimg);
        //imshow("input", view);
        //cv::waitKey(10000);
        
        calibrate_images.push_back(view);
        if(view.rows==0) break;
        if(cnt>=1){
            //the first 15 images will be used calidate.
            Ptr<DescriptorMatcher> matcher  = DescriptorMatcher::create ( "BruteForce-Hamming" );
            vector<DMatch> matches;
            Mat img_match;
    //BFMatcher matcher ( NORM_HAMMING );
            matcher->match ( calibrate_descriptors[cnt-1],  calibrate_descriptors[cnt], matches );
            //optimize
            double min_dist=10000, max_dist=0;
            //找出所有匹配之间的最小距离和最大距离, 即是最相似的和最不相似的两组点之间的距离
            for ( int i = 0; i < calibrate_descriptors[cnt-1].rows; i++ )
            {
                double dist = matches[i].distance;
                if ( dist < min_dist ) min_dist = dist;
                if ( dist > max_dist ) max_dist = dist;
            }
            std::vector< DMatch > good_matches;
            for ( int i = 0; i < calibrate_descriptors[cnt-1].rows; i++ )
            {
                if ( matches[i].distance <= max ( 2*min_dist, 30.0 ) )
                {
                    good_matches.push_back ( matches[i] );
                }
            }
            drawMatches ( calibrate_images[cnt-1], calibrate_keypoints[cnt-1], calibrate_images[cnt], calibrate_keypoints[cnt], good_matches, img_match );
            //imshow("ORB特征点2",img_match);
            
            //calibrate here
            //**********************************************//
            vector<Point2f> imagePoints1, imagePoints2;

            for (int i = 0; i<good_matches.size(); i++)
            {
                imagePoints2.push_back(calibrate_keypoints[cnt-1][good_matches[i].queryIdx].pt);
                imagePoints1.push_back(calibrate_keypoints[cnt][good_matches[i].trainIdx].pt);
            }

            Mat homo = findHomography(imagePoints1, imagePoints2, CV_RANSAC);
            if(cnt==1) save_homo=homo.clone();
            else save_homo=save_homo*homo;
            homo=save_homo;

           // cout<<homo<<endl;
            Mat imageTransform;
            warpPerspective(calibrate_images[cnt], imageTransform, homo, Size(calibrate_images[cnt].cols+1000, calibrate_images[cnt].rows+100));
            //imshow("直接经过透视矩阵变换", imageTransform);
            //imwrite("trans1.jpg", imageTransform);
            int dst_width = imageTransform.cols;  //取最右点的长度为拼接图的长度
            
            int dst_height =calibrate_images[cnt].rows;
            
  
            Mat tem_dst(720+100,1280*6, CV_8UC3);tem_dst.setTo(0);
            imageTransform.copyTo(tem_dst(Rect(0, 0, imageTransform.cols, imageTransform.rows)));
            Mat tem_dst2=dst.clone();
            //calibrate_images[cnt].copyTo(dst(Rect(0, 0, calibrate_images[cnt].cols, calibrate_images[cnt].rows)));
            //max(tem_dst,tem_dst2,dst);
            //addWeighted(tem_dst, 0.1,tem_dst2,0.9, 0, dst);
            for (int j = 0; j < tem_dst.rows; j++) {
                for (int k = 0; k < tem_dst.cols; k++) {
                    for (int m = 0; m < 3; m++) {
                        if (tem_dst.at<Vec3b>(j, k)[m] <= 130) {
                            dst.at<Vec3b>(j, k)[m] = max(tem_dst2.at<Vec3b>(j, k)[m], tem_dst.at<Vec3b>(j, k)[m]);
                        }
                        else {
                            dst.at<Vec3b>(j, k)[m] = (tem_dst2.at<Vec3b>(j, k)[m]*5+ tem_dst.at<Vec3b>(j, k)[m]*5)/10;
                        }
                    }
//								cout<< tem_dst.at
                    //if(tem_dst.at)
                }
            }
        imshow("b_dst", dst);cv::waitKey(1000);    
        }
        
        for(int i=0;i<5;i++){
            input.read(view);
        }
        cnt++;
        
    }
    
    cv::waitKey(1000000);
    cout<<cnt<<endl;
}

2.在线合成

在线合成前面已经提到了图像融合的算法.除此之外,在线和成主要是使用basler相机的代码.代码如下

#include <pylon/PylonIncludes.h>
#ifdef PYLON_WIN_BUILD
#    include <pylon/PylonGUI.h>
#endif
#include <opencv2/opencv.hpp>

// Namespace for using pylon objects.
using namespace Pylon;

// Namespace for using cout.
using namespace std;
using namespace cv;

// Number of images to be grabbed.
static const uint32_t c_countOfImagesToGrab = 1000;

int main(int argc, char* argv[])
{
	// The exit code of the sample application.
	int exitCode = 0;

	// Before using any pylon methods, the pylon runtime must be initialized. 
	PylonInitialize();
	Mat frame;
	//****************************************************
	//[2592 x 1944]
	Mat view;
	int cnt = 0;
	vector<Mat> calibrate_images;
	vector<Mat> calibrate_descriptors;
	vector<vector<KeyPoint> > calibrate_keypoints;
	Mat dst(194 + 50, 259 * 10, CV_8UC3); dst.setTo(0);
	Mat save_homo;
	//****************************************************
	try
	{
		// Create an instant camera object with the camera device found first.
		CInstantCamera camera(CTlFactory::GetInstance().CreateFirstDevice());

		// Print the model name of the camera.
		cout << "Using device " << camera.GetDeviceInfo().GetModelName() << endl;

		// The parameter MaxNumBuffer can be used to control the count of buffers
		// allocated for grabbing. The default value of this parameter is 10.
		camera.MaxNumBuffer = 5;

		// Start the grabbing of c_countOfImagesToGrab images.
		// The camera device is parameterized with a default configuration which
		// sets up free-running continuous acquisition.
		camera.StartGrabbing(c_countOfImagesToGrab);

		// This smart pointer will receive the grab result data.
		CGrabResultPtr ptrGrabResult;
		
		/// new image that convert to cv::Mat
		CImageFormatConverter formatConverter;
		formatConverter.OutputPixelFormat = PixelType_BGR8packed;
		CPylonImage pylonImage;
		
		// Camera.StopGrabbing() is called automatically by the RetrieveResult() method
		// when c_countOfImagesToGrab images have been retrieved.
		char c;
		while (c = waitKey(1) && camera.IsGrabbing())
		{
			// Wait for an image and then retrieve it. A timeout of 5000 ms is used.
			camera.RetrieveResult(5000, ptrGrabResult, TimeoutHandling_ThrowException);

			// Image grabbed successfully?
			if (ptrGrabResult->GrabSucceeded())
			{
				// Access the image data.
				cout << "SizeX: " << ptrGrabResult->GetWidth() << endl;
				cout << "SizeY: " << ptrGrabResult->GetHeight() << endl;
				const uint8_t *pImageBuffer = (uint8_t *)ptrGrabResult->GetBuffer();
				cout << "Gray value of first pixel: " << (uint32_t)pImageBuffer[0] << endl << endl;

//#ifdef PYLON_WIN_BUILD
				// Display the grabbed image.
				//Pylon::DisplayImage(1, ptrGrabResult);
//#endif
				///convert to cv::Mat
				formatConverter.Convert(pylonImage, ptrGrabResult);
				frame = cv::Mat(ptrGrabResult->GetHeight(), ptrGrabResult->GetWidth(), CV_8UC3, (uint8_t *)pylonImage.GetBuffer());
				/// show
				//resize(frame, frame, Size(frame.cols / 2, frame.rows / 2));
				cout << frame.size() << endl;
				imshow("OpenCV Display Window", frame);
				if(true)
				{
					Mat view(194, 259, frame.type());
					resize(frame, view, view.size(), 0, 0, INTER_LINEAR);
					//*****************************************************************************8
					std::vector<KeyPoint> keypoints;
					keypoints.resize(500);
					Mat descriptors;
					Ptr<FeatureDetector> detector = ORB::create();
					Ptr<DescriptorExtractor> descriptor = ORB::create();
					//Ptr<DescriptorMatcher> matcher  = DescriptorMatcher::create ( "BruteForce-Hamming" );
					cout << view.size << endl;
					detector->detect(view, keypoints);
					descriptor->compute(view, keypoints, descriptors);
					Mat outimg;
					calibrate_keypoints.push_back(keypoints);
					calibrate_descriptors.push_back(descriptors);
					drawKeypoints(view, keypoints, outimg, Scalar::all(-1), DrawMatchesFlags::DEFAULT);
					//imshow("ORB特征点",outimg);
					//imshow("input", view);
					//cv::waitKey(10000);

					calibrate_images.push_back(view);
					if (cnt >= 1) {
						//the first 15 images will be used calidate.
						Ptr<DescriptorMatcher> matcher = DescriptorMatcher::create("BruteForce-Hamming");
						vector<DMatch> matches;
						Mat img_match;
						//BFMatcher matcher ( NORM_HAMMING );
						matcher->match(calibrate_descriptors[cnt - 1], calibrate_descriptors[cnt], matches);
						//optimize
						double min_dist = 10000, max_dist = 0;
						//找出所有匹配之间的最小距离和最大距离, 即是最相似的和最不相似的两组点之间的距离
						for (int i = 0; i < calibrate_descriptors[cnt - 1].rows; i++)
						{
							double dist = matches[i].distance;
							if (dist < min_dist) min_dist = dist;
							if (dist > max_dist) max_dist = dist;
						}
						std::vector< DMatch > good_matches;
						for (int i = 0; i < calibrate_descriptors[cnt - 1].rows; i++)
						{
							if (matches[i].distance <= max(2 * min_dist, 30.0))
							{
								good_matches.push_back(matches[i]);
							}
						}
						drawMatches(calibrate_images[cnt - 1], calibrate_keypoints[cnt - 1], calibrate_images[cnt], calibrate_keypoints[cnt], good_matches, img_match);
						//imshow("ORB特征点2",img_match);

						//calibrate here
						//**********************************************//
						vector<Point2f> imagePoints1, imagePoints2;

						for (int i = 0; i < good_matches.size(); i++)
						{
							imagePoints2.push_back(calibrate_keypoints[cnt - 1][good_matches[i].queryIdx].pt);
							imagePoints1.push_back(calibrate_keypoints[cnt][good_matches[i].trainIdx].pt);
						}

						Mat homo = findHomography(imagePoints1, imagePoints2, CV_RANSAC);
						if (cnt == 1) save_homo = homo.clone();
						else save_homo = save_homo * homo;
						homo = save_homo;

						// cout<<homo<<endl;
						Mat imageTransform;
						warpPerspective(calibrate_images[cnt], imageTransform, homo, Size(calibrate_images[cnt].cols + 1000, calibrate_images[cnt].rows + 50));
						//imshow("直接经过透视矩阵变换", imageTransform);
						//imwrite("trans1.jpg", imageTransform);
						int dst_width = imageTransform.cols;  //取最右点的长度为拼接图的长度

						int dst_height = calibrate_images[cnt].rows;
	

						Mat tem_dst(194 + 50,  259 * 10, CV_8UC3); tem_dst.setTo(0);
						imageTransform.copyTo(tem_dst(Rect(0, 0, imageTransform.cols, imageTransform.rows)));
						Mat tem_dst2 = dst.clone();
						//calibrate_images[cnt].copyTo(dst(Rect(0, 0, calibrate_images[cnt].cols, calibrate_images[cnt].rows)));
						max(tem_dst, tem_dst2, dst);
						/*for (int j = 0; j < tem_dst.rows; j++) {
							for (int k = 0; k < tem_dst.cols; k++) {
								for (int m = 0; m < 3; m++) {
									if (tem_dst.at<Vec3b>(j, k)[m] <= 10) {
										dst.at<Vec3b>(j, k)[m] = max(tem_dst2.at<Vec3b>(j, k)[m], tem_dst.at<Vec3b>(j, k)[m]);
									}
									else {
										dst.at<Vec3b>(j, k)[m] = tem_dst2.at<Vec3b>(j, k)[m]*0.3+ tem_dst.at<Vec3b>(j, k)[m]*0.7;
									}
								}
//								cout<< tem_dst.at
								//if(tem_dst.at)
							}
						}
						*/
						//max(tem_dst,0)
						//cout << dst << endl;

						//addWeighted(tem_dst, 0.3,tem_dst2,0.7, 0, dst);

						imshow("b_dst", dst);
					}
					
				}
					cnt++;
	
				
				//******************************************************************************
			}
			else
			{
				cout << "Error: " << ptrGrabResult->GetErrorCode() << " " << ptrGrabResult->GetErrorDescription() << endl;
			}
		}
	}
	catch (const GenericException &e)
	{
		// Error handling.
		cerr << "An exception occurred." << endl
			<< e.GetDescription() << endl;
		exitCode = 1;
	}

	// Comment the following two lines to disable waiting on exit.
	cerr << endl << "Press Enter to exit." << endl;
	while (cin.get() != '\n');

	// Releases all pylon resources. 
	PylonTerminate();

	return exitCode;
}

3结果

4.硬件部分

还在搭建中