几何视觉的编程实践——相机参数计算——基于ipyvolume的3D可视化——透视投影

本文主要侧重上手实践，理论部分可以先参考其他文章学习

---

前言

本次实践是几何视觉的编程实践，是对计算机视觉课程的一次巩固复习，从中查缺补漏完善知识体系。主要实现了相机内外参的计算，标定板的三维可视化，最后还添加新的模型实测透视效果。

使用的是jupterlab的环境，基于ipyvolume实现的3D可视化

图片数据和源码文件在此下载：几何视觉视觉代码+图片

一、环境配置

1.JupyterLab

jupterlab可以在Anaconda Navigator中打开，也可以使用终端打开。

如图，点击Launch即可启动，界面和jupyter类似。

2.安装包

想要实现3D效果，需要提前导入ipyvolume和matplotlib库；
如图在base环境下搜索安装ipyvolume，选择安装0.6.0a8版本
（旧版本可能会无法运行，尽量升级到最新版）

matplotlib库也是类似，我使用的版本是matplotlib==3.5.0

3.数据准备

提取准备好带有标定板的十张图片，以及chessboard

二、代码实现

1.相机校准得到相机内外参

camera_calibration.py 代码如下：

import matplotlib.pyplot as plt
import ipyvolume as ipv
import numpy as np
import cv2

def calibrate_cameras() :
    """ Calibrates cameras from chessboard images. 
    
    Returns: 
        images (list[np.ndarray]): Images containing the chessboard.
        intrinsics (np.ndarray): An upper triangular 4x4 full-rank matrix containing camera intrinsics.
        distortions (np.ndarray): Radial distortion coefficients.
        rotation_vectors (list[np.ndarray]): Rodrigues rotation vectors.
        translation_vectors (list[np.ndarray]): Translation vectors.
        object_points: (np.ndarray): A (4, 54) point array, representing the [x,y,z,w]
            of 54 chessboard points (homogenous coordiantes).
    """    
    
    # Read images
    images = list() 
    for i in range(11): 
        img = cv2.imread(f'./images/{
     
     i}.jpg')
        img = cv2.resize(img, None, fx=0.25, fy=0.25)
        images.append(img)     
                
    # chessboard corner search with opencv.
    shape = (6, 9) # The default opencv chessboard has 6 rows, 9 columns 
    object_points_all = [] # 3D world coordinates
    image_points_all = [] # 2D image coordinates
    flags = cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_FAST_CHECK + cv2.CALIB_CB_NORMALIZE_IMAGE
    refinement_criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)    
    
    # For each image, store the object points and image points of chessboard corners.对于每个图像，存储棋盘角的对象点和图像点。
    images_filtered = list()    
    object_points = np.zeros((1, shape[0] * shape[1], 3), np.float32)
    object_points[0, :, :2] = np.mgrid[0:shape[0], 0:shape[1]].T.reshape(-1, 2)
    for idx, image in enumerate(images): 
        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        succes, corners = cv2.findChessboardCorners(image=gray, 
                                                    patternSize=shape,
                                                    flags=flags) 
        if succes:
            images_filtered.append(image)
            corners = cv2.cornerSubPix(image=gray, 
                                       corners=corners, 
                                       winSize=(11,11), 
                                       zeroZone=(-1,-1), 
                                       criteria=refinement_criteria)
            object_points_all.append(object_points)
            image_points_all.append(corners)    
            
    images = images_filtered
    
    # Calibrate the cameras by using the 3D <-> 2D point correspondences.使用3D<->2D点对应关系校准相机。
    ret, intrinsics, distortions, rotation_vectors, translation_vectors = cv2.calibrateCamera(object_points_all, image_points_all, gray.shape[::-1], None, None)
        
    intrinsics = np.hstack([intrinsics, np.zeros((3, 1))])
    intrinsics = np.vstack([intrinsics, [[0, 0, 0, 1]]])

    # Convert chessboard object points to homogeneous coordinates for later use.将棋盘对象点转换为同质坐标以供以后使用。
    object_points = object_points[0].reshape((-1, 3)).T
    object_points = np.vstack([object_points, np.ones((1, object_points.shape[1]))])
    
    return images, intrinsics, distortions, rotation_vectors, translation_vectors, object_points

if __name__ == "__main__":

    # camera calibration
    images, intrinsics, distortions, rotation_vectors, translation_vectors, object_points = calibrate_cameras()

    print(f'\n\nNumber of calibration images: {
     
     len(images)}')
    print(f'Number of calibration points: {
     
     object_points.shape[1]}')
    print(f'\n\nSample of imags used')
    plt.figure(figsize=(20,20))
    
    _ = plt.imshow(cv2.hconcat([cv2.cvtColor(im, cv2.COLOR_RGB2BGR) for im in images[:5]]))    

结果如图：

2.对棋盘标定板和相机中心进行三维可视化

camera_center.py代码：

import matplotlib.pyplot as plt
import ipyvolume as ipv
import numpy as np
import cv2

cam_sphere_size = 1

#from camera_calibration import calibrate_cameras

def calibrate_cameras() :
    """ Calibrates cameras from chessboard images. 
    
    Returns: 
        images (list[np.ndarray]): Images containing the chessboard.
        intrinsics (np.ndarray): An upper triangular 4x4 full-rank matrix containing camera intrinsics.
        distortions (np.ndarray): Radial distortion coefficients.
        rotation_vectors (list[np.ndarray]): Rodrigues rotation vectors.
        translation_vectors (list[np.ndarray]): Translation vectors.
        object_points: (np.ndarray): A (4, 54) point array, representing the [x,y,z,w]
            of 54 chessboard points (homogenous coordiantes).
    """    
    
    # Read images
    images = list() 
    for i in range(11): 
        img = cv2.imread(f'./images/{
     
     i}.jpg')
        img = cv2.resize(img, None, fx=0.25, fy=0.25)
        images.append(img)     
                
    # chessboard corner search with opencv.
    shape = (6, 9) # The default opencv chessboard has 6 rows, 9 columns 
    object_points_all = [] # 3D world coordinates
    image_points_all = [] # 2D image coordinates
    flags = cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_FAST_CHECK + cv2.CALIB_CB_NORMALIZE_IMAGE
    refinement_criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)    
    
    # For each image, store the object points and image points of chessboard corners.
    images_filtered = list()    
    object_points = np.zeros((1, shape[0] * shape[1], 3), np.float32)
    object_points[0, :, :2] = np.mgrid[0:shape[0], 0:shape[1]].T.reshape(-1, 2)
    for idx, image in enumerate(images): 
        gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        succes, corners = cv2.findChessboardCorners(image=gray, 
                                                    patternSize=shape,
                                                    flags=flags) 
        if succes:
            images_filtered.append(image)
            corners = cv2.cornerSubPix(image=gray, 
                                       corners=corners, 
                                       winSize=(11,11), 
                                       zeroZone=(-1,-1), 
                                       criteria=refinement_criteria)
            object_points_all.append(object_points)
            image_points_all.append(corners)    
            
    images = images_filtered
    
    # Calibrate the cameras by using the 3D <-> 2D point correspondences.
    ret, intrinsics, distortions, rotation_vectors, translation_vectors = cv2.calibrateCamera(object_points_all, image_points_all, gray.shape[::-1], None, None)
        
    intrinsics = np.hstack([intrinsics, np.zeros((3, 1))])
    intrinsics = np.vstack([intrinsics, [[0, 0, 0, 1]]])

    # Convert chessboard object points to homogeneous coordinates for later use.
    object_points = object_points[0].reshape((-1, 3)).T
    object_points = np.vstack([object_points, np.ones((1, object_points.shape[1]))])
    
    return images, intrinsics, distortions, rotation_vectors, translation_vectors, object_points


def extrinsics_from_calibration(rotation_vectors, translation_vectors):
    """ Calculates extrinsic matrices from calibration output. 
        
    Args: 
        rotation_vectors (list[np.ndarray]): Rodrigues rotation vectors.
        translation_vectors (list[np.ndarray]): Translation vectors.
    Returns: 
        extrinsics (list[np.ndarray]): A list of camera extrinsic matrices.
            These matrices are 4x4 full-rank.
    """
    
    rotation_matrices = list() 
    for rot in rotation_vectors:
        rotation_matrices.append(cv2.Rodrigues(rot