EvTexture自制数据集

pip install pybind11

pip install .

 发现没有安装pybind11：

安装pybind11：(Ubuntu pybind11教程_ubuntu 编译pybind11 源码-优快云博客)

a.先下载pybind11的源代码：https://github.com/pybind/pybind11

2）编译安装

（pybind11的安装以及库的使用（ubuntu20.14）_ubuntu安装pybind11-优快云博客）

cd  pybind11
mkdir build
cd build
cmake ..
make check -j 4 
sudo make install  #(如果使用python2需要禁用/usr/bin/下的python3)

在这里检查pybind11是否有安装好：

再pip install .    这次是没有opencv:

检查opencv是否安装：pkg-config --modversion opencv

没有装opencv

安装opencv：（https://zhuanlan.zhihu.com/p/647779108）

sudo apt-get install libopencv-dev

再pip install . 

再装boost (在Ubuntu上安装Boost的五种方法(全网最全，建议收藏)_ubuntu安装boost-优快云博客)

sudo apt-get install libboost-all-dev

 再pip install . 

这次终于成功了

然后要生成timestamps.txt

0.04秒的间隔。

降低libffi的版本：（源码编译、安装libffi_libffi源码-优快云博客）

conda install libffi==3.3

生成.h文件

运行python ./02_event_packagers.py

from abc import ABCMeta, abstractmethod
import h5py
import cv2 as cv
import numpy as np
import torch
import bisect
import math
import random
import esim_py
import cv2
import os

def events_to_voxel_torch(xs, ys, ts, ps, B, device=None, sensor_size=(180, 240), temporal_bilinear=True):
    """
    Turn set of events to a voxel grid tensor, using temporal bilinear interpolation
    @param xs List of event x coordinates (torch tensor)
    @param ys List of event y coordinates (torch tensor)
    @param ts List of event timestamps (torch tensor)
    @param ps List of event polarities (torch tensor)
    @param B Number of bins in output voxel grids (int)
    @param device Device to put voxel grid. If left empty, same device as events
    @param sensor_size The size of the event sensor/output voxels
    @param temporal_bilinear Whether the events should be naively
        accumulated to the voxels (faster), or properly
        temporally distributed
    @returns Voxel of the events between t0 and t1
    """
    if device is None:
        device = xs.device
    assert(len(xs)==len(ys) and len(ys)==len(ts) and len(ts)==len(ps))
    bins = []
    dt = ts[-1]-ts[0]
    t_norm = (ts-ts[0])/dt*(B-1)
    zeros = torch.zeros(t_norm.size())
    for bi in range(B):
        if temporal_bilinear:
            bilinear_weights = torch.max(zeros, 1.0-torch.abs(t_norm-bi))
            weights = ps*bilinear_weights
            vb = events_to_image_torch(xs, ys,
                    weights, device, sensor_size=sensor_size,
                    clip_out_of_range=False)
        else:
            tstart = t[0] + dt*bi
            tend = tstart + dt
            beg = binary_search_torch_tensor(t, 0, len(ts)-1, tstart)
            end = binary_search_torch_tensor(t, 0, len(ts)-1, tend)
            vb = events_to_image_torch(xs[beg:end], ys[beg:end],
                    ps[beg:end], device, sensor_size=sensor_size,
                    clip_out_of_range=False)
        bins.append(vb)
    bins = torch.stack(bins)
    return bins

def events_to_image_torch(xs, ys, ps,
        device=None, sensor_size=(180, 240), clip_out_of_range=True,
        interpolation=None, padding=True, default=0):
    """
    Method to turn event tensor to image. Allows for bilinear interpolation.
    @param xs Tensor of x coords of events
    @param ys Tensor of y coords of events
    @param ps Tensor of event polarities/weights
    @param device The device on which the image is. If none, set to events device
    @param sensor_size The size of the image sensor/output image
    @param clip_out_of_range If the events go beyond the desired image size,
       clip the events to fit into the image
    @param interpolation Which interpolation to use. Options=None,'bilinear'
    @param padding If bilinear interpolation, allow padding the image by 1 to allow events to fit:
    @returns Event image from the events
    """
    if device is None:
        device = xs.device
    if interpolation == 'bilinear' and padding:
        img_size = (sensor_size[0]+1, sensor_size[1]+1)
    else:
        img_size = list(sensor_size)

    mask = torch.ones(xs.size(), device=device)
    if clip_out_of_range:
        zero_v = torch.tensor([0.], device=device)
        ones_v = torch.tensor([1.], device=device)
        clipx = img_size[1] if interpolation is None and padding==False else img_size[1]-1
        clipy = img_size[0] if interpolation is None and padding==False else img_size[0]-1
        mask = torch.where(xs>=clipx, zero_v, ones_v)*torch.where(ys>=clipy, zero_v, ones_v)

    img = (torch.ones(img_size)*default).to(device)
    if interpolation == 'bilinear' and xs.dtype is not torch.long and xs.dtype is not torch.long:
        pxs = (xs.floor()).float()
        pys = (ys.floor()).float()
        dxs = (xs-pxs).float()
        dys = (ys-pys).float()
        pxs = (pxs*mask).long()
        pys = (pys*mask).long()
        masked_ps = ps.squeeze()*mask
        interpolate_to_image(pxs, pys, dxs, dys, masked_ps, img)
    else:
        if xs.dtype is not torch.long:
            xs = xs.long().to(device)
        if ys.dtype is not torch.long:
            ys = ys.long().to(device)
        try:
            mask = mask.long().to(device)
            xs, ys = xs*mask, ys*mask
            img.index_put_((ys, xs), ps, accumulate=True)
        except Exception as e:
            print("Unable to put tensor {} positions ({}, {}) into {}. Range = {},{}".format(
                ps.shape, ys.shape, xs.shape, img.shape,  torch.max(ys), torch.max(xs)))
            raise e
    return img


def interpolate_to_image(pxs, pys, dxs, dys, weights, img):
    """
    Accumulate x and y coords to an image using bilinear interpolation
    @param pxs Numpy array of integer typecast x coords of events
    @param pys Numpy array of integer typecast y coords of events
    @param dxs Numpy array of residual difference between x coord and int(x coord)
    @param dys Numpy array of residual difference between y coord and int(y coord)
    @returns Image
    """
    img.index_put_((pys,   pxs  ), weights*(1.0-dxs)*(1.0-dys), accumulate=True)
    img.index_put_((pys,   pxs+1), weights*dxs*(1.0-dys), accumulate=True)
    img.index_put_((pys+1, pxs  ), weights*(1.0-dxs)*dys, accumulate=True)
    img.index_put_((pys+1, pxs+1), weights*dxs*dys, accumulate=True)
    return img


def voxel_normalization(voxel):
    """
     normalize the voxel same as https://arxiv.org/abs/1912.01584 Section 3.1
     Params:
         voxel: torch.Tensor, shape is [num_bins, H, W]

     return:
         normalized voxel
    """
 	# check if voxel all element is 0
    a,b,c = voxel.shape
    tmp = torch.zeros(a, b, c)
    if torch.equal(voxel, tmp):
        return voxel
    abs_voxel, _ = torch.sort(torch.abs(voxel).view(-1, 1).squeeze(1))
    first_non_zero_idx = torch.nonzero(abs_voxel)[0].item()
    non_zero_voxel = abs_voxel[first_non_zero_idx:]
    norm_idx = math.floor(non_zero_voxel.shape[0] * 0.98)
    ones = torch.ones_like(voxel)
    normed_voxel = torch.where(torch.abs(voxel) < non_zero_voxel[norm_idx], voxel / non_zero_voxel[norm_idx], voxel)
    normed_voxel = torch.where(normed_voxel >= non_zero_voxel[norm_idx], ones, normed_voxel)
    normed_voxel = torch.where(normed_voxel <= -non_zero_voxel[norm_idx], -ones, normed_voxel)
    return normed_voxel


class packager():

    __metaclass__ = ABCMeta

    def __init__(self, name, output_path, max_buffer_size=1000000):
        self.name = name
        self.output_path = output_path
        self.max_buffer_size = max_buffer_size

    @abstractmethod
    def package_events(self, xs, ys, ts, ps):
        pass

    @abstractmethod
    def package_image(self, frame, timestamp):
        pass

    @abstractmethod
    def package_flow(self, flow, timestamp):
        pass

    @abstractmethod
    def add_metadata(self, num_events, num_pos, num_neg,
            duration, t0, tk, num_imgs, num_flow):
        pass

    @abstractmethod
    def set_data_available(self, num_images, num_flow):
        pass

class hdf5_packager(packager):
    """
    This class packages data to hdf5 files
    """
    def __init__(self, output_path, max_buffer_size=1000000):
        packager.__init__(self, 'hdf5', output_path, max_buffer_size)
        print("CREATING FILE IN {}".format(output_path))
        self.events_file = h5py.File(output_path, 'w')
        self.event_xs = self.events_file.create_dataset("events/xs", (0, ), dtype=np.dtype(np.int16), maxshape=(None, ), chunks=True)
        self.event_ys = self.events_file.create_dataset("events/ys", (0, ), dtype=np.dtype(np.int16), maxshape=(None, ), chunks=True)
        self.event_ts = self.events_file.create_dataset("events/ts", (0, ), dtype=np.dtype(np.float64), maxshape=(None, ), chunks=True)
        self.event_ps = self.events_file.create_dataset("events/ps", (0, ), dtype=np.dtype(np.bool_), maxshape=(None, ), chunks=True)

    def package_bidirectional_event_voxels(self, x, y, t, p, timestamp_list, backward, bins, sensor_size, h5_name, error_txt):
        """
            params:
                x: ndarray, x-position of events
                y: ndarray, y-position of events
                t: ndarray, timestamp of events
                p: ndarray, polarity of events
                backward: bool, if forward or backward
                timestamp_list: list, to split events via timestamp
                bins: voxel num_bins
            returns:
                no return.
        """
        # Step 1: convert data type
        assert x.shape == y.shape == t.shape == p.shape

        # x = torch.from_numpy(x.dtype(np.int16))
        # y = torch.from_numpy(y.dtype(np.int16))
        # t = torch.from_numpy(t.dtype(np.float64))
        # p = torch.from_numpy(p.dtype(np.int16))

        assert x.shape == y.shape == t.shape == p.shape

        # Step 2: select events between two frames according to timestamp
        temp = t.numpy().tolist()
        output = [
            temp[
                bisect.bisect_left(temp, timestamp_list[i]):bisect.bisect_left(temp, timestamp_list[i+1])
            ]
            for i in range(len(timestamp_list) - 1)
        ]

        # Debug: Check if data error!!!
        assert len(output) == len(timestamp_list) - 1, f"len(output) is {len(output)}, but len(timestamp_list) is {len(timestamp_list)}"
        sum_output = []
        sum = 0
        for i in range(len(output)):
            if len(output[i]) == 0:
                raise ValueError(f"{h5_name} len(output[{i}] == 0)")
            elif len(output[i]) == 1:
                raise ValueError(f"{h5_name} len(output[{i}] == 1)") 
            sum += len(output[i])
            sum_output.append(sum)

        assert len(sum_output) == len(output)

        # Step 3: After checking data, continue.
        start_idx = 0
        for voxel_idx in range(len(timestamp_list) - 1):

            if len(output[voxel_idx]) == 0 or len(output[voxel_idx]) == 1:
                print(f'{h5_name} len(output[{voxel_idx}])): ', len(
                    output[voxel_idx]))
                with open(error_txt, 'a+') as f:
                    f.write(h5_name + '\n')
                return
            end_idx = start_idx + len(output[voxel_idx])
            

            if end_idx > len(t):
                with open(error_txt, 'a+') as f:
                    f.write(f"{h5_name} voxel_idx: {voxel_idx}, start_idx {start_idx} end_idx {end_idx} exceed bound." + '\n')
                print(f"{h5_name} voxel_idx: {voxel_idx}, start_idx {start_idx} end_idx {end_idx} with exceed bound len(t) {len(t)}.")
                return
            
            image_folder = '/data/zhz/code/EvTexture-main/custom_dataset/img_seq_1/'
            image = cv2.imread(image_folder + '{:05d}'.format(voxel_idx+1) + '.jpg')
            self.package_image(image, timestamp_list, voxel_idx)

            xs = x[start_idx:end_idx]
            ys = y[start_idx:end_idx]
            ts = t[start_idx:end_idx]
            ps = p[start_idx:end_idx]
            if ts == torch.Size([]) or ts.shape == torch.Size([1]) or ts.shape == torch.Size([0]):
                with open(error_txt, 'a+') as f:
                    f.write(f"{h5_name} len(output[{voxel_idx}]) backward {backward} start_idx {start_idx} end_idx {end_idx} is error! Please check the data." + '\n')
                print(f"{h5_name} len(output[{voxel_idx}]) backward {backward} start_idx {start_idx} end_idx {end_idx} is error! Please check the data.")
                return

            voxel = events_to_voxel_torch(
                xs, ys, ts, ps, bins, device=None, sensor_size=sensor_size)
            normed_voxel = voxel_normalization(voxel)
            np_voxel = normed_voxel.numpy()
            self.events_file.create_dataset("voxels_f/{:06d}".format(
                    voxel_idx), data=np_voxel, dtype=np.dtype(np.float64), compression="gzip")
            
            if backward:
                t_start = timestamp_list[voxel_idx]
                t_end = timestamp_list[voxel_idx + 1]
                xs = torch.flip(xs, dims=[0])
                ys = torch.flip(ys, dims=[0])
                ts = torch.flip(t_end - ts + t_start, dims=[0])
                ps = torch.flip(-ps, dims=[0])
            voxel = events_to_voxel_torch(
                xs, ys, ts, ps, bins, device=None, sensor_size=sensor_size)
            normed_voxel = voxel_normalization(voxel)
            np_voxel = normed_voxel.numpy()

            if backward:
                self.events_file.create_dataset("voxels_b/{:06d}".format(
                    voxel_idx), data=np_voxel, dtype=np.dtype(np.float64), compression="gzip")
            else:
                self.events_file.create_dataset("voxels_f/{:06d}".format(
                    voxel_idx), data=np_voxel, dtype=np.dtype(np.float64), compression="gzip")

        
        image_folder = '/data/zhz/code/EvTexture-main/custom_dataset/img_seq_1/'
        image = cv2.imread(image_folder + '{:05d}'.format(voxel_idx+2) + '.jpg')    # 因为上面的for循环只进行到倒数第二个，所以最后还得再加上最后一帧
        self.package_image(image, timestamp_list, voxel_idx+1)
        
    def append_to_dataset(self, dataset, data):
        dataset.resize(dataset.shape[0] + len(data), axis=0)
        if len(data) == 0:
            return
        dataset[-len(data):] = data[:]

    def package_events(self, xs, ys, ts, ps):
        self.append_to_dataset(self.event_xs, xs)
        self.append_to_dataset(self.event_ys, ys)
        self.append_to_dataset(self.event_ts, ts)
        self.append_to_dataset(self.event_ps, ps)

    def package_image(self, image, timestamp, img_idx):
        # image_dset = self.events_file.create_dataset("images/image{:09d}".format(img_idx),
        image_dset = self.events_file.create_dataset("images/{:06d}".format(img_idx),
                data=image, dtype=np.dtype(np.uint8))
        image_dset.attrs['size'] = image.shape
        image_dset.attrs['timestamp'] = timestamp
        image_dset.attrs['type'] = "greyscale" if image.shape[-1] == 1 or len(image.shape) == 2 else "color_bgr" 

    def package_flow(self, flow_image, timestamp, flow_idx):
        flow_dset = self.events_file.create_dataset("flow/flow{:09d}".format(flow_idx),
                data=flow_image, dtype=np.dtype(np.float32))
        flow_dset.attrs['size'] = flow_image.shape
        flow_dset.attrs['timestamp'] = timestamp

    def add_event_indices(self):
        datatypes = ['images', 'flow']
        for datatype in datatypes:
            if datatype in self.events_file.keys():
                s = 0
                added = 0
                ts = self.events_file["events/ts"][s:s+self.max_buffer_size]
                for image in self.events_file[datatype]:
                    img_ts = self.events_file[datatype][image].attrs['timestamp']
                    event_idx = np.searchsorted(ts, img_ts)
                    if event_idx == len(ts):
                        added += len(ts)
                        s += self.max_buffer_size
                        ts = self.events_file["events/ts"][s:s+self.max_buffer_size]
                        event_idx = np.searchsorted(ts, img_ts)
                    event_idx = max(0, event_idx-1)
                    self.events_file[datatype][image].attrs['event_idx'] = event_idx + added

    def add_metadata(self, num_pos, num_neg,
            duration, t0, tk, num_imgs, num_flow, sensor_size):
        self.events_file.attrs['num_events'] = num_pos+num_neg
        self.events_file.attrs['num_pos'] = num_pos
        self.events_file.attrs['num_neg'] = num_neg
        self.events_file.attrs['duration'] = tk-t0
        self.events_file.attrs['t0'] = t0
        self.events_file.attrs['tk'] = tk
        self.events_file.attrs['num_imgs'] = num_imgs
        self.events_file.attrs['num_flow'] = num_flow
        self.events_file.attrs['sensor_resolution'] = sensor_size
        self.add_event_indices()

    def set_data_available(self, num_images, num_flow):
        if num_images > 0:
            self.image_dset = self.events_file.create_group("images")
            self.image_dset.attrs['num_images'] = num_images
        if num_flow > 0:
            self.flow_dset = self.events_file.create_group("flow")
            self.flow_dset.attrs['num_images'] = num_flow

config = {
	'refractory_period': 1e-4,
	'CT_range': [0.05, 0.5],
	'max_CT': 0.5,
	'min_CT': 0.02,
	'mu': 1,
	'sigma': 0.1,
	# 'H': clip.height,
    # 'H': 1216,
    # 'H': 576,
    'H': 180,
	# 'W': clip.width,
    # 'W': 1936,
    # 'W': 800,
    'W': 240,
	'log_eps': 1e-3,
	'use_log': True,
}

Cp = random.uniform(config['CT_range'][0], config['CT_range'][1])
Cn = random.gauss(config['mu'], config['sigma']) * Cp
Cp = min(max(Cp, config['min_CT']), config['max_CT'])
Cn = min(max(Cn, config['min_CT']), config['max_CT'])
esim = esim_py.EventSimulator(Cp,
							Cn,
							config['refractory_period'],
							config['log_eps'],
							config['use_log'])
image_folder = '/data/zhz/code/EvTexture-main/custom_dataset/img_seq_1'
timestamps_file = '/data/zhz/code/EvTexture-main/custom_dataset/timestamps.txt'
events = esim.generateFromFolder(image_folder, timestamps_file) # Generate events with shape [N, 4]  (20686327, 4)
xs = torch.tensor(events[:,0], dtype=torch.int16)
ys = torch.tensor(events[:,1], dtype=torch.int16)
ts = torch.tensor(events[:,2], dtype=torch.float32)
# ps = torch.tensor(events[:,3], dtype=torch.bool)
ps = torch.tensor(events[:,3], dtype=torch.int16)

backward = True
bins = 5
sensor_size = (180, 240)
h5_name = 'my'
error_txt = '/data/zhz/code/EvTexture-main/custom_dataset/error.txt'
timestamp_list = [0.04*i for i in range(30)]
 
my_package = hdf5_packager(output_path = '/data/zhz/code/EvTexture-main/custom_dataset/my.h5')
my_package.package_bidirectional_event_voxels(xs, ys, ts, ps, timestamp_list, True, bins, sensor_size, h5_name, error_txt)

 再搞一个高分辨率的数据集：

推理：

python basicsr/test.py -opt options/test/EvTexture/test_EvTexture_my_Bl×4.yml

原始180*240图像：

原始720*960图像：

网络输出超分后的720*960图像：