# ===================================================================
# FULL IMPLEMENTATION: Lane Detection on TuSimple
# Model: ResNet50-DCNv2 + FPN + MultiTaskHead
# Loss: Focal + Regress + Distance + Variance
# Dataset: Auto-download & organize TuSimple (JSONL support)
# Eval Metrics: Acc, P, R, F1, FPR, FNR
# ===================================================================
import os
import json
import cv2
import numpy as np
from PIL import Image
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as T
import torchvision.models as models
from torchvision.ops import DeformConv2d
# ======================================
# 1. 可变形卷积模块
# ======================================
class DeformableConv(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1):
super().__init__()
self.offset_conv = nn.Conv2d(in_channels, 2 * kernel_size * kernel_size, kernel_size, stride, padding)
self.dcn = DeformConv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)
def forward(self, x):
offset = self.offset_conv(x)
return self.dcn(x, offset)
# ======================================
# 2. 支持 DCN 的 Bottleneck
# ======================================
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None, dilation=1, use_dcn=False):
super(Bottleneck, self).__init__()
self.use_dcn = use_dcn
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn1 = nn.BatchNorm2d(planes)
if use_dcn:
self.conv2 = DeformableConv(planes, planes, kernel_size=3, stride=stride, padding=dilation)
else:
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation, bias=False)
self.bn2 = nn.BatchNorm2d(planes)
self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False)
self.bn3 = nn.BatchNorm2d(planes * self.expansion)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def forward(self, x):
identity = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
if not self.use_dcn:
out = self.bn2(out)
out = self.relu(out)
out = self.conv3(out)
out = self.bn3(out)
if self.downsample is not None:
identity = self.downsample(x)
# 自动对齐 spatial 维度
if out.shape[-1] != identity.shape[-1] or out.shape[-2] != identity.shape[-2]:
h, w = identity.shape[2:]
out = F.interpolate(out, size=(h, w), mode='bilinear', align_corners=False)
out += identity
out = self.relu(out)
return out
# ======================================
# 3. 构建残差层
# ======================================
def make_res_layer(block, inplanes, planes, blocks, stride=1, dilation=1, use_dcn=False):
downsample = None
if stride != 1 or inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv2d(inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False),
nn.BatchNorm2d(planes * block.expansion),
)
layers = []
first_dilation = 1 if dilation == 1 else dilation // 2
layers.append(block(inplanes, planes, stride, downsample, dilation=first_dilation, use_dcn=False))
for _ in range(1, blocks):
layers.append(block(planes * block.expansion, planes, dilation=dilation, use_dcn=use_dcn))
return nn.Sequential(*layers)
# ======================================
# 4. 修改版 ResNet50 主干
# ======================================
class ModifiedResNet50(nn.Module):
def __init__(self, pretrained=True):
super(ModifiedResNet50, self).__init__()
weights = models.ResNet50_Weights.IMAGENET1K_V1 if pretrained else None
original = models.resnet50(weights=weights)
self.inplanes = 64
self.conv1 = original.conv1
self.bn1 = original.bn1
self.relu = original.relu
self.maxpool = original.maxpool
self.layer1 = original.layer1 # out: 256
self.layer2 = original.layer2 # out: 512
self.layer3 = make_res_layer(Bottleneck, 512, 256, 6, stride=1, dilation=2, use_dcn=True) # 256*4=1024
self.layer4 = make_res_layer(Bottleneck, 1024, 512, 3, stride=1, dilation=4, use_dcn=True) # 512*4=2048
def forward(self, x):
c1 = self.relu(self.bn1(self.conv1(x)))
c1 = self.maxpool(c1)
c2 = self.layer1(c1) # H/4
c3 = self.layer2(c2) # H/8
c4 = self.layer3(c3) # H/8, d=2
c5 = self.layer4(c4) # H/8, d=4
return c3, c4, c5
# ======================================
# 5. FPN Neck
# ======================================
class FPN(nn.Module):
def __init__(self, in_channels_list=[512, 1024, 2048], out_channels=256):
super(FPN, self).__init__()
self.lateral_convs = nn.ModuleList([
nn.Conv2d(in_c, out_channels, 1) for in_c in in_channels_list
])
self.fusion_conv = nn.Conv2d(out_channels * 3, out_channels, 3, padding=1, bias=False)
self.norm = nn.BatchNorm2d(out_channels)
self.relu = nn.ReLU(inplace=True)
def forward(self, inputs):
c3, c4, c5 = inputs
l3 = self.lateral_convs[0](c3)
l4 = self.lateral_convs[1](c4)
l5 = self.lateral_convs[2](c5)
fused = torch.cat([l3, l4, l5], dim=1)
return self.relu(self.norm(self.fusion_conv(fused)))
# ======================================
# 6. 多任务 Head
# ======================================
class MultiTaskHead(nn.Module):
def __init__(self, in_channels=256, num_classes=5, output_size=(384, 640)):
super(MultiTaskHead, self).__init__()
self.cls = nn.Conv2d(in_channels, num_classes, 1)
self.offset = nn.Conv2d(in_channels, 2, 1)
self.distance = nn.Conv2d(in_channels, 1, 1)
self.variance = nn.Conv2d(in_channels, 2, 1)
self.output_size = output_size
def forward(self, x):
upsample = lambda pred: F.interpolate(pred, size=self.output_size, mode='bilinear', align_corners=False)
return {
'cls': upsample(self.cls(x)),
'offset': upsample(self.offset(x)),
'distance': upsample(self.distance(x)),
'variance': upsample(self.variance(x))
}
# ======================================
# 7. 主模型
# ======================================
class LaneSegNet_MultiTask(nn.Module):
def __init__(self, num_classes=5):
super(LaneSegNet_MultiTask, self).__init__()
self.backbone = ModifiedResNet50(pretrained=True)
self.fpn = FPN(out_channels=256)
self.head = MultiTaskHead(256, num_classes, output_size=(384, 640))
def forward(self, x):
feats = self.backbone(x)
fpn_out = self.fpn(feats)
return self.head(fpn_out)
# ======================================
# 8. TuSimple Dataset (支持 JSONL)
# ======================================
class TuSimpleDataset(Dataset):
def __init__(self, root, split='train', img_size=(384, 640), transform=None):
self.root = root
self.img_size = img_size
self.transform = transform or T.Compose([
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
self.image_paths = []
self.lanes_data = []
if split == 'train':
label_files = [
os.path.join(root, 'train_set', 'label_data_0313.json'),
os.path.join(root, 'train_set', 'label_data_0531.json'),
os.path.join(root, 'train_set', 'label_data_0601.json')
]
require_lanes = True
else:
label_files = [os.path.join(root, 'test_set', 'test_tasks_0627.json')]
require_lanes = False # 测试集不需要 lanes 标注
for file in label_files:
if not os.path.exists(file):
print(f"[WARNING] Label file not found: {file}")
continue
with open(file, 'r') as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if not line:
continue
try:
data = json.loads(line)
raw_file = data['raw_file']
# === 修复路径映射问题 ===
if split == 'train':
if not raw_file.startswith('train_set'):
raw_file = os.path.join('train_set', raw_file)
img_path = os.path.join(root, raw_file)
else: # split == 'test'
# 测试集图片位于 test_set/images/clips/xxx
img_path = os.path.join(root, 'test_set', 'images', raw_file)
# 检查图像是否存在
if not os.path.exists(img_path):
# 可选：打印缺失文件调试
# print(f"[DEBUG] Image not found: {img_path}")
continue
# 是否需要标注字段？
has_annotations = ('lanes' in data and 'h_samples' in data)
if require_lanes and (not has_annotations or len(data['lanes']) == 0):
continue
self.image_paths.append(img_path)
self.lanes_data.append(data)
except json.JSONDecodeError as e:
print(f"[ERROR] Failed to parse line: {line[:60]}... | Error: {e}")
continue
self.image_paths.append(img_path)
self.lanes_data.append(data)
except json.JSONDecodeError as e:
print(f"[ERROR] Failed to parse line: {line[:60]}... | Error: {e}")
continue
assert len(self.image_paths) > 0, f"No valid samples found for {split} set under {root}"
print(f"✅ Loaded {len(self)} samples for {split} set.")
def __len__(self):
return len(self.image_paths)
def __getitem__(self, idx):
img_path = self.image_paths[idx]
data = self.lanes_data[idx]
image = cv2.imread(img_path)
if image is None:
print(f"[WARNING] Failed to load image: {img_path}")
return self[(idx + 1) % len(self)]
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
orig_h, orig_w = image.shape[:2]
image_pil = Image.fromarray(image)
image_resized = image_pil.resize((self.img_size[1], self.img_size[0]), Image.BILINEAR)
cls_map = np.zeros((self.img_size[0], self.img_size[1]), dtype=np.int64)
offset_map = np.zeros((2, self.img_size[0], self.img_size[1]), dtype=np.float32)
distance_map = np.full((1, self.img_size[0], self.img_size[1]), 1e6, dtype=np.float32)
ratio_h, ratio_w = self.img_size[0] / orig_h, self.img_size[1] / orig_w
lanes = data['lanes']
h_samples = data['h_samples']
all_lane_points = []
for xs in lanes:
points = [(int(x * ratio_w), int(y * ratio_h)) for x, y in zip(xs, h_samples) if x >= 0]
points = [(px, py) for px, py in points if 0 <= px < self.img_size[1] and 0 <= py < self.img_size[0]]
if len(points) > 1:
all_lane_points.append(np.array(points))
if all_lane_points:
yy, xx = np.mgrid[0:self.img_size[0], 0:self.img_size[1]]
coords = np.stack([xx, yy], axis=-1).astype(np.float32)
for points in all_lane_points:
dists = np.linalg.norm(coords[:, :, None] - points[None, None, :], axis=-1)
min_dists = dists.min(axis=-1)
nearest_idx = dists.argmin(axis=-1)
nearest_pts = points[nearest_idx]
offsets = nearest_pts - coords
update_mask = min_dists < distance_map[0]
offset_map[:, update_mask] = offsets[update_mask].T
distance_map[0][update_mask] = min_dists[update_mask]
offset_map /= 16.0
distance_map = np.clip(distance_map, 0, 100) / 50.0
image_tensor = self.transform(image_resized)
label_tensor = torch.from_numpy(cls_map).long()
offset_tensor = torch.from_numpy(offset_map).float()
distance_tensor = torch.from_numpy(distance_map).float()
return {
'image': image_tensor,
'label': label_tensor,
'offset': offset_tensor,
'distance': distance_tensor
}
# ======================================
# 9. 损失函数
# ======================================
class FocalLoss(nn.Module):
def __init__(self, alpha=1, gamma=2):
super().__init__()
self.alpha = alpha
self.gamma = gamma
def forward(self, pred, target):
ce_loss = F.cross_entropy(pred, target, ignore_index=0, reduction='none')
pt = torch.exp(-ce_loss)
focal_loss = self.alpha * (1 - pt) ** self.gamma * ce_loss
return focal_loss.mean()
class RegressLoss(nn.Module):
def __init__(self):
super().__init__()
self.criterion = nn.SmoothL1Loss(reduction='mean')
def forward(self, pred, target, mask):
if mask.sum() == 0:
return pred.new_zeros([])
return self.criterion(pred[mask], target[mask])
class DistanceLoss(nn.Module):
def __init__(self):
super().__init__()
self.criterion = nn.MSELoss()
def forward(self, pred, target):
return self.criterion(pred, target)
class VarianceLoss(nn.Module):
def __init__(self):
super().__init__()
def forward(self, pred_mean, pred_logvar, target):
precision = torch.exp(-pred_logvar)
loss = precision * (target - pred_mean) ** 2 + pred_logvar
return loss.mean()
# ======================================
# 10. 评估函数
# ======================================
@torch.no_grad()
def compute_metrics(pred_cls, target, num_classes=5, ignore_index=0):
pred_label = torch.argmax(pred_cls, dim=1)
pred_flat = pred_label.view(-1)
target_flat = target.view(-1)
mask = (target_flat >= ignore_index) & (target_flat < num_classes)
n = num_classes
hist = torch.bincount(n * target_flat[mask] + pred_flat[mask], minlength=n**2).reshape(n, n).float()
tp = hist[1:, 1:].diag().sum().item()
fp = (hist.sum(dim=0)[1:] - hist[1:, 1:].diag()).sum().item()
fn = (hist.sum(dim=1)[1:] - hist[1:, 1:].diag()).sum().item()
tn = hist[0, 0].item()
eps = 1e-8
acc = (tp + tn) / (tp + tn + fp + fn + eps)
prec = tp / (tp + fp + eps)
rec = tp / (tp + fn + eps)
f1 = 2 * prec * rec / (prec + rec + eps)
fpr = fp / (fp + tn + eps)
fnr = fn / (fn + tp + eps)
return {'accuracy': acc, 'precision': prec, 'recall': rec, 'f1': f1, 'fpr': fpr, 'fnr': fnr}
@torch.no_grad()
def evaluate(model, dataloader, device):
model.eval()
total_metrics = {k: 0.0 for k in ['accuracy', 'precision', 'recall', 'f1', 'fpr', 'fnr']}
count = 0
for data in dataloader:
images = data['image'].to(device)
labels = data['label'].to(device)
outputs = model(images)
metrics = compute_metrics(outputs['cls'], labels)
for k in total_metrics:
total_metrics[k] += metrics[k]
count += 1
return {k: v / count for k, v in total_metrics.items()} if count else total_metrics
# ======================================
# 11. 自动准备 TuSimple 数据集
# ======================================
def prepare_tusimple_dataset():
try:
import kagglehub
except ImportError:
raise ImportError(
"Please install kagglehub:\n"
"pip install kagglehub\n\n"
"Or download dataset manually from:\n"
"https://www.kaggle.com/datasets/manideep1108/tusimple"
)
candidate_paths = [
os.getenv("TUSIMPLE_DATA_PATH"),
os.path.join(os.getcwd(), "TUSimple"),
os.path.expanduser("~/TUSimple")
]
dataset_root = None
for path in candidate_paths:
if not path or not os.path.exists(path):
continue
has_train = os.path.isdir(os.path.join(path, "train_set"))
has_test = os.path.isdir(os.path.join(path, "test_set"))
if has_train or has_test:
dataset_root = path
print(f"✅ Found existing dataset at: {dataset_root}")
break
if dataset_root is None:
print("🔍 No local dataset found. Downloading via kagglehub...")
try:
dataset_root = kagglehub.dataset_download("manideep1108/tusimple")
print(f"🎉 Dataset downloaded to: {dataset_root}")
except Exception as e:
raise RuntimeError(
f"Download failed: {e}\n"
"👉 Please place the dataset in one of these paths:\n"
f" - {candidate_paths[1]}\n"
f" - {candidate_paths[2]}"
)
return dataset_root
# ======================================
# 12. 主函数
# ======================================
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"🚀 Using device: {device}")
try:
TUSIMPLE_ROOT = prepare_tusimple_dataset()
except Exception as e:
print(e)
return
transform = T.Compose([
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
train_dataset = TuSimpleDataset(TUSIMPLE_ROOT, split='train', img_size=(384, 640), transform=transform)
val_dataset = TuSimpleDataset(TUSIMPLE_ROOT, split='test', img_size=(384, 640), transform=transform)
train_loader = DataLoader(train_dataset, batch_size=4, shuffle=True, num_workers=4, pin_memory=True)
val_loader = DataLoader(val_dataset, batch_size=4, shuffle=False, num_workers=4)
model = LaneSegNet_MultiTask(num_classes=5).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
criterion_focal = FocalLoss().to(device)
criterion_regress = RegressLoss().to(device)
criterion_distance = DistanceLoss().to(device)
criterion_variance = VarianceLoss().to(device)
w_cls, w_reg, w_dist, w_var = 1.0, 1.0, 0.5, 0.3
epochs = 50
for epoch in range(epochs):
model.train()
total_loss = 0.0
for i, data in enumerate(train_loader):
images = data['image'].to(device)
labels = data['label'].to(device)
offsets = data['offset'].to(device)
distances = data['distance'].to(device)
masks = (labels > 0).unsqueeze(1).expand_as(offsets)
optimizer.zero_grad()
outputs = model(images)
loss_cls = criterion_focal(outputs['cls'], labels)
loss_reg = criterion_regress(outputs['offset'], offsets, masks)
loss_dist = criterion_distance(outputs['distance'], distances)
loss_var = criterion_variance(outputs['offset'], outputs['variance'], offsets)
loss = w_cls * loss_cls + w_reg * loss_reg + w_dist * loss_dist + w_var * loss_var
loss.backward()
optimizer.step()
total_loss += loss.item()
if (i + 1) % 20 == 0:
print(f"Epoch [{epoch + 1}/50], Step [{i + 1}/{len(train_loader)}], Loss: {loss.item():.4f}")
avg_loss = total_loss / len(train_loader)
print(f"Epoch [{epoch + 1}/50] Average Loss: {avg_loss:.4f}")
val_metrics = evaluate(model, val_loader, device)
print(f"Val Acc: {val_metrics['accuracy']:.4f}, F1: {val_metrics['f1']:.4f}, "
f"Prec: {val_metrics['precision']:.4f}, Rec: {val_metrics['recall']:.4f}, "
f"FPR: {val_metrics['fpr']:.4f}, FNR: {val_metrics['fnr']:.4f}")
save_path = os.path.join(os.path.dirname(__file__), "tusimple_lane_model.pth")
torch.save(model.state_dict(), save_path)
print(f"✅ Training completed. Model saved to {save_path}")
if __name__ == "__main__":
main()
增加特征融合使用特征金字塔（FPN） 融合低层高分辨率特征与高层高语义特征，提升复杂场景特征提取能力
增加注意力机制使用CBAM 混合注意力 结合通道注意力（SE 模块）与空间注意力（SAM 模块），全面捕捉关键特征，降低噪声干扰
增加BEV 视角转换使用IPM+MLP IPM 计算快但依赖参数，MLP 轻量易部署，结合后平衡精度与效率
最新发布
扫一扫
230
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
