extract_frames_with_overlap

// extract_frames_with_overlap.jsx
// 功能:从一张大图中提取12帧,每帧宽600px,水平步进100px,高400px
// 要求:当前文档宽度 >= 1600px,高度 >= 400px

#target photoshop

var doc = app.activeDocument;
var frameWidth = 600;
var frameHeight = 400;
var step = 100;          // 每帧偏移量("不同"的部分)
var numFrames = 12;

// 检查文档尺寸是否足够
if (doc.width.as("px") < frameWidth + step * (numFrames - 1) || doc.height.as("px") < frameHeight) {
    alert("错误:文档尺寸太小!\n需要宽度至少 " + (frameWidth + step * (numFrames - 1)) + "px,高度至少 " + frameHeight + "px");
    exit();
}

// 设置保存路径(桌面/frames/)
var saveFolder = new Folder("~/Desktop/frames/");
if (!saveFolder.exists) saveFolder.create();

// 辅助函数:数字补零
function padZero(num, size) {
    var s = num + "";
    while (s.length < size) s = "0" + s;
    return s;
}

// 确保工作在像素单位
app.preferences.rulerUnits = Units.PIXELS;

for (var i = 0; i < numFrames; i++) {
    var x = i * step;
    var y = 0;
    
    // 创建选区边界数组 [left, top, right, bottom]
    var bounds = [
        [x, y],
        [x + frameWidth, y],
        [x + frameWidth, y + frameHeight],
        [x, y + frameHeight]
    ];

    // 创建选区
    doc.selection.select(bounds);
    
    // 复制选区
    doc.selection.copy();
    doc.selection.deselect();
    
    // 新建文档(使用像素单位)
    var newDoc = app.documents.add(
        frameWidth, 
        frameHeight, 
        doc.resolution, 
        "frame_" + (i + 1), 
        NewDocumentMode.RGB, 
        DocumentFill.TRANSPARENT
    );
    
    // 如果原图是灰度模式,转换新文档
    if (doc.mode == DocumentMode.GRAYSCALE) {
        newDoc.changeMode(ChangeMode.GRAYSCALE);
    }
    
    // 粘贴并栅格化
    newDoc.paste();
    
    // 栅格化图层以确保正确尺寸
    newDoc.activeLayer.rasterize(RasterizeType.ENTIRELAYER);
    
    // 裁剪到画布大小(确保没有超出边界的部分)
    newDoc.crop([0, 0, frameWidth, frameHeight]);
    
    // 拼合图像(替代合并可见图层)
    newDoc.flatten();
    
    // 保存为 PNG
    var filePath = new File(saveFolder + "/frame_" + padZero(i + 1, 3) + ".png");
    var pngSaveOptions = new PNGSaveOptions();
    pngSaveOptions.compression = 9;
    pngSaveOptions.interlaced = false;
    newDoc.saveAs(filePath, pngSaveOptions, true);
    newDoc.close(SaveOptions.DONOTSAVECHANGES);
}

alert("✅ 成功导出 " + numFrames + " 帧到桌面 /frames 文件夹!");
import json import os import jsonlines from collections import defaultdict from typing import Dict, List, Any, Tuple, Union import logging import time from dataclasses import dataclass, field from math import isclose # 配置日志 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler("collision_processing.log"), logging.StreamHandler() ] ) logger = logging.getLogger("CollisionProcessor") # 数据类定义 @dataclass class FrameData: timestamp: float road_points: List[Dict[str, float]] record: Dict[str, Any] group_key: str = "" scene_id: str = "" processed: bool = False @dataclass class CameraStats: cam_tag: str total_frames: int = 0 yes_count: int = 0 no_count: int = 0 yes_ratio: float = 0.0 no_ratio: float = 0.0 @dataclass class SceneStats: scene_id: str total_frames: int = 0 yes_count: int = 0 no_count: int = 0 yes_ratio: float = 0.0 no_ratio: float = 0.0 cam_tags: Dict[str, CameraStats] = field(default_factory=dict) @dataclass class OverallStats: total_scenes: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 yes_ratio: float = 0.0 no_ratio: float = 0.0 def extract_group_key(image_path: str) -> str: """ 解析文件路径中的倒数第二个值作为分组键 :param image_path: 图片路径字符串 :return: 分组键字符串 """ try: parts = image_path.split("/") if len(parts) < 2: return "default_group" # 返回倒数第二部分作为分组键 return parts[-2] except Exception as e: logger.error(f"提取分组键错误: {e}, 路径: {image_path}") return "default_group" def round_coordinate(value: float, decimals: int = 2) -> float: """ 四舍五入坐标值到指定小数位数 :param value: 原始坐标值 :param decimals: 保留的小数位数 :return: 四舍五入后的坐标值 """ if value is None: return 0.0 factor = 10 ** decimals return round(value * factor) / factor def process_road_points(road_points: List[Dict[str, float]]) -> List[Tuple[float, float]]: """ 处理道路点数据,提取并四舍五入坐标 :param road_points: 原始道路点列表 :return: 处理后的坐标元组列表 """ if not road_points: return [] return [ (round_coordinate(point.get('x', 0.0)), round_coordinate(point.get('y', 0.0))) for point in road_points ] def calculate_overlap_ratio( points1: List[Tuple[float, float]], points2: List[Tuple[float, float]], tolerance: float = 0.01 ) -> float: """ 计算两组点之间的重叠率 :param points1: 第一组点 :param points2: 第二组点 :param tolerance: 坐标比较容差 :return: 重叠率 (0.0-1.0) """ if not points1 or not points2: return 0.0 # 使用集合存储点信息 set1 = set() for x, y in points1: # 使用容差处理浮点数精度问题 set1.add((round(x, 2), round(y, 2))) set2 = set() for x, y in points2: set2.add((round(x, 2), round(y, 2))) # 计算交集和并集 intersection = set1 & set2 union = set1 | set2 if not union: return 0.0 return len(intersection) / len(union) def load_data(input_file: str) -> Dict[str, Dict[str, List[FrameData]]]: """ 从JSONL文件加载数据并分组 :param input_file: 输入文件路径 :return: 分组后的数据结构 {scene_id: {group_key: [FrameData]}} """ grouped_data = defaultdict(lambda: defaultdict(list)) total_records = 0 logger.info(f"开始读取输入文件: {input_file}") try: with open(input_file, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): try: record = json.loads(line.strip()) scene_id = record.get('scene_id', 'unknown_scene') image_path = record.get('image_path', '') timestamp = record.get('timestamp', 0.0) # 提取道路点信息 mop_info = record.get('mop_pnc_info', {}) if not isinstance(mop_info, dict): mop_info = {} road_points = mop_info.get('roadPoints', []) # 处理道路点 processed_points = process_road_points(road_points) # 提取分组键 group_key = extract_group_key(image_path) # 创建FrameData对象 frame_data = FrameData( timestamp=timestamp, road_points=processed_points, record=record, group_key=group_key, scene_id=scene_id ) # 添加到分组数据 grouped_data[scene_id][group_key].append(frame_data) total_records += 1 except json.JSONDecodeError as e: logger.error(f"JSON解析错误 (行 {line_num}): {e}") except Exception as e: logger.error(f"处理记录错误 (行 {line_num}): {e}") except IOError as e: logger.error(f"文件读取错误: {e}") return {} logger.info(f"成功读取 {total_records} 条记录,分组到 {len(grouped_data)} 个场景") return grouped_data def process_frame_group( frames: List[FrameData], threshold: float = 0.7, debug: bool = False ) -> CameraStats: """ 处理一组帧数据,生成标签并返回相机统计 :param frames: 帧数据列表 :param threshold: 重叠度阈值 :param debug: 是否输出调试信息 :return: 相机统计信息 """ if not frames: return CameraStats(cam_tag="") # 获取相机标签(从第一帧获取) cam_tag = frames[0].group_key if frames else "" cam_stats = CameraStats(cam_tag=cam_tag) cam_stats.total_frames = len(frames) # 按时间戳降序排序(最新时间在前) frames.sort(key=lambda x: x.timestamp, reverse=True) # 处理第一帧(最新帧) if frames: frames[0].record['mask'] = 'yes' cam_stats.yes_count += 1 frames[0].processed = True if debug: logger.debug(f"相机 {cam_tag}: 帧0 (最新) 标签设置为 'yes'") # 处理第二帧 if len(frames) > 1: # 与第一帧比较 overlap = calculate_overlap_ratio( frames[0].road_points, frames[1].road_points ) if overlap > threshold: frames[1].record['mask'] = 'yes' cam_stats.yes_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧1 与帧0重叠度 {overlap:.2f} > {threshold} -> 标签设置为 'yes'") else: frames[1].record['mask'] = 'no' cam_stats.no_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧1 与帧0重叠度 {overlap:.2f} <= {threshold} -> 标签设置为 'no'") frames[1].processed = True # 处理后续帧(第三帧及以后) for i in range(2, len(frames)): # 与前一帧比较 overlap = calculate_overlap_ratio( frames[i - 1].road_points, frames[i].road_points ) if overlap > threshold: frames[i].record['mask'] = frames[i - 1].record.get('mask', 'yes') if frames[i].record['mask'] == 'yes': cam_stats.yes_count += 1 else: cam_stats.no_count += 1 if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} > {threshold} -> 继承标签 '{frames[i].record['mask']}'") else: # 反转前一帧的标签 prev_mask = frames[i - 1].record.get('mask', 'yes') frames[i].record['mask'] = 'no' if prev_mask == 'yes' else 'yes' if frames[i].record['mask'] == 'yes': cam_stats.yes_count += 1 else: cam_stats.no_count += 1 if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} <= {threshold} -> 反转标签为 '{frames[i].record['mask']}'") frames[i].processed = True # 计算比例 if cam_stats.total_frames > 0: cam_stats.yes_ratio = cam_stats.yes_count / cam_stats.total_frames cam_stats.no_ratio = cam_stats.no_count / cam_stats.total_frames return cam_stats def process_collision_data( input_file: str, output_file: str, stats_file: str, threshold: float = 0.7, debug: bool = False ) -> Dict[str, Any]: """ 处理碰撞数据的核心函数 :param input_file: 输入JSONL文件路径 :param output_file: 输出JSONL文件路径 :param stats_file: 统计信息输出文件路径 :param threshold: 重叠度阈值 :param debug: 是否输出调试信息 :return: 统计信息字典 """ start_time = time.time() logger.info(f"开始处理碰撞数据: {input_file}") # 加载数据 grouped_data = load_data(input_file) if not grouped_data: logger.error("未加载到有效数据,处理终止") return {} # 初始化统计数据结构 overall_stats = OverallStats(total_scenes=len(grouped_data)) scene_stats_dict = {} labeled_data = [] # 处理每个场景 for scene_id, cam_groups in grouped_data.items(): scene_stats = SceneStats(scene_id=scene_id) # 处理每个相机组 for group_key, frames in cam_groups.items(): cam_stats = process_frame_group(frames, threshold, debug) # 更新场景统计 scene_stats.cam_tags[group_key] = cam_stats scene_stats.total_frames += cam_stats.total_frames scene_stats.yes_count += cam_stats.yes_count scene_stats.no_count += cam_stats.no_count # 收集处理后的数据 labeled_data.extend(frame.record for frame in frames if frame.processed) # 计算场景比例 if scene_stats.total_frames > 0: scene_stats.yes_ratio = scene_stats.yes_count / scene_stats.total_frames scene_stats.no_ratio = scene_stats.no_count / scene_stats.total_frames # 更新总体统计 scene_stats_dict[scene_id] = scene_stats overall_stats.total_frames += scene_stats.total_frames overall_stats.yes_count += scene_stats.yes_count overall_stats.no_count += scene_stats.no_count # 计算总体比例 if overall_stats.total_frames > 0: overall_stats.yes_ratio = overall_stats.yes_count / overall_stats.total_frames overall_stats.no_ratio = overall_stats.no_count / overall_stats.total_frames # 按scene_id、cam_tag和timestamp降序排序 labeled_data.sort(key=lambda x: ( x.get('scene_id', ''), x.get('image_path', '').split('/')[-2] if 'image_path' in x else '', -x.get('timestamp', 0) )) # 准备输出统计信息 stats = { "stats_file": os.path.basename(stats_file), "total_scenes": overall_stats.total_scenes, "scenes": {scene_id: scene_stats.__dict__ for scene_id, scene_stats in scene_stats_dict.items()}, "overall": { "total_frames": overall_stats.total_frames, "yes_count": overall_stats.yes_count, "no_count": overall_stats.no_count, "yes_ratio": overall_stats.yes_ratio, "no_ratio": overall_stats.no_ratio } } # 确保输出目录存在 os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True) os.makedirs(os.path.dirname(stats_file) or '.', exist_ok=True) # 写入输出文件 logger.info(f"写入处理后的数据到: {output_file}") with jsonlines.open(output_file, 'w') as writer: for record in labeled_data: writer.write(record) # 写入统计文件 logger.info(f"写入统计信息到: {stats_file}") with open(stats_file, 'w', encoding='utf-8') as f: json.dump(stats, f, ensure_ascii=False, indent=4) # 计算处理时间 processing_time = time.time() - start_time logger.info(f"处理完成! 总耗时: {processing_time:.2f}秒") return stats if __name__ == "__main__": # 使用指定的文件路径调用处理函数 stats = process_collision_data( input_file="./basicData_historyDepth_v2_img106k_250723__huanshi_6ver_4fish_large_model_900_792874_10frames.jsonl", output_file="processed_106k_0723_tag02.jsonl", stats_file="collision_stats_0723_tag02.json", threshold=0.7, debug=False # 设置为True可查看详细处理日志 ) # 打印摘要统计 if stats: print("\n处理完成!") print(f"总场景数: {stats['total_scenes']}") print(f"总帧数: {stats['overall']['total_frames']}") print(f"YES标签数: {stats['overall']['yes_count']}") print(f"NO标签数: {stats['overall']['no_count']}") print(f"YES占比: {stats['overall']['yes_ratio']:.2%}") print(f"NO占比: {stats['overall']['no_ratio']:.2%}") print(f"输出文件: {stats['output_file']}") print(f"统计文件: {stats['stats_file']}") else: print("处理失败,请检查日志文件获取详细信息") 优化统计信息,只统计 "scene_id": scene_id, "total_cam_tags": len(cam_groups), "cam_tags": {}, "total_frames": 0, "yes_count": 0, "no_count": 0
07-30
import json import os from collections import defaultdict from typing import List, Dict, Tuple, Any def overlap_ratio(road_points_1: List[Dict[str, float]], road_points_2: List[Dict[str, float]]) -> float: """ 计算两条轨迹的重叠度,使用Jaccard相似度:交集 / 并集 """ if not road_points_1 or not road_points_2: return 0.0 # 创建点集(使用元组确保可哈希) set_1 = set((p['x'], p['y']) for p in road_points_1) set_2 = set((p['x'], p['y']) for p in road_points_2) intersection = len(set_1.intersection(set_2)) union = len(set_1.union(set_2)) return intersection / union if union != 0 else 0.0 def extract_group_key(image_path: str) -> str: """ 解析文件路径中的倒数第二个值作为分组键 """ try: parts = image_path.strip('/').split('/') if len(parts) < 2: return "default_group" return parts[-2] # 获取倒数第二个部分 except Exception as e: print(f"提取分组键错误: {e}, 路径: {image_path}") return "default_group" def process_collision_data( input_file: str, output_file: str, stats_file: str, threshold: float = 0.7, debug: bool = False ) -> Dict[str, Any]: """ 处理碰撞数据的核心函数 :param input_file: 输入JSONL文件路径 :param output_file: 输出JSONL文件路径 :param stats_file: 统计信息输出文件路径 :param threshold: 重叠度阈值,默认0.7 :param debug: 是否输出调试信息 :return: 统计信息字典 """ # 使用嵌套字典存储分组数据 grouped_data = defaultdict(lambda: defaultdict(list)) # 读取JSONL文件并解析 total_records = 0 print(f"开始读取输入文件: {input_file}") with open(input_file, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): try: record = json.loads(line.strip()) sence_id = record.get('sence_id', 'unknown_sence') image_path = record.get('image_path', '') timestamp = record.get('timestamp', 0) # 获取道路点信息 mop_info = record.get('mop_info', {}) road_points = mop_info.get('roadPoints', []) if isinstance(mop_info, dict) else [] # 提取分组键 group_key = extract_group_key(image_path) # 存储记录 grouped_data[sence_id][group_key].append({ 'timestamp': timestamp, 'road_points': road_points, 'record': record }) total_records += 1 except Exception as e: print(f"处理记录错误 (行 {line_num}): {e}") print(f"成功读取 {total_records} 条记录,分组到 {len(grouped_data)} 个场景") # 初始化统计数据结构 stats = { "input_file": os.path.basename(input_file), "output_file": os.path.basename(output_file), "stats_file": os.path.basename(stats_file), "total_scenes": len(grouped_data), "scenes": {}, "overall": { "total_frames": 0, "yes_count": 0, "no_count": 0, "yes_ratio": 0.0, "no_ratio": 0.0 } } labeled_data = [] # 处理每个场景和每个分组 for sence_id, groups in grouped_data.items(): scene_stats = { "scene_id": sence_id, "cam_tags": {}, "total_frames": 0, "yes_count": 0, "no_count": 0, "yes_ratio": 0.0, "no_ratio": 0.0 } for group_key, records in groups.items(): # 初始化相机统计 cam_stats = { "cam_tag": group_key, "total_frames": len(records), "yes_count": 0, "no_count": 0, "yes_ratio": 0.0, "no_ratio": 0.0 } # 按时间戳降序排列(最新时间在前) records.sort(key=lambda x: x['timestamp'], reverse=True) # 改进的标签生成逻辑 no_occurred = False # 标记是否已出现第一个no for idx, record_data in enumerate(records): current_record = record_data['record'] if idx == 0: # 第一帧(最新帧) current_record['mask'] = 'yes' cam_stats['yes_count'] += 1 if debug: print(f"场景 {sence_id} 相机 {group_key}: 帧0 (最新) 标签设置为 'yes'") elif idx == 1: # 第二帧 # 与第一帧比较 overlap = overlap_ratio( records[0]['road_points'], record_data['road_points'] ) if overlap > threshold: current_record['mask'] = 'yes' cam_stats['yes_count'] += 1 if debug: print( f"场景 {sence_id} 相机 {group_key}: 帧1 与帧0重叠度 {overlap:.2f} > {threshold} -> 标签设置为 'yes'") else: current_record['mask'] = 'no' cam_stats['no_count'] += 1 no_occurred = True # 标记已出现第一个no if debug: print( f"场景 {sence_id} 相机 {group_key}: 帧1 与帧0重叠度 {overlap:.2f} <= {threshold} -> 标签设置为 'no' (第一个no出现)") else: # 第三帧及以后 if no_occurred: current_record['mask'] = 'no' cam_stats['no_count'] += 1 if debug: print(f"场景 {sence_id} 相机 {group_key}: 帧{idx} 分组中已出现no -> 标签设置为 'no'") else: # 与前一帧比较 overlap = overlap_ratio( records[idx - 1]['road_points'], record_data['road_points'] ) if overlap > threshold: current_record['mask'] = 'yes' cam_stats['yes_count'] += 1 if debug: print( f"场景 {sence_id} 相机 {group_key}: 帧{idx} 与帧{idx - 1}重叠度 {overlap:.2f} > {threshold} -> 标签设置为 'yes'") else: current_record['mask'] = 'no' cam_stats['no_count'] += 1 no_occurred = True # 标记已出现第一个no if debug: print( f"场景 {sence_id} 相机 {group_key}: 帧{idx} 与帧{idx - 1}重叠度 {overlap:.2f} <= {threshold} -> 标签设置为 'no' (第一个no出现)") # 计算相机统计比例 if cam_stats['total_frames'] > 0: cam_stats['yes_ratio'] = cam_stats['yes_count'] / cam_stats['total_frames'] cam_stats['no_ratio'] = cam_stats['no_count'] / cam_stats['total_frames'] # 更新场景统计 scene_stats['cam_tags'][group_key] = cam_stats scene_stats['total_frames'] += cam_stats['total_frames'] scene_stats['yes_count'] += cam_stats['yes_count'] scene_stats['no_count'] += cam_stats['no_count'] # 添加到标签化数据 for record_data in records: labeled_data.append(record_data['record']) # 计算场景统计比例 if scene_stats['total_frames'] > 0: scene_stats['yes_ratio'] = scene_stats['yes_count'] / scene_stats['total_frames'] scene_stats['no_ratio'] = scene_stats['no_count'] / scene_stats['total_frames'] # 添加到总体统计 stats['scenes'][sence_id] = scene_stats stats['overall']['total_frames'] += scene_stats['total_frames'] stats['overall']['yes_count'] += scene_stats['yes_count'] stats['overall']['no_count'] += scene_stats['no_count'] # 计算总体统计比例 if stats['overall']['total_frames'] > 0: stats['overall']['yes_ratio'] = stats['overall']['yes_count'] / stats['overall']['total_frames'] stats['overall']['no_ratio'] = stats['overall']['no_count'] / stats['overall']['total_frames'] # 按scene_id、cam_tag和timestamp降序排序 labeled_data.sort(key=lambda x: ( x.get('sence_id', ''), x.get('cam_tag', ''), -x.get('timestamp', 0) )) # 确保输出目录存在 os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True) os.makedirs(os.path.dirname(stats_file) or '.', exist_ok=True) # 写入输出文件 print(f"写入处理后的数据到: {output_file}") with open(output_file, 'w', encoding='utf-8') as f: for record in labeled_data: f.write(json.dumps(record, ensure_ascii=False) + '\n') # 写入统计文件 print(f"写入统计信息到: {stats_file}") with open(stats_file, 'w', encoding='utf-8') as f: json.dump(stats, f, ensure_ascii=False, indent=4) return stats if __name__ == "__main__": # 使用指定的文件路径调用处理函数 stats = process_collision_data( input_file="./basicData_historyDepth_v2_img106k_250723__huanshi_6ver_4fish_large_model_900_792874_10frames.jsonl", output_file="processed_106k_0723_tag02.jsonl", stats_file="collision_stats_0723_tag02.json", threshold=0.7, debug=False # 设置为True可查看详细处理日志 ) # 打印摘要统计 print("\n处理完成!") print(f"总场景数: {stats['total_scenes']}") print(f"总帧数: {stats['overall']['total_frames']}") print(f"YES标签数: {stats['overall']['yes_count']}") print(f"NO标签数: {stats['overall']['no_count']}") print(f"YES占比: {stats['overall']['yes_ratio']:.2%}") print(f"NO占比: {stats['overall']['no_ratio']:.2%}") print(f"输出文件: {stats['output_file']}") print(f"统计文件: {stats['stats_file']}") 以该脚本 为基础,上述生成脚本为样本,优化此脚本
07-30
import json import os from math import trunc import jsonlines from collections import defaultdict from typing import Dict, List, Any, Tuple import logging import time from dataclasses import dataclass, field # 配置日志 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler("collision_processing.log"), logging.StreamHandler() ] ) logger = logging.getLogger("CollisionProcessor") # 数据类定义 @dataclass class FrameData: timestamp: float road_points: List[Tuple[float, float]] record: Dict[str, Any] group_key: str = "" scene_id: str = "" processed: bool = False @dataclass class CameraStats: cam_tag: str total_frames: int = 0 yes_count: int = 0 no_count: int = 0 @dataclass class SceneStats: scene_id: str total_cam_tags: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 cam_tags: Dict[str, CameraStats] = field(default_factory=dict) @dataclass class OverallStats: total_scenes: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 def extract_group_key(image_path: str) -> str: """解析文件路径中的相机标签""" try: parts = image_path.replace("\\", "/").strip("/").split("/") return parts[-2] if len(parts) >= 2 else "default_group" except Exception as e: logger.error(f"提取分组键错误: {e}, 路径: {image_path}") return "default_group" def round_coordinate(value: float, decimals: int = 2) -> float: """四舍五入坐标值到指定小数位数""" if value is None: return 0.0 factor = 10 ** decimals return round(value * factor) / factor def process_road_points(road_points: List[Dict[str, float]]) -> List[Tuple[float, float]]: """处理道路点数据,提取并四舍五入坐标""" if not road_points: return [] return [ (round_coordinate(point.get('x', 0.0)), round_coordinate(point.get('y', 0.0))) for point in road_points ] def calculate_overlap_ratio( points1: List[Tuple[float, float]], points2: List[Tuple[float, float]], tolerance: float = 0.01 ) -> float: """计算两组点之间的重叠率""" if not points1 or not points2: return 0.0 # 使用集合计算重叠率 set1 = {(trunc(x), trunc(y)) for x, y in points1} set2 = {(trunc(x), trunc(y)) for x, y in points2} # set1 = {(round(x, 2), round(y, 2)) for x, y in points1} # set2 = {(round(x, 2), round(y, 2)) for x, y in points2} intersection = set1 & set2 union = set1 | set2 return len(intersection) / len(union) if union else 0.0 def load_data(input_file: str) -> Dict[str, Dict[str, List[FrameData]]]: """从JSONL文件加载数据并分组""" grouped_data = defaultdict(lambda: defaultdict(list)) total_records = 0 logger.info(f"开始读取输入文件: {input_file}") try: with open(input_file, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): try: record = json.loads(line.strip()) scene_id = record.get('scene_id', 'unknown_scene') image_path = record.get('image_path', '') timestamp = record.get('timestamp', 0.0) # 提取道路点信息 mop_info = record.get('mop_pnc_info', {}) if not isinstance(mop_info, dict): mop_info = {} road_points = mop_info.get('roadPoints', []) # 处理道路点 processed_points = process_road_points(road_points) # 提取分组键 group_key = extract_group_key(image_path) # 创建FrameData对象 frame_data = FrameData( timestamp=timestamp, road_points=processed_points, record=record, group_key=group_key, scene_id=scene_id ) # 添加到分组数据 grouped_data[scene_id][group_key].append(frame_data) total_records += 1 except json.JSONDecodeError as e: logger.error(f"JSON解析错误 (行 {line_num}): {e}") except Exception as e: logger.error(f"处理记录错误 (行 {line_num}): {e}") except IOError as e: logger.error(f"文件读取错误: {e}") return {} logger.info(f"成功读取 {total_records} 条记录,分组到 {len(grouped_data)} 个场景") return grouped_data def process_frame_group( frames: List[FrameData], threshold: float = 0.7, debug: bool = False ) -> CameraStats: """处理一组帧数据,使用新的打标逻辑""" if not frames: return CameraStats(cam_tag="") # 获取相机标签 cam_tag = frames[0].group_key cam_stats = CameraStats(cam_tag=cam_tag) cam_stats.total_frames = len(frames) # 按时间戳降序排序(最新时间在前) frames.sort(key=lambda x: x.timestamp, reverse=True) # 标记是否已出现第一个no no_occurred = False # 处理所有帧 for i, frame in enumerate(frames): if i == 0: # 第一帧(最新帧) frame.record['mask'] = 'yes' cam_stats.yes_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧0 (最新) 标签设置为 'yes'") elif i == 1: # 第二帧 # 与第一帧比较 overlap = calculate_overlap_ratio( frames[0].road_points, frame.road_points ) if overlap >= threshold: frame.record['mask'] = 'yes' cam_stats.yes_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧1 与帧0重叠度 {overlap:.2f} >= {threshold} -> 标签设置为 'yes'") else: frame.record['mask'] = 'no' cam_stats.no_count += 1 no_occurred = True # 标记已出现第一个no if debug: logger.debug( f"相机 {cam_tag}: 帧1 与帧0重叠度 {overlap:.2f} < {threshold} -> 标签设置为 'no' (首次出现)") else: # 第三帧及以后 if no_occurred: # 一旦出现第一个no,后续所有帧都是no frame.record['mask'] = 'no' cam_stats.no_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧{i} 已出现no -> 标签设置为 'no'") else: # 与前一帧比较 overlap = calculate_overlap_ratio( frames[i - 1].road_points, frame.road_points ) if overlap >= threshold: frame.record['mask'] = frames[i - 1].record['mask'] if frame.record['mask'] == 'yes': cam_stats.yes_count += 1 else: cam_stats.no_count += 1 if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} >= {threshold} -> 继承标签 '{frame.record['mask']}'") else: frame.record['mask'] = 'no' cam_stats.no_count += 1 no_occurred = True # 标记已出现第一个no if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} < {threshold} -> 标签设置为 'no' (首次出现)") frame.processed = True return cam_stats def process_collision_data( input_file: str, output_file: str, stats_file: str, threshold: float = 0.7, debug: bool = False ) -> Dict[str, Any]: """处理碰撞数据的核心函数""" start_time = time.time() logger.info(f"开始处理碰撞数据: {input_file}") # 加载数据 grouped_data = load_data(input_file) if not grouped_data: logger.error("未加载到有效数据,处理终止") return {} # 初始化统计数据结构 overall_stats = OverallStats(total_scenes=len(grouped_data)) scene_stats_dict = {} labeled_data = [] # 处理每个场景 for scene_id, cam_groups in grouped_data.items(): scene_stats = SceneStats( scene_id=scene_id, total_cam_tags=len(cam_groups) ) # 处理每个相机组 for group_key, frames in cam_groups.items(): cam_stats = process_frame_group(frames, threshold, debug) # 更新场景统计 scene_stats.cam_tags[group_key] = cam_stats scene_stats.total_frames += cam_stats.total_frames scene_stats.yes_count += cam_stats.yes_count scene_stats.no_count += cam_stats.no_count # 收集处理后的数据 labeled_data.extend(frame.record for frame in frames if frame.processed) # 更新总体统计 scene_stats_dict[scene_id] = scene_stats overall_stats.total_frames += scene_stats.total_frames overall_stats.yes_count += scene_stats.yes_count overall_stats.no_count += scene_stats.no_count # 按scene_id、cam_tag和timestamp降序排序 labeled_data.sort(key=lambda x: ( x.get('scene_id', ''), x.get('image_path', '').split('/')[-2] if 'image_path' in x else '', -x.get('timestamp', 0) )) # 准备输出统计信息 stats = { "stats_file": os.path.basename(stats_file), "total_scenes": overall_stats.total_scenes, "scenes": {}, "overall": { "total_frames": overall_stats.total_frames, "yes_count": overall_stats.yes_count, "no_count": overall_stats.no_count } } # 添加场景统计 for scene_id, scene_stats in scene_stats_dict.items(): stats["scenes"][scene_id] = { "scene_id": scene_stats.scene_id, "total_cam_tags": scene_stats.total_cam_tags, "total_frames": scene_stats.total_frames, "yes_count": scene_stats.yes_count, "no_count": scene_stats.no_count, "cam_tags": {} } # 添加相机统计 for cam_tag, cam_stats in scene_stats.cam_tags.items(): stats["scenes"][scene_id]["cam_tags"][cam_tag] = { "cam_tag": cam_stats.cam_tag, "total_frames": cam_stats.total_frames, "yes_count": cam_stats.yes_count, "no_count": cam_stats.no_count } # 确保输出目录存在 os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True) os.makedirs(os.path.dirname(stats_file) or '.', exist_ok=True) # 写入输出文件 logger.info(f"写入处理后的数据到: {output_file}") with jsonlines.open(output_file, 'w') as writer: for record in labeled_data: writer.write(record) # 写入统计文件 logger.info(f"写入统计信息到: {stats_file}") with open(stats_file, 'w', encoding='utf-8') as f: json.dump(stats, f, ensure_ascii=False, indent=4) # 计算处理时间 processing_time = time.time() - start_time logger.info(f"处理完成! 总耗时: {processing_time:.2f}秒") return stats if __name__ == "__main__": # 使用指定的文件路径调用处理函数 stats = process_collision_data( #input_file="./basicData_historyDepth_v2_img106k_250723__huanshi_6ver_4fish_large_model_900_792874_10frames.jsonl", input_file="./basicData_realCollision_v2_img42k_250726__real_collision_dataset_pt1_to_8.jsonl", #input_file="./basicData_vggtHDepth_v2_img42k_250725__huanshi_6ver_6pinhole__large_model_900_792874_10frames__pt1__250722.jsonl", output_file="processed_42k_0726_tag03.jsonl", stats_file="collision_stats_0726_tag03.json", threshold=0.1, debug=False # 设置为True可查看详细处理日志 ) # 打印摘要统计 if stats: total_frames = stats['overall']['total_frames'] yes_count = stats['overall']['yes_count'] no_count = stats['overall']['no_count'] print("\n处理完成!") print(f"总场景数: {stats['total_scenes']}") print(f"总帧数: {total_frames}") print(f"YES标签数: {yes_count} ({yes_count / total_frames:.2%})") print(f"NO标签数: {no_count} ({no_count / total_frames:.2%})") print(f"输出文件: processed_106k_0723_tag03.jsonl") print(f"统计文件: collision_stats_0723_tag03.json") else: print("处理失败,请检查日志文件获取详细信息") 修改代码对roadpoint的值取小数点后两位
08-01
import json import os import jsonlines from collections import defaultdict from typing import Dict, List, Any, Tuple import logging import time from dataclasses import dataclass, field # 配置日志 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler("collision_processing.log"), logging.StreamHandler() ] ) logger = logging.getLogger("CollisionProcessor") # 数据类定义 @dataclass class FrameData: timestamp: float road_points: List[Tuple[float, float]] record: Dict[str, Any] group_key: str = "" scene_id: str = "" processed: bool = False @dataclass class CameraStats: cam_tag: str total_frames: int = 0 yes_count: int = 0 no_count: int = 0 @dataclass class SceneStats: scene_id: str total_cam_tags: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 cam_tags: Dict[str, CameraStats] = field(default_factory=dict) @dataclass class OverallStats: total_scenes: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 def extract_group_key(image_path: str) -> str: """解析文件路径中的相机标签""" try: parts = image_path.replace("\\", "/").strip("/").split("/") return parts[-2] if len(parts) >= 2 else "default_group" except Exception as e: logger.error(f"提取分组键错误: {e}, 路径: {image_path}") return "default_group" def round_coordinate(value: float, decimals: int = 2) -> float: """四舍五入坐标值到指定小数位数""" if value is None: return 0.0 return round(value, decimals) def process_road_points(road_points: List[Dict[str, float]]) -> List[Tuple[float, float]]: """处理道路点数据,提取并四舍五入坐标""" if not road_points: return [] return [ (round_coordinate(point.get('x', 0.0)), round_coordinate(point.get('y', 0.0))) for point in road_points ] def calculate_overlap_ratio( points1: List[Tuple[float, float]], points2: List[Tuple[float, float]], tolerance: float = 0.01 ) -> float: """计算两组点之间的重叠率(带容忍度)""" if not points1 or not points2: return 0.0 # 创建带容忍度的点集 def create_tolerant_set(points): return {(round(x / tolerance) * tolerance, round(y / tolerance) * tolerance) for x, y in points} set1 = create_tolerant_set(points1) set2 = create_tolerant_set(points2) intersection = set1 & set2 union = set1 | set2 return len(intersection) / len(union) if union else 0.0 def load_data(input_file: str) -> Dict[str, Dict[str, List[FrameData]]]: """从JSONL文件加载数据并分组""" grouped_data = defaultdict(lambda: defaultdict(list)) total_records = 0 logger.info(f"开始读取输入文件: {input_file}") try: with open(input_file, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): try: record = json.loads(line.strip()) scene_id = record.get('scene_id', 'unknown_scene') image_path = record.get('image_path', '') timestamp = record.get('timestamp', 0.0) # 提取道路点信息 mop_info = record.get('mop_pnc_info', {}) if not isinstance(mop_info, dict): mop_info = {} road_points = mop_info.get('roadPoints', []) # 处理道路点 processed_points = process_road_points(road_points) # 提取分组键 group_key = extract_group_key(image_path) # 创建FrameData对象 frame_data = FrameData( timestamp=timestamp, road_points=processed_points, record=record, group_key=group_key, scene_id=scene_id ) # 添加到分组数据 grouped_data[scene_id][group_key].append(frame_data) total_records += 1 except json.JSONDecodeError as e: logger.error(f"JSON解析错误 (行 {line_num}): {e}") except Exception as e: logger.error(f"处理记录错误 (行 {line_num}): {e}") except IOError as e: logger.error(f"文件读取错误: {e}") return {} logger.info(f"成功读取 {total_records} 条记录,分组到 {len(grouped_data)} 个场景") return grouped_data def process_frame_group( frames: List[FrameData], threshold: float = 0.7, debug: bool = False ) -> CameraStats: """处理一组帧数据,使用新的打标逻辑""" if not frames: return CameraStats(cam_tag="") # 获取相机标签 cam_tag = frames[0].group_key cam_stats = CameraStats(cam_tag=cam_tag) cam_stats.total_frames = len(frames) # 按时间戳降序排序(最新时间在前) frames.sort(key=lambda x: x.timestamp, reverse=True) # 标记是否已出现第一个no no_occurred = False # 处理所有帧 for i, frame in enumerate(frames): if i == 0: # 第一帧(最新帧) frame.record['mask'] = 'yes' cam_stats.yes_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧0 (最新) 标签设置为 'yes'") else: # 后续帧 if no_occurred: # 一旦出现第一个no,后续所有帧都是no frame.record['mask'] = 'no' cam_stats.no_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧{i} 已出现no -> 标签设置为 'no'") else: # 与前一帧比较 overlap = calculate_overlap_ratio( frames[i - 1].road_points, frame.road_points ) if overlap >= threshold: # 继承前一帧的mask frame.record['mask'] = frames[i - 1].record['mask'] if frame.record['mask'] == 'yes': cam_stats.yes_count += 1 else: cam_stats.no_count += 1 if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} >= {threshold} -> 继承标签 '{frame.record['mask']}'") else: frame.record['mask'] = 'no' cam_stats.no_count += 1 no_occurred = True # 标记已出现第一个no if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} < {threshold} -> 标签设置为 'no' (首次出现)") frame.processed = True return cam_stats def process_collision_data( input_file: str, output_file: str, stats_file: str, threshold: float = 0.7, debug: bool = False ) -> Dict[str, Any]: """处理碰撞数据的核心函数""" start_time = time.time() logger.info(f"开始处理碰撞数据: {input_file}") # 加载数据 grouped_data = load_data(input_file) if not grouped_data: logger.error("未加载到有效数据,处理终止") return {} # 初始化统计数据结构 overall_stats = OverallStats(total_scenes=len(grouped_data)) scene_stats_dict = {} labeled_data = [] # 处理每个场景 for scene_id, cam_groups in grouped_data.items(): scene_stats = SceneStats( scene_id=scene_id, total_cam_tags=len(cam_groups) ) # 处理每个相机组 for group_key, frames in cam_groups.items(): # 在每个相机组内按时间戳降序排序 frames.sort(key=lambda x: x.timestamp, reverse=True) cam_stats = process_frame_group(frames, threshold, debug) # 更新场景统计 scene_stats.cam_tags[group_key] = cam_stats scene_stats.total_frames += cam_stats.total_frames scene_stats.yes_count += cam_stats.yes_count scene_stats.no_count += cam_stats.no_count # 收集处理后的数据 labeled_data.extend(frame.record for frame in frames if frame.processed) # 更新总体统计 scene_stats_dict[scene_id] = scene_stats overall_stats.total_frames += scene_stats.total_frames overall_stats.yes_count += scene_stats.yes_count overall_stats.no_count += scene_stats.no_count # 按scene_id、cam_tag和timestamp降序排序 labeled_data.sort(key=lambda x: ( x.get('scene_id', ''), x.get('image_path', '').split('/')[-2] if 'image_path' in x else '', -x.get('timestamp', 0) )) # 准备输出统计信息 stats = { "stats_file": os.path.basename(stats_file), "total_scenes": overall_stats.total_scenes, "scenes": {}, "overall": { "total_frames": overall_stats.total_frames, "yes_count": overall_stats.yes_count, "no_count": overall_stats.no_count } } # 添加场景统计 for scene_id, scene_stats in scene_stats_dict.items(): stats["scenes"][scene_id] = { "scene_id": scene_stats.scene_id, "total_cam_tags": scene_stats.total_cam_tags, "total_frames": scene_stats.total_frames, "yes_count": scene_stats.yes_count, "no_count": scene_stats.no_count, "cam_tags": {} } # 添加相机统计 for cam_tag, cam_stats in scene_stats.cam_tags.items(): stats["scenes"][scene_id]["cam_tags"][cam_tag] = { "cam_tag": cam_stats.cam_tag, "total_frames": cam_stats.total_frames, "yes_count": cam_stats.yes_count, "no_count": cam_stats.no_count } # 确保输出目录存在 os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True) os.makedirs(os.path.dirname(stats_file) or '.', exist_ok=True) # 写入输出文件 logger.info(f"写入处理后的数据到: {output_file}") with jsonlines.open(output_file, 'w') as writer: for record in labeled_data: writer.write(record) # 写入统计文件 logger.info(f"写入统计信息到: {stats_file}") with open(stats_file, 'w', encoding='utf-8') as f: json.dump(stats, f, ensure_ascii=False, indent=4) # 计算处理时间 processing_time = time.time() - start_time logger.info(f"处理完成! 总耗时: {processing_time:.2f}秒") return stats if __name__ == "__main__": # 使用指定的文件路径调用处理函数 stats = process_collision_data( input_file="./basicData_realCollision_v2_img42k_250726__real_collision_dataset_pt1_to_8.jsonl", output_file="processed_42k_0726_tag03.jsonl", stats_file="collision_stats_0726_tag03.json", threshold=0.7, # 使用0.7作为阈值 debug=False # 设置为True可查看详细处理日志 ) # 打印摘要统计 if stats: total_frames = stats['overall']['total_frames'] yes_count = stats['overall']['yes_count'] no_count = stats['overall']['no_count'] print("\n处理完成!") print(f"总场景数: {stats['total_scenes']}") print(f"总帧数: {total_frames}") print(f"YES标签数: {yes_count} ({yes_count / total_frames:.2%})") print(f"NO标签数: {no_count} ({no_count / total_frames:.2%})") print(f"输出文件: processed_42k_0726_tag03.jsonl") print(f"统计文件: collision_stats_0726_tag03.json") else: print("处理失败,请检查日志文件获取详细信息") import json import os import jsonlines from collections import defaultdict from typing import Dict, List, Any, Tuple import logging import time from dataclasses import dataclass, field # 配置日志 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler("collision_processing.log"), logging.StreamHandler() ] ) logger = logging.getLogger("CollisionProcessor") # 数据类定义 @dataclass class FrameData: timestamp: float road_points: List[Tuple[float, float]] record: Dict[str, Any] group_key: str = "" scene_id: str = "" processed: bool = False @dataclass class CameraStats: cam_tag: str total_frames: int = 0 yes_count: int = 0 no_count: int = 0 @dataclass class SceneStats: scene_id: str total_cam_tags: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 cam_tags: Dict[str, CameraStats] = field(default_factory=dict) @dataclass class OverallStats: total_scenes: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 def extract_group_key(image_path: str) -> str: """解析文件路径中的相机标签""" try: parts = image_path.replace("\\", "/").strip("/").split("/") return parts[-2] if len(parts) >= 2 else "default_group" except Exception as e: logger.error(f"提取分组键错误: {e}, 路径: {image_path}") return "default_group" def round_coordinate(value: float, decimals: int = 2) -> float: """四舍五入坐标值到指定小数位数""" if value is None: return 0.0 return round(value, decimals) def process_road_points(road_points: List[Dict[str, float]]) -> List[Tuple[float, float]]: """处理道路点数据,提取并四舍五入坐标""" if not road_points: return [] return [ (round_coordinate(point.get('x', 0.0)), round_coordinate(point.get('y', 0.0))) for point in road_points ] def calculate_overlap_ratio( points1: List[Tuple[float, float]], points2: List[Tuple[float, float]], tolerance: float = 0.01 ) -> float: """计算两组点之间的重叠率(带容忍度)""" if not points1 or not points2: return 0.0 # 创建带容忍度的点集 def create_tolerant_set(points): return {(round(x / tolerance) * tolerance, round(y / tolerance) * tolerance) for x, y in points} set1 = create_tolerant_set(points1) set2 = create_tolerant_set(points2) intersection = set1 & set2 union = set1 | set2 return len(intersection) / len(union) if union else 0.0 def load_data(input_file: str) -> Dict[str, Dict[str, List[FrameData]]]: """从JSONL文件加载数据并分组""" grouped_data = defaultdict(lambda: defaultdict(list)) total_records = 0 logger.info(f"开始读取输入文件: {input_file}") try: with open(input_file, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): try: record = json.loads(line.strip()) scene_id = record.get('scene_id', 'unknown_scene') image_path = record.get('image_path', '') timestamp = record.get('timestamp', 0.0) # 提取道路点信息 mop_info = record.get('mop_pnc_info', {}) if not isinstance(mop_info, dict): mop_info = {} road_points = mop_info.get('roadPoints', []) # 处理道路点 processed_points = process_road_points(road_points) # 提取分组键 group_key = extract_group_key(image_path) # 创建FrameData对象 frame_data = FrameData( timestamp=timestamp, road_points=processed_points, record=record, group_key=group_key, scene_id=scene_id ) # 添加到分组数据 grouped_data[scene_id][group_key].append(frame_data) total_records += 1 except json.JSONDecodeError as e: logger.error(f"JSON解析错误 (行 {line_num}): {e}") except Exception as e: logger.error(f"处理记录错误 (行 {line_num}): {e}") except IOError as e: logger.error(f"文件读取错误: {e}") return {} logger.info(f"成功读取 {total_records} 条记录,分组到 {len(grouped_data)} 个场景") return grouped_data def process_frame_group( frames: List[FrameData], threshold: float = 0.7, debug: bool = False ) -> CameraStats: """处理一组帧数据,使用新的打标逻辑""" if not frames: return CameraStats(cam_tag="") # 获取相机标签 cam_tag = frames[0].group_key cam_stats = CameraStats(cam_tag=cam_tag) cam_stats.total_frames = len(frames) # 按时间戳降序排序(最新时间在前) frames.sort(key=lambda x: x.timestamp, reverse=True) # 标记是否已出现第一个no no_occurred = False # 处理所有帧 for i, frame in enumerate(frames): if i == 0: # 第一帧(最新帧) frame.record['mask'] = 'yes' cam_stats.yes_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧0 (最新) 标签设置为 'yes'") else: # 后续帧 if no_occurred: # 一旦出现第一个no,后续所有帧都是no frame.record['mask'] = 'no' cam_stats.no_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧{i} 已出现no -> 标签设置为 'no'") else: # 与前一帧比较 overlap = calculate_overlap_ratio( frames[i - 1].road_points, frame.road_points ) if overlap >= threshold: # 继承前一帧的mask frame.record['mask'] = frames[i - 1].record['mask'] if frame.record['mask'] == 'yes': cam_stats.yes_count += 1 else: cam_stats.no_count += 1 if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} >= {threshold} -> 继承标签 '{frame.record['mask']}'") else: frame.record['mask'] = 'no' cam_stats.no_count += 1 no_occurred = True # 标记已出现第一个no if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} < {threshold} -> 标签设置为 'no' (首次出现)") frame.processed = True return cam_stats def process_collision_data( input_file: str, output_file: str, stats_file: str, threshold: float = 0.7, debug: bool = False ) -> Dict[str, Any]: """处理碰撞数据的核心函数""" start_time = time.time() logger.info(f"开始处理碰撞数据: {input_file}") # 加载数据 grouped_data = load_data(input_file) if not grouped_data: logger.error("未加载到有效数据,处理终止") return {} # 初始化统计数据结构 overall_stats = OverallStats(total_scenes=len(grouped_data)) scene_stats_dict = {} labeled_data = [] # 处理每个场景 for scene_id, cam_groups in grouped_data.items(): scene_stats = SceneStats( scene_id=scene_id, total_cam_tags=len(cam_groups) ) # 处理每个相机组 for group_key, frames in cam_groups.items(): # 在每个相机组内按时间戳降序排序 frames.sort(key=lambda x: x.timestamp, reverse=True) cam_stats = process_frame_group(frames, threshold, debug) # 更新场景统计 scene_stats.cam_tags[group_key] = cam_stats scene_stats.total_frames += cam_stats.total_frames scene_stats.yes_count += cam_stats.yes_count scene_stats.no_count += cam_stats.no_count # 收集处理后的数据 labeled_data.extend(frame.record for frame in frames if frame.processed) # 更新总体统计 scene_stats_dict[scene_id] = scene_stats overall_stats.total_frames += scene_stats.total_frames overall_stats.yes_count += scene_stats.yes_count overall_stats.no_count += scene_stats.no_count # 按scene_id、cam_tag和timestamp降序排序 labeled_data.sort(key=lambda x: ( x.get('scene_id', ''), x.get('image_path', '').split('/')[-2] if 'image_path' in x else '', -x.get('timestamp', 0) )) # 准备输出统计信息 stats = { "stats_file": os.path.basename(stats_file), "total_scenes": overall_stats.total_scenes, "scenes": {}, "overall": { "total_frames": overall_stats.total_frames, "yes_count": overall_stats.yes_count, "no_count": overall_stats.no_count } } # 添加场景统计 for scene_id, scene_stats in scene_stats_dict.items(): stats["scenes"][scene_id] = { "scene_id": scene_stats.scene_id, "total_cam_tags": scene_stats.total_cam_tags, "total_frames": scene_stats.total_frames, "yes_count": scene_stats.yes_count, "no_count": scene_stats.no_count, "cam_tags": {} } # 添加相机统计 for cam_tag, cam_stats in scene_stats.cam_tags.items(): stats["scenes"][scene_id]["cam_tags"][cam_tag] = { "cam_tag": cam_stats.cam_tag, "total_frames": cam_stats.total_frames, "yes_count": cam_stats.yes_count, "no_count": cam_stats.no_count } # 确保输出目录存在 os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True) os.makedirs(os.path.dirname(stats_file) or '.', exist_ok=True) # 写入输出文件 logger.info(f"写入处理后的数据到: {output_file}") with jsonlines.open(output_file, 'w') as writer: for record in labeled_data: writer.write(record) # 写入统计文件 logger.info(f"写入统计信息到: {stats_file}") with open(stats_file, 'w', encoding='utf-8') as f: json.dump(stats, f, ensure_ascii=False, indent=4) # 计算处理时间 processing_time = time.time() - start_time logger.info(f"处理完成! 总耗时: {processing_time:.2f}秒") return stats if __name__ == "__main__": # 使用指定的文件路径调用处理函数 stats = process_collision_data( input_file="./basicData_realCollision_v2_img42k_250726__real_collision_dataset_pt1_to_8.jsonl", output_file="processed_42k_0726_tag03.jsonl", stats_file="collision_stats_0726_tag03.json", threshold=0.7, # 使用0.7作为阈值 debug=False # 设置为True可查看详细处理日志 ) # 打印摘要统计 if stats: total_frames = stats['overall']['total_frames'] yes_count = stats['overall']['yes_count'] no_count = stats['overall']['no_count'] print("\n处理完成!") print(f"总场景数: {stats['total_scenes']}") print(f"总帧数: {total_frames}") print(f"YES标签数: {yes_count} ({yes_count / total_frames:.2%})") print(f"NO标签数: {no_count} ({no_count / total_frames:.2%})") print(f"输出文件: processed_42k_0726_tag03.jsonl") print(f"统计文件: collision_stats_0726_tag03.json") else: print("处理失败,请检查日志文件获取详细信息") 并按分组按顺序保存roadpoint数据到excel
08-05
import json import os from math import trunc import jsonlines from collections import defaultdict from typing import Dict, List, Any, Tuple import logging import time from dataclasses import dataclass, field 配置日志 logging.basicConfig( level=logging.INFO, format=‘%(asctime)s - %(name)s - %(levelname)s - %(message)s’, handlers=[ logging.FileHandler(“collision_processing.log”), logging.StreamHandler() ] ) logger = logging.getLogger(“CollisionProcessor”) 数据类定义 @dataclass class FrameData: timestamp: float road_points: List[Tuple[float, float]] record: Dict[str, Any] group_key: str = “” scene_id: str = “” processed: bool = False @dataclass class CameraStats: cam_tag: str total_frames: int = 0 yes_count: int = 0 no_count: int = 0 @dataclass class SceneStats: scene_id: str total_cam_tags: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 cam_tags: Dict[str, CameraStats] = field(default_factory=dict) @dataclass class OverallStats: total_scenes: int = 0 total_frames: int = 0 yes_count: int = 0 no_count: int = 0 def extract_group_key(image_path: str) -> str: “”“解析文件路径中的相机标签”“” try: parts = image_path.replace(“\”, “/”).strip(“/”).split(“/”) return parts[-2] if len(parts) >= 2 else “default_group” except Exception as e: logger.error(f"提取分组键错误: {e}, 路径: {image_path}") return “default_group” def round_coordinate(value: float, decimals: int = 2) -> float: “”“四舍五入坐标值到指定小数位数”“” if value is None: return 0.0 factor = 10 ** decimals return round(value * factor) / factor def process_road_points(road_points: List[Dict[str, float]]) -> List[Tuple[float, float]]: “”“处理道路点数据,提取并四舍五入坐标”“” if not road_points: return [] return [ (round_coordinate(point.get(‘x’, 0.0)), round_coordinate(point.get(‘y’, 0.0))) for point in road_points ] def calculate_overlap_ratio( points1: List[Tuple[float, float]], points2: List[Tuple[float, float]], tolerance: float = 0.01 ) -> float: “”“计算两组点之间的重叠率”“” if not points1 or not points2: return 0.0 # 使用集合计算重叠率 set1 = {(trunc(x), trunc(y)) for x, y in points1} set2 = {(trunc(x), trunc(y)) for x, y in points2} # set1 = {(round(x, 2), round(y, 2)) for x, y in points1} # set2 = {(round(x, 2), round(y, 2)) for x, y in points2} intersection = set1 & set2 union = set1 | set2 return len(intersection) / len(union) if union else 0.0 def load_data(input_file: str) -> Dict[str, Dict[str, List[FrameData]]]: “”“从JSONL文件加载数据并分组”“” grouped_data = defaultdict(lambda: defaultdict(list)) total_records = 0 logger.info(f"开始读取输入文件: {input_file}") try: with open(input_file, 'r', encoding='utf-8') as f: for line_num, line in enumerate(f, 1): try: record = json.loads(line.strip()) scene_id = record.get('scene_id', 'unknown_scene') image_path = record.get('image_path', '') timestamp = record.get('timestamp', 0.0) # 提取道路点信息 mop_info = record.get('mop_pnc_info', {}) if not isinstance(mop_info, dict): mop_info = {} road_points = mop_info.get('roadPoints', []) # 处理道路点 processed_points = process_road_points(road_points) # 提取分组键 group_key = extract_group_key(image_path) # 创建FrameData对象 frame_data = FrameData( timestamp=timestamp, road_points=processed_points, record=record, group_key=group_key, scene_id=scene_id ) # 添加到分组数据 grouped_data[scene_id][group_key].append(frame_data) total_records += 1 except json.JSONDecodeError as e: logger.error(f"JSON解析错误 (行 {line_num}): {e}") except Exception as e: logger.error(f"处理记录错误 (行 {line_num}): {e}") except IOError as e: logger.error(f"文件读取错误: {e}") return {} logger.info(f"成功读取 {total_records} 条记录,分组到 {len(grouped_data)} 个场景") return grouped_data def process_frame_group( frames: List[FrameData], threshold: float = 0.7, debug: bool = False ) -> CameraStats: “”“处理一组帧数据,使用新的打标逻辑”“” if not frames: return CameraStats(cam_tag=“”) # 获取相机标签 cam_tag = frames[0].group_key cam_stats = CameraStats(cam_tag=cam_tag) cam_stats.total_frames = len(frames) # 按时间戳降序排序(最新时间在前) frames.sort(key=lambda x: x.timestamp, reverse=True) # 标记是否已出现第一个no no_occurred = False # 处理所有帧 for i, frame in enumerate(frames): if i == 0: # 第一帧(最新帧) frame.record['mask'] = 'yes' cam_stats.yes_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧0 (最新) 标签设置为 'yes'") elif i == 1: # 第二帧 # 与第一帧比较 overlap = calculate_overlap_ratio( frames[0].road_points, frame.road_points ) if overlap >= threshold: frame.record['mask'] = 'yes' cam_stats.yes_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧1 与帧0重叠度 {overlap:.2f} >= {threshold} -> 标签设置为 'yes'") else: frame.record['mask'] = 'no' cam_stats.no_count += 1 no_occurred = True # 标记已出现第一个no if debug: logger.debug( f"相机 {cam_tag}: 帧1 与帧0重叠度 {overlap:.2f} < {threshold} -> 标签设置为 'no' (首次出现)") else: # 第三帧及以后 if no_occurred: # 一旦出现第一个no,后续所有帧都是no frame.record['mask'] = 'no' cam_stats.no_count += 1 if debug: logger.debug(f"相机 {cam_tag}: 帧{i} 已出现no -> 标签设置为 'no'") else: # 与前一帧比较 overlap = calculate_overlap_ratio( frames[i - 1].road_points, frame.road_points ) if overlap >= threshold: frame.record['mask'] = frames[i - 1].record['mask'] if frame.record['mask'] == 'yes': cam_stats.yes_count += 1 else: cam_stats.no_count += 1 if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} >= {threshold} -> 继承标签 '{frame.record['mask']}'") else: frame.record['mask'] = 'no' cam_stats.no_count += 1 no_occurred = True # 标记已出现第一个no if debug: logger.debug( f"相机 {cam_tag}: 帧{i} 与帧{i - 1}重叠度 {overlap:.2f} < {threshold} -> 标签设置为 'no' (首次出现)") frame.processed = True return cam_stats def process_collision_data( input_file: str, output_file: str, stats_file: str, threshold: float = 0.7, debug: bool = False ) -> Dict[str, Any]: “”“处理碰撞数据的核心函数”“” start_time = time.time() logger.info(f"开始处理碰撞数据: {input_file}") # 加载数据 grouped_data = load_data(input_file) if not grouped_data: logger.error("未加载到有效数据,处理终止") return {} # 初始化统计数据结构 overall_stats = OverallStats(total_scenes=len(grouped_data)) scene_stats_dict = {} labeled_data = [] # 处理每个场景 for scene_id, cam_groups in grouped_data.items(): scene_stats = SceneStats( scene_id=scene_id, total_cam_tags=len(cam_groups) ) # 处理每个相机组 for group_key, frames in cam_groups.items(): cam_stats = process_frame_group(frames, threshold, debug) # 更新场景统计 scene_stats.cam_tags[group_key] = cam_stats scene_stats.total_frames += cam_stats.total_frames scene_stats.yes_count += cam_stats.yes_count scene_stats.no_count += cam_stats.no_count # 收集处理后的数据 labeled_data.extend(frame.record for frame in frames if frame.processed) # 更新总体统计 scene_stats_dict[scene_id] = scene_stats overall_stats.total_frames += scene_stats.total_frames overall_stats.yes_count += scene_stats.yes_count overall_stats.no_count += scene_stats.no_count # 按scene_id、cam_tag和timestamp降序排序 labeled_data.sort(key=lambda x: ( x.get('scene_id', ''), x.get('image_path', '').split('/')[-2] if 'image_path' in x else '', -x.get('timestamp', 0) )) # 准备输出统计信息 stats = { "stats_file": os.path.basename(stats_file), "total_scenes": overall_stats.total_scenes, "scenes": {}, "overall": { "total_frames": overall_stats.total_frames, "yes_count": overall_stats.yes_count, "no_count": overall_stats.no_count } } # 添加场景统计 for scene_id, scene_stats in scene_stats_dict.items(): stats["scenes"][scene_id] = { "scene_id": scene_stats.scene_id, "total_cam_tags": scene_stats.total_cam_tags, "total_frames": scene_stats.total_frames, "yes_count": scene_stats.yes_count, "no_count": scene_stats.no_count, "cam_tags": {} } # 添加相机统计 for cam_tag, cam_stats in scene_stats.cam_tags.items(): stats["scenes"][scene_id]["cam_tags"][cam_tag] = { "cam_tag": cam_stats.cam_tag, "total_frames": cam_stats.total_frames, "yes_count": cam_stats.yes_count, "no_count": cam_stats.no_count } # 确保输出目录存在 os.makedirs(os.path.dirname(output_file) or '.', exist_ok=True) os.makedirs(os.path.dirname(stats_file) or '.', exist_ok=True) # 写入输出文件 logger.info(f"写入处理后的数据到: {output_file}") with jsonlines.open(output_file, 'w') as writer: for record in labeled_data: writer.write(record) # 写入统计文件 logger.info(f"写入统计信息到: {stats_file}") with open(stats_file, 'w', encoding='utf-8') as f: json.dump(stats, f, ensure_ascii=False, indent=4) # 计算处理时间 processing_time = time.time() - start_time logger.info(f"处理完成! 总耗时: {processing_time:.2f}秒") return stats if name == “main”: # 使用指定的文件路径调用处理函数 stats = process_collision_data( #input_file=“./basicData_historyDepth_v2_img106k_250723__huanshi_6ver_4fish_large_model_900_792874_10frames.jsonl”, input_file=“./basicData_realCollision_v2_img42k_250726__real_collision_dataset_pt1_to_8.jsonl”, #input_file=“./basicData_vggtHDepth_v2_img42k_250725__huanshi_6ver_6pinhole__large_model_900_792874_10frames__pt1__250722.jsonl”, output_file=“processed_42k_0726_tag03.jsonl”, stats_file=“collision_stats_0726_tag03.json”, threshold=0.1, debug=False # 设置为True可查看详细处理日志 ) 修改数据处理逻辑:按照scene_id分组,按照image_path按照/分割后取倒数第二个值分组,分组内按照timestamp降序排列,然后进行处理:默认分组内第一条数据的mask = "yes",后续的处理逻辑为取出前一帧数据的roadpoint的坐标值列表和当前数据的roadpoint值进行取交集和并集,允许数据值误差0.01,求交集/丙级>0.7的继承前一条数据的mask,否则对前一条数据的mask取否,当第一次出现no后,该分组的后续数据都标记为no。
08-01
Matlab基于粒子群优化算法及鲁棒MPPT控制器提高光伏并网的效率内容概要:本文围绕Matlab在电力系统优化与控制领域的应用展开,重点介绍了基于粒子群优化算法(PSO)和鲁棒MPPT控制器提升光伏并网效率的技术方案。通过Matlab代码实现,结合智能优化算法与先进控制策略,对光伏发电系统的最大功率点跟踪进行优化,有效提高了系统在不同光照条件下的能量转换效率和并网稳定性。同时,文档还涵盖了多种电力系统应用场景,如微电网调度、储能配置、鲁棒控制等,展示了Matlab在科研复现与工程仿真中的强大能力。; 适合人群:具备一定电力系统基础知识和Matlab编程能力的高校研究生、科研人员及从事新能源系统开发的工程师;尤其适合关注光伏并网技术、智能优化算法应用与MPPT控制策略研究的专业人士。; 使用场景及目标:①利用粒子群算法优化光伏系统MPPT控制器参数,提升动态响应速度与稳态精度;②研究鲁棒控制策略在光伏并网系统中的抗干扰能力;③复现已发表的高水平论文(如EI、SCI)中的仿真案例,支撑科研项目与学术写作。; 阅读建议:建议结合文中提供的Matlab代码与Simulink模型进行实践操作,重点关注算法实现细节与系统参数设置,同时参考链接中的完整资源下载以获取更多复现实例,加深对优化算法与控制系统设计的理解。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值