""" Full COCO Benchmark Parser COCO val2017 (5000 images), 80 classes """ import re from pathlib import Path from datetime import datetime def parse_coco_result(filepath): with open(filepath, 'r', encoding='utf-8', errors='replace') as f: content = f.read() result = {'file': filepath} # Check for errors if 'Traceback' in content: result['error'] = True return result result['error'] = False # Speed speed_match = re.search( r'Speed:\s*([\d.]+)ms\s+preprocess,\s*([\d.]+)ms\s+inference.*?([\d.]+)ms\s+postprocess', content, re.DOTALL ) if speed_match: result['preprocess_ms'] = float(speed_match.group(1)) result['inference_ms'] = float(speed_match.group(2)) result['postprocess_ms'] = float(speed_match.group(3)) result['total_ms'] = result['preprocess_ms'] + result['inference_ms'] + result['postprocess_ms'] result['fps'] = round(1000 / result['inference_ms'], 1) # Box metrics (from per-class metrics table) # Looking for the summary line: all, 5000, ... overall = re.search( r'^\s*all\s+(\d+)\s+(\d+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)', content, re.MULTILINE ) if overall: result['images'] = int(overall.group(1)) result['instances'] = int(overall.group(2)) result['P'] = float(overall.group(3)) result['R'] = float(overall.group(4)) result['mAP50'] = float(overall.group(5)) result['mAP50_95'] = float(overall.group(6)) # Key classes key_classes = { 'person': 0, 'bicycle': 1, 'car': 2, 'motorcycle': 3, 'truck': 7, 'bus': 5, 'traffic light': 9, 'stop sign': 11 } result['key_metrics'] = {} for name, idx in key_classes.items(): pattern = rf'^\s*{name}\s+\d+\s+\d+\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)' match = re.search(pattern, content, re.MULTILINE) if match: result['key_metrics'][name] = { 'P': float(match.group(1)), 'R': float(match.group(2)), 'AP50': float(match.group(3)), 'AP50_95': float(match.group(4)) } return result print("="*70) print("Full COCO Benchmark: FP16-480p vs INT8-640p") print(f"Dataset: COCO val2017 (5000 images), 80 classes") print(f"Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print("="*70) print() # Parse both results fp16 = parse_coco_result('fp16_480_full_results.txt') int8 = parse_coco_result('int8_640_full_results.txt') # 1. Overall Performance print("-"*70) print("1. Overall Performance (COCO val2017, 5000 images)") print("-"*70) print(f"{'Config':<15} {'Images':<8} {'Instances':<10} {'P':<8} {'R':<8} {'mAP50':<10} {'mAP50-95':<10}") print("-"*70) for name, data in [("FP16-480p", fp16), ("INT8-640p", int8)]: if data.get('error'): print(f"{name:<15} ERROR") continue img = data.get('images', 'N/A') inst = data.get('instances', 'N/A') p = data.get('P', 0) r = data.get('R', 0) map50 = data.get('mAP50', 0) map5095 = data.get('mAP50_95', 0) print(f"{name:<15} {img:<8} {inst:<10} {p:<8.3f} {r:<8.3f} {map50:<10.4f} {map5095:<10.4f}") print() # 2. Speed Analysis print("-"*70) print("2. Inference Speed Analysis") print("-"*70) print(f"{'Config':<15} {'Preprocess':<12} {'Inference':<12} {'Postprocess':<12} {'Total':<10} {'FPS':<10}") print("-"*70) for name, data in [("FP16-480p", fp16), ("INT8-640p", int8)]: if data.get('error'): print(f"{name:<15} ERROR") continue pre = data.get('preprocess_ms', 0) inf = data.get('inference_ms', 0) post = data.get('postprocess_ms', 0) total = data.get('total_ms', 0) fps = data.get('fps', 0) print(f"{name:<15} {pre:<12.2f} {inf:<12.2f} {post:<12.2f} {total:<10.2f} {fps:<10.1f}") print() # 3. Key Detection Classes print("-"*70) print("3. Key Detection Classes Performance (AP50-95)") print("-"*70) print(f"{'Class':<15} {'FP16-480p':<15} {'INT8-640p':<15} {'Diff':<10}") print("-"*70) key_names = ['person', 'car', 'motorcycle', 'bicycle', 'truck', 'bus', 'traffic light', 'stop sign'] for name in key_names: fp16_val = fp16.get('key_metrics', {}).get(name, {}).get('AP50_95', 0) int8_val = int8.get('key_metrics', {}).get(name, {}).get('AP50_95', 0) diff = (fp16_val - int8_val) / max(int8_val, 0.0001) * 100 if int8_val > 0 else 0 diff_str = f"+{diff:.1f}%" if diff > 0 else f"{diff:.1f}%" print(f"{name:<15} {fp16_val:<15.4f} {int8_val:<15.4f} {diff_str:<10}") print() # 4. Summary print("="*70) print("4. Summary & Recommendations") print("="*70) # Compare accuracy fp16_map = fp16.get('mAP50_95', 0) int8_map = int8.get('mAP50_95', 0) # Compare speed fp16_fps = fp16.get('fps', 0) int8_fps = int8.get('fps', 0) print() if fp16_map > int8_map and fp16_fps > int8_fps: print(" WINNER: FP16-480p (faster AND more accurate)") elif int8_map > fp16_map and int8_fps > fp16_fps: print(" WINNER: INT8-640p (faster AND more accurate)") else: print(f" Accuracy: FP16-480p {fp16_map:.4f} vs INT8-640p {int8_map:.4f}") print(f" Speed: FP16-480p {fp16_fps:.1f} FPS vs INT8-640p {int8_fps:.1f} FPS") print() if fp16_fps > int8_fps: print(f" → FP16-480p is {fp16_fps/int8_fps:.2f}x faster") if fp16_map > int8_map: print(f" → FP16-480p has +{(fp16_map-int8_map)/int8_map*100:.1f}% better mAP") print() print(" Recommendations:") print(" - For MAX ACCURACY: Use FP16-480p" if fp16_map > int8_map else " - For MAX ACCURACY: Use INT8-640p") print(" - For MAX SPEED: Use FP16-480p" if fp16_fps > int8_fps else " - For MAX SPEED: Use INT8-640p") print(" - For BEST BALANCE: FP16-480p" if fp16_fps/int8_fps > 0.9 else " - For BEST BALANCE: INT8-640p") print() print("="*70) print("Benchmark Complete!") print("="*70)