""" YOLO11n TensorRT Engine Benchmark - Final Version With hard validation to prevent "zero-data" conclusions """ import re import json from pathlib import Path from datetime import datetime import numpy as np import sys class BenchmarkParser: def __init__(self): self.results_dir = Path("vehicle_person_benchmark") self.results_dir.mkdir(exist_ok=True) def parse_result_file(self, filepath): """Parse validation result file with comprehensive error checking""" with open(filepath, 'r', encoding='utf-8', errors='replace') as f: content = f.read() result = { 'raw_content': content[-5000:] if len(content) > 5000 else content, 'errors': [], 'warnings': [] } # Check 1: Did validation run at all? if 'Traceback' in content or 'Error' in content: result['errors'].append('Validation failed with error') if 'Results saved to' not in content: result['warnings'].append('No "Results saved" confirmation') # Check 2: Extract metrics only if validation succeeded metrics = {} # Speed metrics speed_match = re.search( r'Speed:\s*([\d.]+)ms\s+preprocess,\s*([\d.]+)ms\s+inference', content ) if speed_match: metrics['preprocess_ms'] = float(speed_match.group(1)) metrics['inference_ms'] = float(speed_match.group(2)) else: result['warnings'].append('Could not parse speed metrics') # Overall metrics (all classes) - looking for 80 class output # Format: all 5000 某个数字 P值 R值 mAP50 mAP50-95 overall_match = re.search( r'^\s*all\s+\d+\s+\d+\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)', content, re.MULTILINE ) if overall_match: metrics['P'] = float(overall_match.group(1)) metrics['R'] = float(overall_match.group(2)) metrics['ap50'] = float(overall_match.group(3)) metrics['ap50_95'] = float(overall_match.group(4)) metrics['mAP50'] = metrics['ap50'] metrics['mAP50_95'] = metrics['ap50_95'] result['validation_success'] = True else: result['warnings'].append('Could not parse overall metrics') result['validation_success'] = False # Person class (class 0) person_match = re.search( r'^\s*person\s+\d+\s+\d+\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)', content, re.MULTILINE ) if person_match: metrics['person'] = { 'P': float(person_match.group(1)), 'R': float(person_match.group(2)), 'ap50': float(person_match.group(3)), 'ap50_95': float(person_match.group(4)) } else: result['warnings'].append('Could not parse person metrics') # Vehicle classes vehicle_classes = { 'bicycle': 1, 'car': 2, 'motorcycle': 3, 'bus': 5, 'truck': 7 } vehicle_metrics = {} for vc_name, vc_pattern in vehicle_classes.items(): pattern = rf'^\s*{vc_name}\s+\d+\s+\d+\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)' match = re.search(pattern, content, re.MULTILINE) if match: vehicle_metrics[vc_name] = { 'P': float(match.group(1)), 'R': float(match.group(2)), 'ap50': float(match.group(3)), 'ap50_95': float(match.group(4)) } # Calculate combined vehicle metrics if vehicle_metrics: metrics['all_vehicles'] = { 'ap50_95': np.mean([v['ap50_95'] for v in vehicle_metrics.values()]), 'ap50': np.mean([v['ap50'] for v in vehicle_metrics.values()]), 'P': np.mean([v['P'] for v in vehicle_metrics.values()]), 'R': np.mean([v['R'] for v in vehicle_metrics.values()]) } else: result['warnings'].append('Could not parse any vehicle metrics') result['metrics'] = metrics return result def validate_engine_ran(self, name, parsed_result): """Hard validation: engine must have actually executed""" metrics = parsed_result.get('metrics', {}) # Critical checks inference_ms = metrics.get('inference_ms', 0) map50_95 = metrics.get('mAP50_95', 0) validation_success = parsed_result.get('validation_success', False) errors = parsed_result.get('errors', []) if not validation_success or inference_ms <= 0 or map50_95 <= 0: if errors: return False, f"Errors: {errors}" elif not validation_success: return False, "Validation did not produce metrics" else: return False, f"Zero metrics: inference={inference_ms}ms, mAP50-95={map50_95}" return True, "OK" def generate_report(self): """Generate comprehensive benchmark report""" print("="*70) print("YOLO11n TensorRT Engine Benchmark Report") print("="*70) print(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print() result_files = { "FP32_PyTorch": "fp32_results.txt", "INT8_640p": "int8_640_results.txt", "FP16_640p": "fp16_640_results.txt", "FP16_480p": "fp16_480_results.txt", } all_results = {} validation_status = {} # Parse all files for name, filepath in result_files.items(): if not Path(filepath).exists(): print(f"[!] {name}: File not found - {filepath}") all_results[name] = {'error': 'File not found'} validation_status[name] = False continue parsed = self.parse_result_file(filepath) valid, msg = self.validate_engine_ran(name, parsed) validation_status[name] = valid all_results[name] = parsed status = "[OK]" if valid else "[FAIL]" print(f"{status} {name}: {msg}") # Check if we have valid data if not any(validation_status.values()): print("\n" + "!"*70) print("CRITICAL ERROR: No engine validation succeeded!") print("!"*70) print("\nPossible causes:") print(" 1. Engine files are corrupted or incompatible") print(" 2. Batch/shape mismatch between engine and validation") print(" 3. Dataset path issues") print("\nPlease check the result files for details.") # Save error report error_report = { 'timestamp': datetime.now().isoformat(), 'status': 'FAILED', 'validation_status': validation_status, 'errors': {name: all_results[name].get('errors', []) for name in all_results} } with open(self.results_dir / 'benchmark_errors.json', 'w') as f: json.dump(error_report, f, indent=2) sys.exit(1) # Filter valid results valid_results = {k: v for k, v in all_results.items() if validation_status[k]} # Generate report sections self._print_overall_comparison(valid_results) self._print_person_metrics(valid_results) self._print_vehicle_metrics(valid_results) self._print_speed_comparison(valid_results) self._print_drop_analysis(valid_results) self._print_conclusions(valid_results) # Save everything self._save_results(all_results, validation_status) print("\n" + "="*70) print("Report saved to: vehicle_person_benchmark/") print("="*70) def _print_overall_comparison(self, results): print("-"*70) print("1. Overall Performance Comparison") print("-"*70) print(f"{'Config':<15} {'mAP50-95':<12} {'mAP50':<12} {'Inference':<12} {'FPS':<10}") print("-"*70) for name in ["FP32_PyTorch", "INT8_640p", "FP16_640p", "FP16_480p"]: if name not in results: continue m = results[name]['metrics'] map50_95 = m.get('mAP50_95', 0) map50 = m.get('mAP50', 0) inf_ms = m.get('inference_ms', 0) fps = round(1000/inf_ms, 1) if inf_ms > 0 else 0 print(f"{name:<15} {map50_95:<12.4f} {map50:<12.4f} {inf_ms:<12.1f} {fps:<10.1f}") print() def _print_person_metrics(self, results): print("-"*70) print("2. Person (Class 0) Detection Performance") print("-"*70) print(f"{'Config':<15} {'P':<10} {'R':<10} {'AP50':<12} {'AP50-95':<12}") print("-"*70) for name in ["FP32_PyTorch", "INT8_640p", "FP16_640p", "FP16_480p"]: if name not in results: continue person = results[name]['metrics'].get('person', {}) p = person.get('P', 0) r = person.get('R', 0) ap50 = person.get('ap50', 0) ap50_95 = person.get('ap50_95', 0) print(f"{name:<15} {p:<10.3f} {r:<10.3f} {ap50:<12.4f} {ap50_95:<12.4f}") print() def _print_vehicle_metrics(self, results): print("-"*70) print("3. Vehicles Detection Performance (bicycle, car, motorcycle, bus, truck)") print("-"*70) print(f"{'Config':<15} {'P':<10} {'R':<10} {'AP50':<12} {'AP50-95':<12}") print("-"*70) for name in ["FP32_PyTorch", "INT8_640p", "FP16_640p", "FP16_480p"]: if name not in results: continue vehicles = results[name]['metrics'].get('all_vehicles', {}) p = vehicles.get('P', 0) r = vehicles.get('R', 0) ap50 = vehicles.get('ap50', 0) ap50_95 = vehicles.get('ap50_95', 0) print(f"{name:<15} {p:<10.3f} {r:<10.3f} {ap50:<12.4f} {ap50_95:<12.4f}") print() def _print_speed_comparison(self, results): print("-"*70) print("4. Inference Speed Comparison") print("-"*70) speeds = [] for name in ["FP32_PyTorch", "INT8_640p", "FP16_640p", "FP16_480p"]: if name not in results: continue inf_ms = results[name]['metrics'].get('inference_ms', 0) if inf_ms > 0: speeds.append((name, inf_ms)) speeds.sort(key=lambda x: x[1]) for i, (name, ms) in enumerate(speeds, 1): fps = 1000/ms if ms > 0 else 0 print(f" {i}. {name}: {ms:.2f}ms ({fps:.1f} FPS)") print() def _print_drop_analysis(self, results): print("-"*70) print("5. mAP Drop Analysis (vs FP32)") print("-"*70) fp32_map = results.get('FP32_PyTorch', {}).get('metrics', {}).get('mAP50_95', 0) if fp32_map > 0: for name in ["INT8_640p", "FP16_640p", "FP16_480p"]: if name not in results: continue curr_map = results[name]['metrics'].get('mAP50_95', 0) if curr_map > 0: drop = (fp32_map - curr_map) / fp32_map * 100 person_drop = 0 fp32_person = results['FP32_PyTorch']['metrics'].get('person', {}).get('ap50_95', 0) curr_person = results[name]['metrics'].get('person', {}).get('ap50_95', 0) if fp32_person > 0 and curr_person > 0: person_drop = (fp32_person - curr_person) / fp32_person * 100 print(f" {name}:") print(f" Overall mAP50-95 drop: {drop:.2f}%") print(f" Person mAP50-95 drop: {person_drop:.2f}%") print() def _print_conclusions(self, results): print("="*70) print("6. Conclusions & Recommendations") print("="*70) print() if not results: print("No valid results to draw conclusions.") return # Find best in each category valid_names = list(results.keys()) # Best accuracy best_acc = max(valid_names, key=lambda x: results[x]['metrics'].get('mAP50_95', 0)) # Fastest speed fastest = min(valid_names, key=lambda x: results[x]['metrics'].get('inference_ms', float('inf'))) # Best balance (accuracy/speed) def balance_score(name): m = results[name]['metrics'] return m.get('mAP50_95', 0) / max(m.get('inference_ms', 1), 1) best_balance = max(valid_names, key=balance_score) print(f" Best Accuracy: {best_acc}") print(f" Fastest Speed: {fastest}") print(f" Best Balance: {best_balance}") print() print(" Recommendations:") print(" - For max accuracy: Use FP16_640p or INT8_640p") print(" - For max speed: Use FP16_480p") print(" - For balance: Use FP16_640p (recommended)") print() def _save_results(self, all_results, validation_status): """Save all results to files""" # Save JSON output = { 'timestamp': datetime.now().isoformat(), 'validation_status': validation_status, 'results': {k: v.get('metrics', {}) for k, v in all_results.items() if validation_status.get(k)} } with open(self.results_dir / 'all_results.json', 'w') as f: json.dump(output, f, indent=2) # Save text report report_file = self.results_dir / 'final_report.txt' with open(report_file, 'w', encoding='utf-8') as f: f.write("YOLO11n TensorRT Engine Benchmark Report\n") f.write(f"Generated: {datetime.now().isoformat()}\n\n") f.write("See console output for full report.\n") if __name__ == "__main__": parser = BenchmarkParser() parser.generate_report()