Test_AI/parse_full_coco.py

"""
Full COCO Benchmark Parser
COCO val2017 (5000 images), 80 classes
"""

import re
from pathlib import Path
from datetime import datetime

def parse_coco_result(filepath):
    with open(filepath, 'r', encoding='utf-8', errors='replace') as f:
        content = f.read()
    
    result = {'file': filepath}
    
    # Check for errors
    if 'Traceback' in content:
        result['error'] = True
        return result
    
    result['error'] = False
    
    # Speed
    speed_match = re.search(
        r'Speed:\s*([\d.]+)ms\s+preprocess,\s*([\d.]+)ms\s+inference.*?([\d.]+)ms\s+postprocess',
        content, re.DOTALL
    )
    if speed_match:
        result['preprocess_ms'] = float(speed_match.group(1))
        result['inference_ms'] = float(speed_match.group(2))
        result['postprocess_ms'] = float(speed_match.group(3))
        result['total_ms'] = result['preprocess_ms'] + result['inference_ms'] + result['postprocess_ms']
        result['fps'] = round(1000 / result['inference_ms'], 1)
    
    # Box metrics (from per-class metrics table)
    # Looking for the summary line: all, 5000, ...
    overall = re.search(
        r'^\s*all\s+(\d+)\s+(\d+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)',
        content, re.MULTILINE
    )
    if overall:
        result['images'] = int(overall.group(1))
        result['instances'] = int(overall.group(2))
        result['P'] = float(overall.group(3))
        result['R'] = float(overall.group(4))
        result['mAP50'] = float(overall.group(5))
        result['mAP50_95'] = float(overall.group(6))
    
    # Key classes
    key_classes = {
        'person': 0, 'bicycle': 1, 'car': 2, 'motorcycle': 3,
        'truck': 7, 'bus': 5, 'traffic light': 9, 'stop sign': 11
    }
    
    result['key_metrics'] = {}
    for name, idx in key_classes.items():
        pattern = rf'^\s*{name}\s+\d+\s+\d+\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)'
        match = re.search(pattern, content, re.MULTILINE)
        if match:
            result['key_metrics'][name] = {
                'P': float(match.group(1)),
                'R': float(match.group(2)),
                'AP50': float(match.group(3)),
                'AP50_95': float(match.group(4))
            }
    
    return result

print("="*70)
print("Full COCO Benchmark: FP16-480p vs INT8-640p")
print(f"Dataset: COCO val2017 (5000 images), 80 classes")
print(f"Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
print("="*70)
print()

# Parse both results
fp16 = parse_coco_result('fp16_480_full_results.txt')
int8 = parse_coco_result('int8_640_full_results.txt')

# 1. Overall Performance
print("-"*70)
print("1. Overall Performance (COCO val2017, 5000 images)")
print("-"*70)
print(f"{'Config':<15} {'Images':<8} {'Instances':<10} {'P':<8} {'R':<8} {'mAP50':<10} {'mAP50-95':<10}")
print("-"*70)

for name, data in [("FP16-480p", fp16), ("INT8-640p", int8)]:
    if data.get('error'):
        print(f"{name:<15} ERROR")
        continue
    img = data.get('images', 'N/A')
    inst = data.get('instances', 'N/A')
    p = data.get('P', 0)
    r = data.get('R', 0)
    map50 = data.get('mAP50', 0)
    map5095 = data.get('mAP50_95', 0)
    print(f"{name:<15} {img:<8} {inst:<10} {p:<8.3f} {r:<8.3f} {map50:<10.4f} {map5095:<10.4f}")

print()

# 2. Speed Analysis
print("-"*70)
print("2. Inference Speed Analysis")
print("-"*70)
print(f"{'Config':<15} {'Preprocess':<12} {'Inference':<12} {'Postprocess':<12} {'Total':<10} {'FPS':<10}")
print("-"*70)

for name, data in [("FP16-480p", fp16), ("INT8-640p", int8)]:
    if data.get('error'):
        print(f"{name:<15} ERROR")
        continue
    pre = data.get('preprocess_ms', 0)
    inf = data.get('inference_ms', 0)
    post = data.get('postprocess_ms', 0)
    total = data.get('total_ms', 0)
    fps = data.get('fps', 0)
    print(f"{name:<15} {pre:<12.2f} {inf:<12.2f} {post:<12.2f} {total:<10.2f} {fps:<10.1f}")

print()

# 3. Key Detection Classes
print("-"*70)
print("3. Key Detection Classes Performance (AP50-95)")
print("-"*70)
print(f"{'Class':<15} {'FP16-480p':<15} {'INT8-640p':<15} {'Diff':<10}")
print("-"*70)

key_names = ['person', 'car', 'motorcycle', 'bicycle', 'truck', 'bus', 'traffic light', 'stop sign']
for name in key_names:
    fp16_val = fp16.get('key_metrics', {}).get(name, {}).get('AP50_95', 0)
    int8_val = int8.get('key_metrics', {}).get(name, {}).get('AP50_95', 0)
    diff = (fp16_val - int8_val) / max(int8_val, 0.0001) * 100 if int8_val > 0 else 0
    diff_str = f"+{diff:.1f}%" if diff > 0 else f"{diff:.1f}%"
    print(f"{name:<15} {fp16_val:<15.4f} {int8_val:<15.4f} {diff_str:<10}")

print()

# 4. Summary
print("="*70)
print("4. Summary & Recommendations")
print("="*70)

# Compare accuracy
fp16_map = fp16.get('mAP50_95', 0)
int8_map = int8.get('mAP50_95', 0)

# Compare speed
fp16_fps = fp16.get('fps', 0)
int8_fps = int8.get('fps', 0)

print()
if fp16_map > int8_map and fp16_fps > int8_fps:
    print("  WINNER: FP16-480p (faster AND more accurate)")
elif int8_map > fp16_map and int8_fps > fp16_fps:
    print("  WINNER: INT8-640p (faster AND more accurate)")
else:
    print(f"  Accuracy: FP16-480p {fp16_map:.4f} vs INT8-640p {int8_map:.4f}")
    print(f"  Speed:    FP16-480p {fp16_fps:.1f} FPS vs INT8-640p {int8_fps:.1f} FPS")
    print()
    if fp16_fps > int8_fps:
        print(f"  → FP16-480p is {fp16_fps/int8_fps:.2f}x faster")
    if fp16_map > int8_map:
        print(f"  → FP16-480p has +{(fp16_map-int8_map)/int8_map*100:.1f}% better mAP")

print()
print("  Recommendations:")
print("    - For MAX ACCURACY: Use FP16-480p" if fp16_map > int8_map else "    - For MAX ACCURACY: Use INT8-640p")
print("    - For MAX SPEED:    Use FP16-480p" if fp16_fps > int8_fps else "    - For MAX SPEED:    Use INT8-640p")
print("    - For BEST BALANCE: FP16-480p" if fp16_fps/int8_fps > 0.9 else "    - For BEST BALANCE: INT8-640p")

print()
print("="*70)
print("Benchmark Complete!")
print("="*70)