From 682063abf11642fde0317465bae26479d35ceb0a Mon Sep 17 00:00:00 2001 From: 16337 <1633794139@qq.com> Date: Mon, 16 Mar 2026 17:38:33 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=B9=E7=94=A8=204-bit=20NF4=20?= =?UTF-8?q?=E7=BA=AF=20GPU=20=E6=8E=A8=E7=90=86=EF=BC=8C=E5=85=B3=E9=97=AD?= =?UTF-8?q?=20thinking=20=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 模型加载改为 bitsandbytes 4-bit NF4 量化，device_map={"":0} 纯 GPU - 关闭 Qwen3.5 thinking 模式 (enable_thinking=False) - 精度从 60% 提升到 90%，推理速度 1-2 tokens/s - GPU 显存 7.13GB/8GB，输出质量正常 - 更新所有测试结果和综合报告 Co-Authored-By: Claude Opus 4.6 --- vsp/qwen3.5-9b/benchmark_speed.py | 16 +-- vsp/qwen3.5-9b/generate_report.py | 32 ++--- vsp/qwen3.5-9b/gpu_requirements.py | 19 +-- vsp/qwen3.5-9b/model_utils.py | 29 ++-- vsp/qwen3.5-9b/results/REPORT.md | 63 +++++--- vsp/qwen3.5-9b/results/accuracy_results.json | 134 ++++++++++++++++++ vsp/qwen3.5-9b/results/benchmark_speed.json | 60 ++++++++ .../results/concurrency_results.json | 40 ++++++ vsp/qwen3.5-9b/results/gpu_requirements.json | 19 +-- vsp/qwen3.5-9b/test_accuracy.py | 4 +- vsp/qwen3.5-9b/test_basic_inference.py | 40 ++---- vsp/qwen3.5-9b/test_concurrency.py | 4 +- 12 files changed, 356 insertions(+), 104 deletions(-) create mode 100644 vsp/qwen3.5-9b/results/accuracy_results.json create mode 100644 vsp/qwen3.5-9b/results/benchmark_speed.json create mode 100644 vsp/qwen3.5-9b/results/concurrency_results.json diff --git a/vsp/qwen3.5-9b/benchmark_speed.py b/vsp/qwen3.5-9b/benchmark_speed.py index 225d299..0067445 100644 --- a/vsp/qwen3.5-9b/benchmark_speed.py +++ b/vsp/qwen3.5-9b/benchmark_speed.py @@ -10,21 +10,21 @@ from transformers import AutoModelForCausalLM, AutoTokenizer from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from model_utils import load_model +from model_utils import load_model, apply_chat -def benchmark_speed(model, tokenizer, num_runs=5): +def benchmark_speed(model, tokenizer, num_runs=2): """测试不同输入长度和输出长度下的推理速度""" print("=" * 60) print("性能基准测试 - 推理速度") print("=" * 60) test_cases = [ - {"name": "短输入短输出", "prompt": "你好", "max_tokens": 50}, - {"name": "短输入中输出", "prompt": "介绍一下人工智能", "max_tokens": 128}, - {"name": "短输入长输出", "prompt": "请详细解释深度学习的原理和应用", "max_tokens": 256}, - {"name": "中输入中输出", "prompt": "以下是一段代码：\n```python\ndef fibonacci(n):\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)\n```\n请分析这段代码的时间复杂度并给出优化方案。", "max_tokens": 256}, - {"name": "长输入短输出", "prompt": "请总结以下内容的关键点：" + "人工智能是计算机科学的一个分支。" * 50, "max_tokens": 64}, + {"name": "短输入短输出", "prompt": "你好", "max_tokens": 32}, + {"name": "短输入中输出", "prompt": "介绍一下人工智能", "max_tokens": 64}, + {"name": "短输入长输出", "prompt": "请详细解释深度学习的原理和应用", "max_tokens": 128}, + {"name": "中输入中输出", "prompt": "以下是一段代码：\n```python\ndef fibonacci(n):\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)\n```\n请分析这段代码的时间复杂度并给出优化方案。", "max_tokens": 128}, + {"name": "长输入短输出", "prompt": "请总结以下内容的关键点：" + "人工智能是计算机科学的一个分支。" * 50, "max_tokens": 32}, ] results = [] @@ -35,7 +35,7 @@ def benchmark_speed(model, tokenizer, num_runs=5): for run in range(num_runs): messages = [{"role": "user", "content": case["prompt"]}] - text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + text = apply_chat(tokenizer, messages) inputs = tokenizer(text, return_tensors="pt").to(model.device) input_len = inputs["input_ids"].shape[1] diff --git a/vsp/qwen3.5-9b/generate_report.py b/vsp/qwen3.5-9b/generate_report.py index 0a444c5..cc97f61 100644 --- a/vsp/qwen3.5-9b/generate_report.py +++ b/vsp/qwen3.5-9b/generate_report.py @@ -29,7 +29,7 @@ def generate_report(): "| 项目 | 值 |", "|------|-----|", "| 模型 | Qwen3.5-9B |", - "| 加载方式 | FP16 + CPU offload (accelerate) |", + "| 加载方式 | 4-bit NF4 量化 (bitsandbytes)，纯 GPU |", "| GPU | NVIDIA GeForce RTX 3050 OEM |", "| GPU 显存 | 8 GB |", "| CUDA | 12.1 |", @@ -39,8 +39,6 @@ def generate_report(): if speed and "memory" in speed: mem = speed["memory"] report_lines.extend([ - f"| GPU | {mem.get('gpu_name', 'N/A')} |", - f"| GPU 显存 | {mem.get('gpu_total_gb', 'N/A')} GB |", f"| 模型显存占用 | {mem.get('gpu_allocated_gb', 'N/A')} GB |", f"| 系统内存占用 | {mem.get('ram_used_gb', 'N/A')} GB |", ]) @@ -106,27 +104,29 @@ def generate_report(): report_lines.extend([ "\n## 6. 实际测试结论", "", - "### RTX 3050 8GB 测试结果", + "### RTX 3050 8GB 测试结果 (4-bit NF4 量化，纯 GPU)", "", "| 指标 | 结果 |", "|------|------|", - "| GPU 显存占用 | 3.91 GB |", - "| 系统内存占用 | 13.60 GB |", - "| 推理速度 | ~0.4 tokens/s |", - "| 输出质量 | 极差（乱码/重复） |", + "| GPU 显存占用 | 7.13 GB / 8 GB |", + "| 系统内存占用 | 7.59 GB |", + "| 推理速度 | 1.0-1.8 tokens/s |", + "| 精度 | 90% (9/10) |", + "| 输出质量 | 正常，回答准确 |", "", - "### 问题分析", + "### 注意事项", "", - "1. **bitsandbytes 4-bit 量化不可用**: 不支持 CPU offload，8GB 显存无法装下完整 4-bit 模型（~5GB 模型 + KV cache + 激活值）", - "2. **bitsandbytes INT8 不可用**: 与 accelerate 新版本存在兼容性问题（Windows）", - "3. **FP16 + CPU offload**: 可以加载模型，但大量层 offload 到 CPU 导致推理极慢（0.4 tokens/s），输出质量不可用", + "1. **必须使用 4-bit NF4 量化**: device_map={\"\":0} 将模型全部放在 GPU 上", + "2. **必须关闭 thinking 模式**: enable_thinking=False，否则输出中包含思考过程且容易被截断", + "3. **显存接近上限**: 7.13GB / 8GB，长文本输入可能导致 OOM", + "4. **并发不可行**: 单 GPU 串行推理，吞吐量恒定 ~2 tokens/s", "", - "### 建议", + "### 部署建议", "", - "1. **RTX 3050 8GB 无法有效运行 Qwen3.5-9B**，显存严重不足", - "2. **最低推荐**: RTX 3060 12GB (INT8) 或 RTX 4060 Ti 16GB (INT8/FP16)", + "1. **RTX 3050 8GB 可用于开发测试**，4-bit 量化后勉强可用", + "2. **推荐**: RTX 3060 12GB (INT8) 或 RTX 4060 Ti 16GB，有更大显存余量", "3. **生产部署**: RTX 4090 24GB (FP16) 或 A100 40GB/80GB + vLLM", - "4. **替代方案**: 使用更小的模型如 Qwen2.5-3B 或 Qwen2.5-7B 在 8GB 显卡上运行", + "4. **替代方案**: 使用更小的模型如 Qwen2.5-3B 或 Qwen2.5-7B 在 8GB 显卡上更流畅运行", ]) # 保存报告 diff --git a/vsp/qwen3.5-9b/gpu_requirements.py b/vsp/qwen3.5-9b/gpu_requirements.py index 825d678..3d78eaa 100644 --- a/vsp/qwen3.5-9b/gpu_requirements.py +++ b/vsp/qwen3.5-9b/gpu_requirements.py @@ -35,16 +35,17 @@ GPU_REQUIREMENTS = { }, "actual_test_results": { "gpu": "NVIDIA GeForce RTX 3050 OEM 8GB", - "method": "FP16 + CPU offload (accelerate device_map=auto)", - "gpu_vram_used_gb": 3.91, - "ram_used_gb": 13.60, - "inference_speed_tokens_per_sec": 0.4, - "output_quality": "极差（乱码/重复输出）", - "conclusion": "RTX 3050 8GB 无法有效运行 Qwen3.5-9B，显存不足导致大量层 offload 到 CPU，推理极慢且输出质量不可用", + "method": "4-bit NF4 量化 (bitsandbytes)，纯 GPU 运行，关闭 thinking 模式", + "gpu_vram_used_gb": 7.13, + "ram_used_gb": 7.59, + "inference_speed_tokens_per_sec": "1.0-1.8", + "accuracy": "90% (9/10)", + "output_quality": "正常，回答准确", + "conclusion": "RTX 3050 8GB 可以运行 Qwen3.5-9B 4-bit 量化版本，显存占用 7.13GB，推理速度 1-2 tokens/s，适合开发测试", "issues": [ - "bitsandbytes 4-bit 量化不支持 CPU offload，8GB 显存装不下完整 4-bit 模型", - "bitsandbytes INT8 与 accelerate 版本不兼容（Windows）", - "FP16 + CPU offload 虽可加载但速度仅 0.4 tokens/s，输出为乱码", + "显存占用 7.13GB，接近 8GB 上限，长文本可能 OOM", + "推理速度较慢（1-2 tokens/s），不适合生产环境", + "需关闭 thinking 模式才能正常输出", ], }, "deployment_recommendations": { diff --git a/vsp/qwen3.5-9b/model_utils.py b/vsp/qwen3.5-9b/model_utils.py index f843acd..f11ea13 100644 --- a/vsp/qwen3.5-9b/model_utils.py +++ b/vsp/qwen3.5-9b/model_utils.py @@ -3,7 +3,7 @@ import os import sys import glob import torch -from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig # 修复 Windows GBK 编码问题 sys.stdout.reconfigure(encoding='utf-8', errors='replace') @@ -19,23 +19,36 @@ def get_model_path(): def load_model(): - """加载模型 (FP16 + GPU/CPU offload) + """加载模型 (4-bit NF4 量化，纯 GPU 运行) - RTX 3050 8GB VRAM 不够放完整模型，使用 FP16 并将部分层 offload 到 CPU。 + 使用 bitsandbytes 4-bit 量化，模型约 5GB，全部放在 GPU 上。 + RTX 3050 8GB 显存刚好够用。 """ model_path = get_model_path() print(f"模型路径: {model_path}") tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) - max_memory = {0: "6GiB", "cpu": "24GiB"} + quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=True, + ) + model = AutoModelForCausalLM.from_pretrained( model_path, - torch_dtype=torch.float16, - device_map="auto", - max_memory=max_memory, - offload_folder="vsp/qwen3.5-9b/offload", + quantization_config=quantization_config, + device_map={"": 0}, trust_remote_code=True, ) return model, tokenizer + + +def apply_chat(tokenizer, messages): + """应用聊天模板，关闭 thinking 模式""" + return tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, + enable_thinking=False, + ) diff --git a/vsp/qwen3.5-9b/results/REPORT.md b/vsp/qwen3.5-9b/results/REPORT.md index 224581a..b310657 100644 --- a/vsp/qwen3.5-9b/results/REPORT.md +++ b/vsp/qwen3.5-9b/results/REPORT.md @@ -1,29 +1,54 @@ # Qwen3.5-9B 性能测试报告 -生成时间: 2026-03-16 13:09:10 +生成时间: 2026-03-16 17:37:52 ## 1. 测试环境 | 项目 | 值 | |------|-----| | 模型 | Qwen3.5-9B | -| 加载方式 | FP16 + CPU offload (accelerate) | +| 加载方式 | 4-bit NF4 量化 (bitsandbytes)，纯 GPU | | GPU | NVIDIA GeForce RTX 3050 OEM | | GPU 显存 | 8 GB | | CUDA | 12.1 | | Python 环境 | conda yolo | +| 模型显存占用 | 7.13 GB | +| 系统内存占用 | 7.59 GB | ## 2. 推理速度 -*未运行速度测试* +| 测试场景 | 输入tokens | 输出tokens | 耗时(s) | 速度(tokens/s) | +|---------|-----------|-----------|---------|---------------| +| 短输入短输出 | 13 | 12.0 | 12.503 | 1.0 | +| 短输入中输出 | 14 | 64.0 | 38.312 | 1.7 | +| 短输入长输出 | 19 | 128.0 | 69.541 | 1.8 | +| 中输入中输出 | 64 | 128.0 | 78.318 | 1.6 | +| 长输入短输出 | 318 | 32.0 | 32.659 | 1.0 | ## 3. 精度评估 -*未运行精度测试* +**总准确率: 90.0% (9/10)** + +| 分类 | 通过/总数 | 准确率 | +|------|---------|--------| +| 知识问答 | 2/3 | 67% | +| 数学推理 | 2/2 | 100% | +| 逻辑推理 | 1/1 | 100% | +| 代码理解 | 1/1 | 100% | +| 翻译 | 1/1 | 100% | +| 摘要 | 1/1 | 100% | +| 情感分类 | 1/1 | 100% | ## 4. 并发性能 -*未运行并发测试* +| 并发数 | 总耗时(s) | 吞吐量(tokens/s) | 平均延迟(s) | +|-------|---------|----------------|-----------| +| 1 | 33.29 | 1.9 | 33.18 | +| 2 | 65.01 | 2.0 | 49.14 | +| 4 | 128.55 | 2.0 | 80.09 | +| 8 | 275.44 | 1.9 | 148.94 | + +> 注: 单GPU串行推理，并发测试主要体现请求排队效果 ## 5. GPU 算力需求 @@ -36,24 +61,26 @@ ## 6. 实际测试结论 -### RTX 3050 8GB 测试结果 +### RTX 3050 8GB 测试结果 (4-bit NF4 量化，纯 GPU) | 指标 | 结果 | |------|------| -| GPU 显存占用 | 3.91 GB | -| 系统内存占用 | 13.60 GB | -| 推理速度 | ~0.4 tokens/s | -| 输出质量 | 极差（乱码/重复） | +| GPU 显存占用 | 7.13 GB / 8 GB | +| 系统内存占用 | 7.59 GB | +| 推理速度 | 1.0-1.8 tokens/s | +| 精度 | 90% (9/10) | +| 输出质量 | 正常，回答准确 | -### 问题分析 +### 注意事项 -1. **bitsandbytes 4-bit 量化不可用**: 不支持 CPU offload，8GB 显存无法装下完整 4-bit 模型（~5GB 模型 + KV cache + 激活值） -2. **bitsandbytes INT8 不可用**: 与 accelerate 新版本存在兼容性问题（Windows） -3. **FP16 + CPU offload**: 可以加载模型，但大量层 offload 到 CPU 导致推理极慢（0.4 tokens/s），输出质量不可用 +1. **必须使用 4-bit NF4 量化**: device_map={"":0} 将模型全部放在 GPU 上 +2. **必须关闭 thinking 模式**: enable_thinking=False，否则输出中包含思考过程且容易被截断 +3. **显存接近上限**: 7.13GB / 8GB，长文本输入可能导致 OOM +4. **并发不可行**: 单 GPU 串行推理，吞吐量恒定 ~2 tokens/s -### 建议 +### 部署建议 -1. **RTX 3050 8GB 无法有效运行 Qwen3.5-9B**，显存严重不足 -2. **最低推荐**: RTX 3060 12GB (INT8) 或 RTX 4060 Ti 16GB (INT8/FP16) +1. **RTX 3050 8GB 可用于开发测试**，4-bit 量化后勉强可用 +2. **推荐**: RTX 3060 12GB (INT8) 或 RTX 4060 Ti 16GB，有更大显存余量 3. **生产部署**: RTX 4090 24GB (FP16) 或 A100 40GB/80GB + vLLM -4. **替代方案**: 使用更小的模型如 Qwen2.5-3B 或 Qwen2.5-7B 在 8GB 显卡上运行 \ No newline at end of file +4. **替代方案**: 使用更小的模型如 Qwen2.5-3B 或 Qwen2.5-7B 在 8GB 显卡上更流畅运行 \ No newline at end of file diff --git a/vsp/qwen3.5-9b/results/accuracy_results.json b/vsp/qwen3.5-9b/results/accuracy_results.json new file mode 100644 index 0000000..3cd9eb8 --- /dev/null +++ b/vsp/qwen3.5-9b/results/accuracy_results.json @@ -0,0 +1,134 @@ +{ + "timestamp": "2026-03-16T16:57:12.572999", + "model": "Qwen3.5-9B", + "quantization": "4-bit NF4", + "accuracy": { + "total": 10, + "passed": 9, + "accuracy": 90.0, + "category_stats": { + "知识问答": { + "total": 3, + "passed": 2 + }, + "数学推理": { + "total": 2, + "passed": 2 + }, + "逻辑推理": { + "total": 1, + "passed": 1 + }, + "代码理解": { + "total": 1, + "passed": 1 + }, + "翻译": { + "total": 1, + "passed": 1 + }, + "摘要": { + "total": 1, + "passed": 1 + }, + "情感分类": { + "total": 1, + "passed": 1 + } + }, + "details": [ + { + "category": "知识问答", + "prompt": "中国的首都是哪个城市？请只回答城市名。", + "response": "北京", + "expected": [ + "北京" + ], + "passed": true + }, + { + "category": "知识问答", + "prompt": "水的化学式是什么？请只回答化学式。", + "response": "H₂O", + "expected": [ + "H2O" + ], + "passed": false + }, + { + "category": "知识问答", + "prompt": "地球到太阳的平均距离大约是多少公里？A. 1.5亿 B. 3亿 C. 5亿 D. 1亿。请只回答选项字母。", + "response": "A", + "expected": [ + "A" + ], + "passed": true + }, + { + "category": "数学推理", + "prompt": "计算 15 * 23 = ? 请只回答数字。", + "response": "345", + "expected": [ + "345" + ], + "passed": true + }, + { + "category": "数学推理", + "prompt": "一个三角形三边分别是3、4、5，它是什么三角形？请只回答类型。", + "response": "直角三角形", + "expected": [ + "直角" + ], + "passed": true + }, + { + "category": "逻辑推理", + "prompt": "所有的狗都是动物。小白是一只狗。所以小白是什么？请只回答一个词。", + "response": "动物", + "expected": [ + "动物" + ], + "passed": true + }, + { + "category": "代码理解", + "prompt": "以下Python代码的输出是什么？\n```python\nprint(len([1, 2, 3, 4, 5]))\n```\n请只回答数字。", + "response": "5", + "expected": [ + "5" + ], + "passed": true + }, + { + "category": "翻译", + "prompt": "将'Hello World'翻译成中文，请只回答翻译结果。", + "response": "你好，世界", + "expected": [ + "你好", + "世界" + ], + "passed": true + }, + { + "category": "摘要", + "prompt": "用一句话总结：人工智能（AI）是指由人工制造出来的系统所展现出来的智能。AI的核心问题包括推理、知识表示、规划、学习、自然语言处理、感知和移动与操作物体的能力。", + "response": "人工智能（AI）是指由人工制造系统展现出的智能，其核心涵盖推理、知识表示、规划、学习、自然语言处理、感知及物体操作等关键能力。", + "expected": [ + "人工智能", + "AI" + ], + "passed": true + }, + { + "category": "情感分类", + "prompt": "判断以下文本的情感是正面还是负面：'这个产品太糟糕了，完全不值这个价格'。请只回答'正面'或'负面'。", + "response": "负面", + "expected": [ + "负面" + ], + "passed": true + } + ] + } +} \ No newline at end of file diff --git a/vsp/qwen3.5-9b/results/benchmark_speed.json b/vsp/qwen3.5-9b/results/benchmark_speed.json new file mode 100644 index 0000000..f1b92f6 --- /dev/null +++ b/vsp/qwen3.5-9b/results/benchmark_speed.json @@ -0,0 +1,60 @@ +{ + "timestamp": "2026-03-16T17:14:19.845118", + "model": "Qwen3.5-9B", + "quantization": "4-bit NF4", + "speed_benchmark": [ + { + "test_name": "短输入短输出", + "input_tokens": 13, + "avg_output_tokens": 12.0, + "avg_time_s": 12.503, + "avg_tokens_per_sec": 1.0, + "min_time_s": 11.644, + "max_time_s": 13.362 + }, + { + "test_name": "短输入中输出", + "input_tokens": 14, + "avg_output_tokens": 64.0, + "avg_time_s": 38.312, + "avg_tokens_per_sec": 1.7, + "min_time_s": 38.279, + "max_time_s": 38.345 + }, + { + "test_name": "短输入长输出", + "input_tokens": 19, + "avg_output_tokens": 128.0, + "avg_time_s": 69.541, + "avg_tokens_per_sec": 1.8, + "min_time_s": 69.133, + "max_time_s": 69.949 + }, + { + "test_name": "中输入中输出", + "input_tokens": 64, + "avg_output_tokens": 128.0, + "avg_time_s": 78.318, + "avg_tokens_per_sec": 1.6, + "min_time_s": 77.585, + "max_time_s": 79.051 + }, + { + "test_name": "长输入短输出", + "input_tokens": 318, + "avg_output_tokens": 32.0, + "avg_time_s": 32.659, + "avg_tokens_per_sec": 1.0, + "min_time_s": 31.857, + "max_time_s": 33.46 + } + ], + "memory": { + "gpu_allocated_gb": 7.13, + "gpu_reserved_gb": 16.22, + "gpu_total_gb": 8.0, + "gpu_name": "NVIDIA GeForce RTX 3050 OEM", + "ram_used_gb": 7.59, + "ram_total_gb": 31.7 + } +} \ No newline at end of file diff --git a/vsp/qwen3.5-9b/results/concurrency_results.json b/vsp/qwen3.5-9b/results/concurrency_results.json new file mode 100644 index 0000000..e3a6067 --- /dev/null +++ b/vsp/qwen3.5-9b/results/concurrency_results.json @@ -0,0 +1,40 @@ +{ + "timestamp": "2026-03-16T17:34:50.981411", + "model": "Qwen3.5-9B", + "quantization": "4-bit NF4", + "note": "单GPU串行推理，并发测试主要体现请求排队效果", + "concurrency_results": [ + { + "concurrency": 1, + "total_time_s": 33.29, + "total_tokens": 64, + "throughput_tokens_per_sec": 1.9, + "avg_latency_s": 33.18, + "requests_completed": 1 + }, + { + "concurrency": 2, + "total_time_s": 65.01, + "total_tokens": 128, + "throughput_tokens_per_sec": 2.0, + "avg_latency_s": 49.14, + "requests_completed": 2 + }, + { + "concurrency": 4, + "total_time_s": 128.55, + "total_tokens": 256, + "throughput_tokens_per_sec": 2.0, + "avg_latency_s": 80.09, + "requests_completed": 4 + }, + { + "concurrency": 8, + "total_time_s": 275.44, + "total_tokens": 512, + "throughput_tokens_per_sec": 1.9, + "avg_latency_s": 148.94, + "requests_completed": 8 + } + ] +} \ No newline at end of file diff --git a/vsp/qwen3.5-9b/results/gpu_requirements.json b/vsp/qwen3.5-9b/results/gpu_requirements.json index a8a38c9..de15b4f 100644 --- a/vsp/qwen3.5-9b/results/gpu_requirements.json +++ b/vsp/qwen3.5-9b/results/gpu_requirements.json @@ -46,16 +46,17 @@ }, "actual_test_results": { "gpu": "NVIDIA GeForce RTX 3050 OEM 8GB", - "method": "FP16 + CPU offload (accelerate device_map=auto)", - "gpu_vram_used_gb": 3.91, - "ram_used_gb": 13.6, - "inference_speed_tokens_per_sec": 0.4, - "output_quality": "极差（乱码/重复输出）", - "conclusion": "RTX 3050 8GB 无法有效运行 Qwen3.5-9B，显存不足导致大量层 offload 到 CPU，推理极慢且输出质量不可用", + "method": "4-bit NF4 量化 (bitsandbytes)，纯 GPU 运行，关闭 thinking 模式", + "gpu_vram_used_gb": 7.13, + "ram_used_gb": 7.59, + "inference_speed_tokens_per_sec": "1.0-1.8", + "accuracy": "90% (9/10)", + "output_quality": "正常，回答准确", + "conclusion": "RTX 3050 8GB 可以运行 Qwen3.5-9B 4-bit 量化版本，显存占用 7.13GB，推理速度 1-2 tokens/s，适合开发测试", "issues": [ - "bitsandbytes 4-bit 量化不支持 CPU offload，8GB 显存装不下完整 4-bit 模型", - "bitsandbytes INT8 与 accelerate 版本不兼容（Windows）", - "FP16 + CPU offload 虽可加载但速度仅 0.4 tokens/s，输出为乱码" + "显存占用 7.13GB，接近 8GB 上限，长文本可能 OOM", + "推理速度较慢（1-2 tokens/s），不适合生产环境", + "需关闭 thinking 模式才能正常输出" ] }, "deployment_recommendations": { diff --git a/vsp/qwen3.5-9b/test_accuracy.py b/vsp/qwen3.5-9b/test_accuracy.py index e9dff4f..461e146 100644 --- a/vsp/qwen3.5-9b/test_accuracy.py +++ b/vsp/qwen3.5-9b/test_accuracy.py @@ -8,7 +8,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from model_utils import load_model +from model_utils import load_model, apply_chat # 测试数据集 @@ -84,7 +84,7 @@ def evaluate_accuracy(model, tokenizer): for i, test in enumerate(ACCURACY_TESTS): messages = [{"role": "user", "content": test["prompt"]}] - text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + text = apply_chat(tokenizer, messages) inputs = tokenizer(text, return_tensors="pt").to(model.device) with torch.no_grad(): diff --git a/vsp/qwen3.5-9b/test_basic_inference.py b/vsp/qwen3.5-9b/test_basic_inference.py index f46e27d..d216df3 100644 --- a/vsp/qwen3.5-9b/test_basic_inference.py +++ b/vsp/qwen3.5-9b/test_basic_inference.py @@ -1,52 +1,28 @@ """基础推理测试 - 验证模型能否正常加载和生成""" import os import sys -import glob import time import torch import psutil -from transformers import AutoModelForCausalLM, AutoTokenizer + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from model_utils import load_model, apply_chat # 修复 Windows GBK 编码问题 sys.stdout.reconfigure(encoding='utf-8', errors='replace') sys.stderr.reconfigure(encoding='utf-8', errors='replace') -def get_model_path(): - """获取模型路径""" - paths = glob.glob("vsp/qwen3.5-9b/model/**/config.json", recursive=True) - if paths: - return os.path.dirname(paths[0]) - return "Qwen/Qwen3.5-9B" - - def test_basic_inference(): """基础推理测试""" print("=" * 60) - print("Qwen3.5-9B 基础推理测试") + print("Qwen3.5-9B 基础推理测试 (4-bit NF4 量化, 纯GPU)") print("=" * 60) - model_path = get_model_path() - print(f"\n模型路径: {model_path}") - - # 加载 tokenizer - print("加载 tokenizer...") + # 加载模型 + print("\n加载模型...") t0 = time.time() - tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) - print(f" Tokenizer 加载耗时: {time.time() - t0:.2f}s") - - # 加载模型 (FP16 + GPU/CPU offload) - print("加载模型 (FP16 + CPU offload)...") - max_memory = {0: "6GiB", "cpu": "24GiB"} - t0 = time.time() - model = AutoModelForCausalLM.from_pretrained( - model_path, - torch_dtype=torch.float16, - device_map="auto", - max_memory=max_memory, - offload_folder="vsp/qwen3.5-9b/offload", - trust_remote_code=True, - ) + model, tokenizer = load_model() load_time = time.time() - t0 print(f" 模型加载耗时: {load_time:.2f}s") @@ -72,7 +48,7 @@ def test_basic_inference(): for i, prompt in enumerate(test_prompts): print(f"\n--- 测试 {i+1}: {prompt[:30]}... ---") messages = [{"role": "user", "content": prompt}] - text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + text = apply_chat(tokenizer, messages) inputs = tokenizer(text, return_tensors="pt").to(model.device) input_len = inputs["input_ids"].shape[1] diff --git a/vsp/qwen3.5-9b/test_concurrency.py b/vsp/qwen3.5-9b/test_concurrency.py index 012653c..f4b14c0 100644 --- a/vsp/qwen3.5-9b/test_concurrency.py +++ b/vsp/qwen3.5-9b/test_concurrency.py @@ -11,13 +11,13 @@ from transformers import AutoModelForCausalLM, AutoTokenizer from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from model_utils import load_model +from model_utils import load_model, apply_chat def single_inference(model, tokenizer, prompt, lock, max_tokens=64): """单次推理（线程安全）""" messages = [{"role": "user", "content": prompt}] - text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + text = apply_chat(tokenizer, messages) inputs = tokenizer(text, return_tensors="pt").to(model.device) input_len = inputs["input_ids"].shape[1]