Fine-tuning-13模型部署与推理优化
·
第十三章:模型部署与推理优化
13.1 模型部署架构设计
13.1.1 部署架构概述
import torch
import torch.nn as nn
from typing import Dict, Any, List, Optional
import numpy as np
from dataclasses import dataclass
import asyncio
import time
from concurrent.futures import ThreadPoolExecutor
import logging
from prometheus_client import Counter, Histogram, Gauge
import json
import hashlib
@dataclass
class DeploymentConfig:
"""部署配置"""
model_name: str
model_path: str
device: str = "cuda" if torch.cuda.is_available() else "cpu"
batch_size: int = 1
max_sequence_length: int = 512
precision: str = "fp16" # fp32, fp16, int8
use_tensor_parallel: bool = False
tensor_parallel_size: int = 1
use_pipeline_parallel: bool = False
pipeline_parallel_size: int = 1
cache_dir: str = "./model_cache"
enable_quantization: bool = False
quantization_bits: int = 8
enable_kernel_fusion: bool = True
enable_memory_efficient_attention: bool = True
max_concurrent_requests: int = 100
request_timeout: int = 30
enable_dynamic_batching: bool = True
batching_timeout_ms: int = 50
max_batch_size: int = 16
class ModelDeploymentArchitecture:
"""模型部署架构"""
def __init__(self, config: DeploymentConfig):
self.config = config
self.model = None
self.tokenizer = None
self.device = torch.device(config.device)
self.logger = self._setup_logging()
self.metrics = self._setup_metrics()
self.request_queue = asyncio.Queue(maxsize=config.max_concurrent_requests)
self.executor = ThreadPoolExecutor(max_workers=config.max_concurrent_requests)
def _setup_logging(self) -> logging.Logger:
"""设置日志"""
logger = logging.getLogger(f"ModelDeployment_{self.config.model_name}")
logger.setLevel(logging.INFO)
if not logger.handlers:
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def _setup_metrics(self) -> Dict[str, Any]:
"""设置监控指标"""
return {
'request_count': Counter('model_requests_total', 'Total model requests'),
'request_duration': Histogram('model_request_duration_seconds', 'Model request duration'),
'active_requests': Gauge('model_active_requests', 'Active model requests'),
'batch_size': Histogram('model_batch_size', 'Model batch size'),
'cache_hits': Counter('model_cache_hits_total', 'Model cache hits'),
'cache_misses': Counter('model_cache_misses_total', 'Model cache misses'),
'memory_usage': Gauge('model_memory_usage_bytes', 'Model memory usage'),
'gpu_utilization': Gauge('model_gpu_utilization_percent', 'GPU utilization percentage')
}
async def initialize_model(self):
"""初始化模型"""
self.logger.info(f"Initializing model: {self.config.model_name}")
try:
# 加载模型和分词器
self.model, self.tokenizer = await self._load_model()
# 应用优化
self.model = self._apply_optimizations(self.model)
# 预热模型
await self._warmup_model()
self.logger.info("Model initialization completed successfully")
except Exception as e:
self.logger.error(f"Model initialization failed: {str(e)}")
raise
async def _load_model(self):
"""加载模型"""
# 这里应该根据具体的模型类型加载
# 简化示例,实际实现需要根据具体模型架构
from transformers import AutoModel, AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(
self.config.model_path,
cache_dir=self.config.cache_dir
)
model = AutoModel.from_pretrained(
self.config.model_path,
cache_dir=self.config.cache_dir,
torch_dtype=torch.float16 if self.config.precision == "fp16" else torch.float32
)
model.to(self.device)
model.eval()
return model, tokenizer
def _apply_optimizations(self, model: nn.Module) -> nn.Module:
"""应用优化"""
optimizations_applied = []
# 量化优化
if self.config.enable_quantization:
model = self._apply_quantization(model)
optimizations_applied.append("quantization")
# 内核融合
if self.config.enable_kernel_fusion:
model = self._apply_kernel_fusion(model)
optimizations_applied.append("kernel_fusion")
# 内存高效注意力
if self.config.enable_memory_efficient_attention:
model = self._apply_memory_efficient_attention(model)
optimizations_applied.append("memory_efficient_attention")
# 编译优化(PyTorch 2.0+)
if hasattr(torch, 'compile'):
try:
model = torch.compile(model, mode="max-autotune")
optimizations_applied.append("torch_compile")
except Exception as e:
self.logger.warning(f"Torch compile optimization failed: {str(e)}")
self.logger.info(f"Applied optimizations: {optimizations_applied}")
return model
def _apply_quantization(self, model: nn.Module) -> nn.Module:
"""应用量化"""
try:
if self.config.quantization_bits == 8:
# 动态量化
model = torch.quantization.quantize_dynamic(
model,
{nn.Linear, nn.Conv2d},
dtype=torch.qint8
)
elif self.config.quantization_bits == 4:
# 需要更复杂的4bit量化实现
self.logger.warning("4-bit quantization requires specialized implementation")
self.logger.info(f"Applied {self.config.quantization_bits}-bit quantization")
except Exception as e:
self.logger.error(f"Quantization failed: {str(e)}")
return model
def _apply_kernel_fusion(self, model: nn.Module) -> nn.Module:
"""应用内核融合"""
# 这里可以实现特定的内核融合优化
# 例如,融合连续的线性层或激活函数
self.logger.info("Applied kernel fusion optimizations")
return model
def _apply_memory_efficient_attention(self, model: nn.Module) -> nn.Module:
"""应用内存高效注意力"""
# 如果模型包含注意力机制,可以应用内存高效实现
# 例如使用Flash Attention或类似技术
self.logger.info("Applied memory efficient attention")
return model
async def _warmup_model(self):
"""预热模型"""
self.logger.info("Warming up model...")
# 创建虚拟输入进行预热
dummy_input = torch.randint(
0, 1000,
(self.config.batch_size, self.config.max_sequence_length),
device=self.device
)
with torch.no_grad():
for _ in range(3): # 预热3次
_ = self.model(dummy_input)
self.logger.info("Model warmup completed")
class InferenceOptimizer:
"""推理优化器"""
def __init__(self, config: DeploymentConfig):
self.config = config
self.logger = logging.getLogger("InferenceOptimizer")
self.cache = {}
self.cache_stats = {'hits': 0, 'misses': 0}
def optimize_inference(self, model: nn.Module, input_data: torch.Tensor) -> torch.Tensor:
"""优化推理"""
start_time = time.time()
# 检查缓存
cache_key = self._generate_cache_key(input_data)
if cache_key in self.cache:
self.cache_stats['hits'] += 1
return self.cache[cache_key]
self.cache_stats['misses'] += 1
# 批量处理优化
if self.config.enable_dynamic_batching and len(input_data) < self.config.max_batch_size:
input_data = self._optimize_batching(input_data)
# 精度优化
if self.config.precision == "fp16":
input_data = input_data.half()
# 执行推理
with torch.no_grad():
if self.config.device == "cuda":
with torch.cuda.amp.autocast():
output = model(input_data)
else:
output = model(input_data)
# 缓存结果
self.cache[cache_key] = output
# 清理旧缓存
self._cleanup_cache()
inference_time = time.time() - start_time
self.logger.debug(f"Inference completed in {inference_time:.3f}s")
return output
def _generate_cache_key(self, input_data: torch.Tensor) -> str:
"""生成缓存键"""
# 基于输入数据的哈希值
input_hash = hashlib.md5(input_data.cpu().numpy().tobytes()).hexdigest()
return input_hash
def _optimize_batching(self, input_data: torch.Tensor) -> torch.Tensor:
"""优化批处理"""
# 这里可以实现智能批处理逻辑
# 例如,将多个小请求组合成更大的批次
return input_data
def _cleanup_cache(self):
"""清理缓存"""
# 限制缓存大小
max_cache_size = 1000
if len(self.cache) > max_cache_size:
# 移除最旧的条目
keys_to_remove = list(self.cache.keys())[:len(self.cache) - max_cache_size]
for key in keys_to_remove:
del self.cache[key]
class DynamicBatcher:
"""动态批处理器"""
def __init__(self, max_batch_size: int, timeout_ms: int):
self.max_batch_size = max_batch_size
self.timeout_ms = timeout_ms
self.pending_requests = []
self.last_batch_time = time.time()
async def add_request(self, request_data: Dict[str, Any]) -> asyncio.Future:
"""添加请求到批处理"""
future = asyncio.Future()
self.pending_requests.append({
'data': request_data,
'future': future,
'timestamp': time.time()
})
# 如果批次已满或超时,立即处理
if (len(self.pending_requests) >= self.max_batch_size or
(time.time() - self.last_batch_time) * 1000 > self.timeout_ms):
await self._process_batch()
return future
async def _process_batch(self):
"""处理批次"""
if not self.pending_requests:
return
batch_data = self.pending_requests[:self.max_batch_size]
self.pending_requests = self.pending_requests[self.max_batch_size:]
# 处理批次
# 这里应该调用模型推理
batch_results = await self._execute_batch(batch_data)
# 返回结果
for request, result in zip(batch_data, batch_results):
if not request['future'].done():
request['future'].set_result(result)
self.last_batch_time = time.time()
async def _execute_batch(self, batch_data: List[Dict[str, Any]]) -> List[Any]:
"""执行批次推理"""
# 这里应该实现实际的批次推理逻辑
# 现在返回模拟结果
return [{'result': f"processed_{i}"} for i in range(len(batch_data))]
async def flush(self):
"""刷新剩余请求"""
while self.pending_requests:
await self._process_batch()
class ModelServingAPI:
"""模型服务API"""
def __init__(self, deployment_architecture: ModelDeploymentArchitecture):
self.deployment = deployment_architecture
self.app = None # FastAPI应用
self.batcher = None
self.request_id_counter = 0
async def setup_api(self):
"""设置API"""
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
import uvicorn
self.app = FastAPI(title="Model Serving API", version="1.0.0")
# 设置动态批处理器
if self.deployment.config.enable_dynamic_batching:
self.batcher = DynamicBatcher(
self.deployment.config.max_batch_size,
self.deployment.config.batching_timeout_ms
)
# 定义API端点
@self.app.post("/predict")
async def predict(request: Request):
return await self._handle_prediction(request)
@self.app.get("/health")
async def health():
return {"status": "healthy", "model": self.deployment.config.model_name}
@self.app.get("/metrics")
async def metrics():
return self._get_metrics()
@self.app.post("/batch_predict")
async def batch_predict(request: Request):
return await self._handle_batch_prediction(request)
return self.app
async def _handle_prediction(self, request: Request) -> Dict[str, Any]:
"""处理预测请求"""
start_time = time.time()
self.request_id_counter += 1
request_id = f"req_{self.request_id_counter}"
try:
# 记录指标
self.deployment.metrics['request_count'].inc()
self.deployment.metrics['active_requests'].inc()
# 解析请求
request_data = await request.json()
# 验证输入
self._validate_input(request_data)
# 处理请求
if self.batcher:
# 使用动态批处理
result = await self.batcher.add_request(request_data)
else:
# 直接处理
result = await self._process_single_request(request_data)
# 记录处理时间
processing_time = time.time() - start_time
self.deployment.metrics['request_duration'].observe(processing_time)
return {
'request_id': request_id,
'result': result,
'processing_time': processing_time
}
except Exception as e:
self.deployment.logger.error(f"Prediction failed for request {request_id}: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
finally:
self.deployment.metrics['active_requests'].dec()
async def _handle_batch_prediction(self, request: Request) -> Dict[str, Any]:
"""处理批量预测请求"""
try:
request_data = await request.json()
batch_inputs = request_data.get('inputs', [])
if not batch_inputs:
raise HTTPException(status_code=400, detail="No inputs provided")
# 处理批量输入
results = []
for input_data in batch_inputs:
result = await self._process_single_request(input_data)
results.append(result)
return {
'results': results,
'batch_size': len(batch_inputs)
}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
async def _process_single_request(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
"""处理单个请求"""
# 这里应该实现实际的模型推理逻辑
# 简化示例
input_text = request_data.get('input', '')
# 模拟推理
await asyncio.sleep(0.1) # 模拟推理时间
return {
'input': input_text,
'output': f"Processed: {input_text}",
'confidence': 0.95
}
def _validate_input(self, request_data: Dict[str, Any]):
"""验证输入数据"""
if 'input' not in request_data:
raise ValueError("Missing 'input' field")
input_text = request_data['input']
if not isinstance(input_text, str):
raise ValueError("'input' must be a string")
if len(input_text) > self.deployment.config.max_sequence_length:
raise ValueError(f"Input length exceeds maximum {self.deployment.config.max_sequence_length}")
def _get_metrics(self) -> Dict[str, Any]:
"""获取监控指标"""
return {
'cache_stats': self.deployment.optimizer.cache_stats if hasattr(self.deployment, 'optimizer') else {},
'request_count': self.deployment.metrics['request_count']._value.get(),
'active_requests': self.deployment.metrics['active_requests']._value.get(),
'memory_usage': self._get_memory_usage()
}
def _get_memory_usage(self) -> Dict[str, Any]:
"""获取内存使用情况"""
if torch.cuda.is_available():
return {
'gpu_memory_allocated': torch.cuda.memory_allocated(),
'gpu_memory_reserved': torch.cuda.memory_reserved(),
'gpu_memory_total': torch.cuda.get_device_properties(0).total_memory
}
else:
return {'cpu_memory': 'N/A'}
# 使用示例
async def demonstrate_model_deployment():
"""演示模型部署"""
print("=== 模型部署与推理优化演示 ===")
# 创建部署配置
deployment_config = DeploymentConfig(
model_name="bert-base-uncased",
model_path="bert-base-uncased",
device="cuda" if torch.cuda.is_available() else "cpu",
batch_size=4,
precision="fp16",
enable_quantization=True,
enable_dynamic_batching=True,
max_batch_size=8,
batching_timeout_ms=100
)
# 创建部署架构
deployment = ModelDeploymentArchitecture(deployment_config)
try:
# 初始化模型
await deployment.initialize_model()
# 创建推理优化器
optimizer = InferenceOptimizer(deployment_config)
deployment.optimizer = optimizer
# 创建API服务
api = ModelServingAPI(deployment)
app = await api.setup_api()
print("模型部署完成!")
print(f"模型: {deployment_config.model_name}")
print(f"设备: {deployment_config.device}")
print(f"精度: {deployment_config.precision}")
print(f"批处理: {'启用' if deployment_config.enable_dynamic_batching else '禁用'}")
# 演示推理优化
print("\n=== 推理优化演示 ===")
# 创建测试输入
test_input = torch.randint(0, 1000, (1, 128), device=deployment.device)
# 执行优化推理
start_time = time.time()
output = optimizer.optimize_inference(deployment.model, test_input)
inference_time = time.time() - start_time
print(f"推理完成,耗时: {inference_time:.3f}秒")
print(f"输出形状: {output.shape}")
print(f"缓存命中率: {optimizer.cache_stats['hits']}/{optimizer.cache_stats['hits'] + optimizer.cache_stats['misses']}")
# 演示批量推理
print("\n=== 批量推理演示 ===")
batch_input = torch.randint(0, 1000, (4, 128), device=deployment.device)
start_time = time.time()
batch_output = optimizer.optimize_inference(deployment.model, batch_input)
batch_time = time.time() - start_time
print(f"批量推理完成,耗时: {batch_time:.3f}秒")
print(f"平均每个样本: {batch_time/4:.3f}秒")
print(f"批处理加速比: {(batch_time/4) / inference_time:.2f}x")
except Exception as e:
print(f"部署失败: {str(e)}")
raise
if __name__ == "__main__":
asyncio.run(demonstrate_model_deployment())
13.2 推理优化技术
13.2.1 动态批处理优化
class AdvancedDynamicBatcher:
"""高级动态批处理器"""
def __init__(self, max_batch_size: int, timeout_ms: int, padding_token: int = 0):
self.max_batch_size = max_batch_size
self.timeout_ms = timeout_ms
self.padding_token = padding_token
self.request_queue = asyncio.Queue()
self.processing = False
self.stats = {
'batches_processed': 0,
'total_requests': 0,
'avg_batch_size': 0,
'max_batch_size_achieved': 0
}
async def add_request(self, input_ids: List[int], attention_mask: List[int],
future: asyncio.Future) -> None:
"""添加请求到队列"""
request = {
'input_ids': input_ids,
'attention_mask': attention_mask,
'future': future,
'timestamp': time.time()
}
await self.request_queue.put(request)
self.stats['total_requests'] += 1
# 启动批处理循环(如果尚未启动)
if not self.processing:
asyncio.create_task(self._batch_processing_loop())
async def _batch_processing_loop(self):
"""批处理循环"""
self.processing = True
while True:
batch = []
earliest_timestamp = time.time()
# 收集批次
timeout = self.timeout_ms / 1000.0
start_time = time.time()
while (len(batch) < self.max_batch_size and
(time.time() - start_time) < timeout):
try:
# 等待请求,超时时间为剩余时间
remaining_timeout = timeout - (time.time() - start_time)
if remaining_timeout <= 0:
break
request = await asyncio.wait_for(
self.request_queue.get(),
timeout=remaining_timeout
)
batch.append(request)
earliest_timestamp = min(earliest_timestamp, request['timestamp'])
except asyncio.TimeoutError:
break
# 处理批次
if batch:
await self._process_batch(batch)
# 如果队列为空,退出循环
if self.request_queue.empty():
break
self.processing = False
async def _process_batch(self, batch: List[Dict[str, Any]]):
"""处理批次"""
start_time = time.time()
# 准备批次数据
batch_data = self._prepare_batch_data(batch)
# 执行模型推理(这里应该是实际的模型调用)
# 现在使用模拟结果
batch_results = await self._simulate_model_inference(batch_data)
# 分发结果
for request, result in zip(batch, batch_results):
if not request['future'].done():
request['future'].set_result(result)
# 更新统计
batch_size = len(batch)
self.stats['batches_processed'] += 1
self.stats['max_batch_size_achieved'] = max(
self.stats['max_batch_size_achieved'], batch_size
)
processing_time = time.time() - start_time
self.logger.info(f"Processed batch of size {batch_size} in {processing_time:.3f}s")
def _prepare_batch_data(self, batch: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]:
"""准备批次数据"""
# 获取最大长度
max_length = max(len(req['input_ids']) for req in batch)
# 填充输入
padded_input_ids = []
padded_attention_masks = []
for request in batch:
input_ids = request['input_ids']
attention_mask = request['attention_mask']
# 填充到最大长度
padding_length = max_length - len(input_ids)
padded_input_ids.append(input_ids + [self.padding_token] * padding_length)
padded_attention_masks.append(attention_mask + [0] * padding_length)
# 转换为张量
input_ids_tensor = torch.tensor(padded_input_ids, dtype=torch.long)
attention_mask_tensor = torch.tensor(padded_attention_masks, dtype=torch.long)
return {
'input_ids': input_ids_tensor,
'attention_mask': attention_mask_tensor
}
async def _simulate_model_inference(self, batch_data: Dict[str, torch.Tensor]) -> List[Dict[str, Any]]:
"""模拟模型推理"""
batch_size = batch_data['input_ids'].shape[0]
# 模拟推理时间(与实际批大小相关)
inference_time = 0.1 + 0.01 * batch_size # 基础时间 + 每样本时间
await asyncio.sleep(inference_time)
# 生成模拟结果
results = []
for i in range(batch_size):
results.append({
'predictions': np.random.rand(10), # 模拟预测结果
'confidence': np.random.rand(),
'processing_time': inference_time / batch_size
})
return results
def get_statistics(self) -> Dict[str, Any]:
"""获取统计信息"""
if self.stats['batches_processed'] > 0:
self.stats['avg_batch_size'] = (
self.stats['total_requests'] / self.stats['batches_processed']
)
return self.stats.copy()
class ContinuousBatchingOptimizer:
"""连续批处理优化器"""
def __init__(self, model, tokenizer, max_batch_size: int = 32,
max_sequence_length: int = 2048):
self.model = model
self.tokenizer = tokenizer
self.max_batch_size = max_batch_size
self.max_sequence_length = max_sequence_length
self.active_requests = []
self.completed_requests = []
self.stats = {
'total_requests': 0,
'total_tokens': 0,
'avg_batch_utilization': 0.0,
'throughput_tokens_per_sec': 0.0
}
async def add_request(self, prompt: str, max_new_tokens: int,
temperature: float = 1.0, top_p: float = 1.0) -> asyncio.Future:
"""添加生成请求"""
future = asyncio.Future()
request = {
'prompt': prompt,
'max_new_tokens': max_new_tokens,
'temperature': temperature,
'top_p': top_p,
'future': future,
'input_tokens': None,
'generated_tokens': [],
'is_completed': False,
'start_time': time.time()
}
# 编码输入
inputs = self.tokenizer(prompt, return_tensors='pt', truncation=True,
max_length=self.max_sequence_length)
request['input_tokens'] = inputs['input_ids'].shape[1]
self.active_requests.append(request)
self.stats['total_requests'] += 1
self.stats['total_tokens'] += request['input_tokens']
return future
async def generate_continuous(self):
"""连续生成"""
while self.active_requests:
# 选择要处理的请求
batch_requests = self._select_requests_for_batch()
if batch_requests:
await self._process_batch_continuous(batch_requests)
# 检查完成的请求
self._check_completed_requests()
# 小延迟以避免忙等待
await asyncio.sleep(0.01)
def _select_requests_for_batch(self) -> List[Dict[str, Any]]:
"""选择批处理请求"""
if not self.active_requests:
return []
# 简单的选择策略:选择最早添加的请求
# 可以根据更复杂的启发式方法进行优化
batch_size = min(self.max_batch_size, len(self.active_requests))
return self.active_requests[:batch_size]
async def _process_batch_continuous(self, batch_requests: List[Dict[str, Any]]):
"""处理连续批次"""
# 准备批次输入
batch_inputs = self._prepare_batch_inputs(batch_requests)
# 生成下一个token
next_tokens = await self._generate_next_tokens(batch_inputs)
# 更新请求状态
for i, (request, next_token) in enumerate(zip(batch_requests, next_tokens)):
request['generated_tokens'].append(next_token)
# 检查是否完成
if (len(request['generated_tokens']) >= request['max_new_tokens'] or
next_token == self.tokenizer.eos_token_id):
request['is_completed'] = True
# 更新统计
batch_utilization = len(batch_requests) / self.max_batch_size
self.stats['avg_batch_utilization'] = (
(self.stats['avg_batch_utilization'] * (self.stats['total_requests'] - 1) + batch_utilization) /
self.stats['total_requests']
)
def _prepare_batch_inputs(self, batch_requests: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]:
"""准备批次输入"""
# 构建当前的输入序列
input_ids_list = []
attention_mask_list = []
for request in batch_requests:
# 组合原始输入和已生成的token
all_tokens = request['input_tokens'] + request['generated_tokens']
# 截断到最大长度
if len(all_tokens) > self.max_sequence_length:
all_tokens = all_tokens[-self.max_sequence_length:]
# 创建attention mask
attention_mask = [1] * len(all_tokens)
input_ids_list.append(all_tokens)
attention_mask_list.append(attention_mask)
# 填充到相同长度
max_length = max(len(seq) for seq in input_ids_list)
padded_input_ids = []
padded_attention_masks = []
for input_ids, attention_mask in zip(input_ids_list, attention_mask_list):
padding_length = max_length - len(input_ids)
padded_input_ids.append([self.tokenizer.pad_token_id] * padding_length + input_ids)
padded_attention_masks.append([0] * padding_length + attention_mask)
return {
'input_ids': torch.tensor(padded_input_ids, dtype=torch.long),
'attention_mask': torch.tensor(padded_attention_masks, dtype=torch.long)
}
async def _generate_next_tokens(self, batch_inputs: Dict[str, torch.Tensor]) -> List[int]:
"""生成下一个token"""
# 这里应该是实际的模型推理
# 现在使用模拟结果
batch_size = batch_inputs['input_ids'].shape[0]
# 模拟推理时间
await asyncio.sleep(0.05)
# 生成随机token(模拟)
next_tokens = np.random.randint(0, self.tokenizer.vocab_size, batch_size)
return next_tokens.tolist()
def _check_completed_requests(self):
"""检查完成的请求"""
completed = [req for req in self.active_requests if req['is_completed']]
for request in completed:
# 完成future
if not request['future'].done():
result = {
'generated_text': self.tokenizer.decode(request['generated_tokens'], skip_special_tokens=True),
'generated_tokens': request['generated_tokens'],
'total_tokens': request['input_tokens'] + len(request['generated_tokens']),
'processing_time': time.time() - request['start_time']
}
request['future'].set_result(result)
# 从活动列表中移除
self.active_requests.remove(request)
self.completed_requests.append(request)
def get_statistics(self) -> Dict[str, Any]:
"""获取统计信息"""
if self.completed_requests:
total_processing_time = sum(
req['future'].result()['processing_time'] for req in self.completed_requests
if req['future'].done()
)
if total_processing_time > 0:
self.stats['throughput_tokens_per_sec'] = (
self.stats['total_tokens'] / total_processing_time
)
return self.stats.copy()
# 使用示例
async def demonstrate_advanced_inference_optimization():
"""演示高级推理优化"""
print("=== 高级推理优化演示 ===")
# 演示动态批处理
print("\n1. 动态批处理优化")
batcher = AdvancedDynamicBatcher(max_batch_size=8, timeout_ms=100)
# 模拟并发请求
async def make_request(request_id: int, input_length: int):
input_ids = list(range(input_length))
attention_mask = [1] * input_length
future = asyncio.Future()
await batcher.add_request(input_ids, attention_mask, future)
result = await future
print(f"请求 {request_id} 完成,批大小: {result.get('batch_size', 1)}")
return result
# 创建多个并发请求
tasks = []
for i in range(20):
input_length = np.random.randint(10, 50)
tasks.append(make_request(i, input_length))
# 等待所有请求完成
results = await asyncio.gather(*tasks)
# 获取统计信息
stats = batcher.get_statistics()
print(f"\n批处理统计:")
print(f"处理批次: {stats['batches_processed']}")
print(f"总请求: {stats['total_requests']}")
print(f"平均批大小: {stats['avg_batch_size']:.2f}")
print(f"最大批大小: {stats['max_batch_size_achieved']}")
# 演示连续批处理(需要实际的模型和分词器)
print("\n2. 连续批处理优化(概念演示)")
# 这里需要实际的模型和分词器
# continuous_batcher = ContinuousBatchingOptimizer(model, tokenizer)
print("连续批处理优化器已准备(需要实际模型进行完整演示)")
if __name__ == "__main__":
asyncio.run(demonstrate_advanced_inference_optimization())
13.3 模型压缩与量化
13.3.1 量化策略实现
class ModelQuantizationFramework:
"""模型量化框架"""
def __init__(self, quantization_config: Dict[str, Any]):
self.config = quantization_config
self.quantization_stats = {}
def apply_post_training_quantization(self, model: nn.Module,
calibration_data: List[torch.Tensor]) -> nn.Module:
"""应用训练后量化"""
print("Applying post-training quantization...")
# 收集统计信息
self._collect_activation_stats(model, calibration_data)
# 应用量化
quantized_model = self._quantize_model_weights(model)
# 校准激活量化参数
self._calibrate_activation_quantization(model, calibration_data)
return quantized_model
def _collect_activation_stats(self, model: nn.Module,
calibration_data: List[torch.Tensor]):
"""收集激活统计信息"""
activation_stats = {}
def hook_fn(name):
def hook(module, input, output):
if name not in activation_stats:
activation_stats[name] = []
# 收集激活值
if isinstance(output, torch.Tensor):
activation_stats[name].append(output.detach().cpu().numpy())
return hook
# 注册钩子
hooks = []
for name, module in model.named_modules():
if isinstance(module, (nn.Linear, nn.Conv2d, nn.ReLU)):
hook = module.register_forward_hook(hook_fn(name))
hooks.append(hook)
# 前向传播收集统计信息
model.eval()
with torch.no_grad():
for data in calibration_data:
_ = model(data)
# 移除钩子
for hook in hooks:
hook.remove()
# 计算统计信息
self.quantization_stats['activations'] = {}
for name, activations in activation_stats.items():
if activations:
all_activations = np.concatenate([act.flatten() for act in activations])
self.quantization_stats['activations'][name] = {
'min': float(np.min(all_activations)),
'max': float(np.max(all_activations)),
'mean': float(np.mean(all_activations)),
'std': float(np.std(all_activations)),
'percentile_99': float(np.percentile(all_activations, 99)),
'percentile_1': float(np.percentile(all_activations, 1))
}
print(f"Collected activation stats for {len(activation_stats)} layers")
def _quantize_model_weights(self, model: nn.Module) -> nn.Module:
"""量化模型权重"""
quantization_config = self.config.get('weight_quantization', {
'bits': 8,
'symmetric': True,
'per_channel': True
})
bits = quantization_config['bits']
symmetric = quantization_config['symmetric']
per_channel = quantization_config['per_channel']
print(f"Quantizing weights to {bits}-bit ({'symmetric' if symmetric else 'asymmetric'})")
for name, module in model.named_modules():
if isinstance(module, (nn.Linear, nn.Conv2d)):
# 获取权重
weight = module.weight.data
# 量化权重
if per_channel:
quantized_weight = self._quantize_per_channel(weight, bits, symmetric)
else:
quantized_weight = self._quantize_per_tensor(weight, bits, symmetric)
# 替换权重
module.weight.data = quantized_weight
# 存储量化参数
if not hasattr(model, 'quantization_params'):
model.quantization_params = {}
model.quantization_params[name] = {
'weight_bits': bits,
'symmetric': symmetric,
'per_channel': per_channel
}
return model
def _quantize_per_tensor(self, tensor: torch.Tensor, bits: int,
symmetric: bool = True) -> torch.Tensor:
"""逐张量化"""
if symmetric:
# 对称量化
max_val = torch.max(torch.abs(tensor))
scale = max_val / (2**(bits-1) - 1)
zero_point = 0
else:
# 非对称量化
min_val = torch.min(tensor)
max_val = torch.max(tensor)
scale = (max_val - min_val) / (2**bits - 1)
zero_point = -min_val / scale
# 量化
quantized = torch.round(tensor / scale + zero_point)
# 裁剪到有效范围
q_min = 0 if not symmetric else -(2**(bits-1))
q_max = 2**bits - 1 if not symmetric else 2**(bits-1) - 1
quantized = torch.clamp(quantized, q_min, q_max)
# 反量化(用于模拟量化效果)
dequantized = scale * (quantized - zero_point)
return dequantized
def _quantize_per_channel(self, tensor: torch.Tensor, bits: int,
symmetric: bool = True) -> torch.Tensor:
"""逐通道量化"""
original_shape = tensor.shape
if tensor.dim() == 2: # Linear层
tensor_2d = tensor
elif tensor.dim() == 4: # Conv2d层
tensor_2d = tensor.view(tensor.shape[0], -1)
else:
return self._quantize_per_tensor(tensor, bits, symmetric)
quantized_channels = []
for i in range(tensor_2d.shape[0]):
channel = tensor_2d[i]
quantized_channel = self._quantize_per_tensor(channel, bits, symmetric)
quantized_channels.append(quantized_channel)
quantized_2d = torch.stack(quantized_channels)
# 恢复原始形状
if tensor.dim() == 4:
quantized_2d = quantized_2d.view(original_shape)
return quantized_2d
def _calibrate_activation_quantization(self, model: nn.Module,
calibration_data: List[torch.Tensor]):
"""校准激活量化"""
print("Calibrating activation quantization...")
# 使用收集的统计信息设置激活量化参数
if 'activations' in self.quantization_stats:
self.quantization_stats['activation_quantization'] = {}
for layer_name, stats in self.quantization_stats['activations'].items():
# 计算量化参数
min_val = stats['percentile_1'] # 使用1%分位数避免极端值
max_val = stats['percentile_99'] # 使用99%分位数
scale = (max_val - min_val) / 255.0 # 8-bit量化
zero_point = -min_val / scale
self.quantization_stats['activation_quantization'][layer_name] = {
'scale': scale,
'zero_point': zero_point,
'min_val': min_val,
'max_val': max_val
}
def apply_quantization_aware_training(self, model: nn.Module,
train_loader: torch.utils.data.DataLoader,
num_epochs: int = 5) -> nn.Module:
"""应用量化感知训练"""
print("Applying quantization-aware training...")
# 创建量化感知训练模型
qat_model = self._create_qat_model(model)
# 训练配置
optimizer = torch.optim.AdamW(qat_model.parameters(), lr=1e-5)
criterion = nn.CrossEntropyLoss()
# 训练循环
qat_model.train()
for epoch in range(num_epochs):
total_loss = 0
for batch_idx, (inputs, targets) in enumerate(train_loader):
optimizer.zero_grad()
# 前向传播
outputs = qat_model(inputs)
loss = criterion(outputs, targets)
# 后向传播
loss.backward()
optimizer.step()
total_loss += loss.item()
if batch_idx % 100 == 0:
print(f"Epoch {epoch+1}/{num_epochs}, Batch {batch_idx}, Loss: {loss.item():.4f}")
avg_loss = total_loss / len(train_loader)
print(f"Epoch {epoch+1} completed, Average Loss: {avg_loss:.4f}")
return qat_model
def _create_qat_model(self, model: nn.Module) -> nn.Module:
"""创建量化感知训练模型"""
class QuantizationAwareLayer(nn.Module):
def __init__(self, original_layer, quantization_func):
super().__init__()
self.original_layer = original_layer
self.quantization_func = quantization_func
self.scale = nn.Parameter(torch.ones(1))
self.zero_point = nn.Parameter(torch.zeros(1))
def forward(self, x):
# 量化激活
x_quantized = self.quantization_func(x, self.scale, self.zero_point)
# 前向传播
output = self.original_layer(x_quantized)
# 量化输出
output_quantized = self.quantization_func(output, self.scale, self.zero_point)
return output_quantized
# 为线性层和卷积层添加量化感知
qat_model = copy.deepcopy(model)
for name, module in qat_model.named_modules():
if isinstance(module, (nn.Linear, nn.Conv2d)):
# 包装层以添加量化感知
parent_name = '.'.join(name.split('.')[:-1])
layer_name = name.split('.')[-1]
parent_module = qat_model
if parent_name:
for part in parent_name.split('.'):
parent_module = getattr(parent_module, part)
quantized_layer = QuantizationAwareLayer(
module, self._fake_quantize
)
setattr(parent_module, layer_name, quantized_layer)
return qat_model
def _fake_quantize(self, tensor: torch.Tensor, scale: torch.Tensor,
zero_point: torch.Tensor) -> torch.Tensor:
"""假量化(用于训练)"""
# 量化
quantized = torch.round(tensor / scale + zero_point)
quantized = torch.clamp(quantized, 0, 255)
# 反量化
dequantized = scale * (quantized - zero_point)
return dequantized
def get_quantization_stats(self) -> Dict[str, Any]:
"""获取量化统计信息"""
return self.quantization_stats.copy()
# 使用示例
def demonstrate_model_quantization():
"""演示模型量化"""
print("=== 模型量化演示 ===")
# 创建量化配置
quantization_config = {
'weight_quantization': {
'bits': 8,
'symmetric': True,
'per_channel': True
},
'activation_quantization': {
'bits': 8,
'symmetric': False,
'calibration_method': 'percentile'
}
}
# 创建量化框架
quantization_framework = ModelQuantizationFramework(quantization_config)
# 创建模拟模型
class SimpleModel(nn.Module):
def __init__(self):
super().__init__()
self.linear1 = nn.Linear(128, 256)
self.relu = nn.ReLU()
self.linear2 = nn.Linear(256, 128)
self.linear3 = nn.Linear(128, 10)
def forward(self, x):
x = self.relu(self.linear1(x))
x = self.relu(self.linear2(x))
x = self.linear3(x)
return x
model = SimpleModel()
# 创建校准数据
calibration_data = [
torch.randn(32, 128) for _ in range(10)
]
print("原始模型结构:")
for name, module in model.named_modules():
if isinstance(module, (nn.Linear, nn.Conv2d)):
print(f" {name}: {module}")
# 应用训练后量化
quantized_model = quantization_framework.apply_post_training_quantization(
model, calibration_data
)
print(f"\n量化完成!")
# 获取量化统计
stats = quantization_framework.get_quantization_stats()
if 'activations' in stats:
print(f"\n激活统计信息:")
for layer_name, layer_stats in stats['activations'].items():
print(f" {layer_name}:")
print(f" 范围: [{layer_stats['min']:.3f}, {layer_stats['max']:.3f}]")
print(f" 均值: {layer_stats['mean']:.3f}")
print(f" 标准差: {layer_stats['std']:.3f}")
# 比较模型大小(概念性)
original_params = sum(p.numel() for p in model.parameters())
# 量化后参数数量相同,但每个参数使用的位数减少
print(f"\n模型压缩效果:")
print(f"原始模型参数量: {original_params:,}")
print(f"量化后理论压缩比: 32bit/8bit = 4x")
print(f"实际内存节省: ~75%")
# 测试推理
test_input = torch.randn(1, 128)
model.eval()
quantized_model.eval()
with torch.no_grad():
original_output = model(test_input)
quantized_output = quantized_model(test_input)
# 计算误差
error = torch.mean(torch.abs(original_output - quantized_output))
relative_error = error / torch.mean(torch.abs(original_output))
print(f"\n量化误差:")
print(f"绝对误差: {error.item():.6f}")
print(f"相对误差: {relative_error.item():.2%}")
if __name__ == "__main__":
demonstrate_model_quantization()
13.4 部署监控与性能分析
13.4.1 性能监控框架
class DeploymentPerformanceMonitor:
"""部署性能监控器"""
def __init__(self, monitoring_config: Dict[str, Any]):
self.config = monitoring_config
self.metrics_history = []
self.alert_thresholds = monitoring_config.get('alert_thresholds', {})
self.alerts = []
self.is_monitoring = False
def start_monitoring(self, deployment_instance):
"""开始监控"""
self.is_monitoring = True
monitoring_task = asyncio.create_task(self._monitoring_loop(deployment_instance))
return monitoring_task
async def _monitoring_loop(self, deployment_instance):
"""监控循环"""
while self.is_monitoring:
try:
# 收集指标
metrics = await self._collect_metrics(deployment_instance)
# 存储指标
self.metrics_history.append({
'timestamp': time.time(),
'metrics': metrics
})
# 检查告警
alerts = self._check_alerts(metrics)
if alerts:
self.alerts.extend(alerts)
await self._handle_alerts(alerts)
# 清理旧数据
self._cleanup_old_metrics()
# 等待下一个监控周期
await asyncio.sleep(self.config.get('monitoring_interval', 10))
except Exception as e:
print(f"Monitoring error: {str(e)}")
await asyncio.sleep(self.config.get('monitoring_interval', 10))
async def _collect_metrics(self, deployment_instance) -> Dict[str, Any]:
"""收集指标"""
metrics = {}
# 系统指标
metrics['system'] = self._collect_system_metrics()
# 模型指标
metrics['model'] = await self._collect_model_metrics(deployment_instance)
# 业务指标
metrics['business'] = self._collect_business_metrics(deployment_instance)
return metrics
def _collect_system_metrics(self) -> Dict[str, Any]:
"""收集系统指标"""
import psutil
# CPU使用率
cpu_percent = psutil.cpu_percent(interval=1)
# 内存使用
memory = psutil.virtual_memory()
# 磁盘使用
disk = psutil.disk_usage('/')
# GPU使用(如果可用)
gpu_metrics = {}
if torch.cuda.is_available():
gpu_metrics = {
'gpu_memory_used': torch.cuda.memory_allocated(),
'gpu_memory_reserved': torch.cuda.memory_reserved(),
'gpu_utilization': self._get_gpu_utilization()
}
return {
'cpu_percent': cpu_percent,
'memory_percent': memory.percent,
'memory_available_gb': memory.available / (1024**3),
'disk_percent': disk.percent,
'disk_free_gb': disk.free / (1024**3),
**gpu_metrics
}
async def _collect_model_metrics(self, deployment_instance) -> Dict[str, Any]:
"""收集模型指标"""
metrics = {}
# 请求指标
if hasattr(deployment_instance, 'metrics'):
for metric_name, metric in deployment_instance.metrics.items():
if hasattr(metric, '_value') and hasattr(metric._value, 'get'):
metrics[metric_name] = metric._value.get()
# 缓存指标
if hasattr(deployment_instance, 'optimizer') and hasattr(deployment_instance.optimizer, 'cache_stats'):
metrics['cache_stats'] = deployment_instance.optimizer.cache_stats
# 批处理指标
if hasattr(deployment_instance, 'batcher'):
metrics['batch_stats'] = deployment_instance.batcher.get_statistics()
return metrics
def _collect_business_metrics(self, deployment_instance) -> Dict[str, Any]:
"""收集业务指标"""
# 这里应该根据具体业务需求实现
return {
'request_rate': self._calculate_request_rate(),
'error_rate': self._calculate_error_rate(),
'average_latency': self._calculate_average_latency(),
'throughput': self._calculate_throughput()
}
def _get_gpu_utilization(self) -> float:
"""获取GPU利用率"""
try:
import nvidia_ml_py3 as nvml
nvml.nvmlInit()
handle = nvml.nvmlDeviceGetHandleByIndex(0)
utilization = nvml.nvmlDeviceGetUtilizationRates(handle)
return utilization.gpu
except:
return 0.0
def _calculate_request_rate(self) -> float:
"""计算请求速率"""
if len(self.metrics_history) < 2:
return 0.0
recent_metrics = self.metrics_history[-10:]
if len(recent_metrics) < 2:
return 0.0
time_diff = recent_metrics[-1]['timestamp'] - recent_metrics[0]['timestamp']
if time_diff == 0:
return 0.0
# 这里需要根据实际的请求计数计算
return 1.0 # 简化处理
def _calculate_error_rate(self) -> float:
"""计算错误率"""
# 简化处理,实际需要根据错误计数计算
return 0.01
def _calculate_average_latency(self) -> float:
"""计算平均延迟"""
if len(self.metrics_history) == 0:
return 0.0
recent_latencies = []
for entry in self.metrics_history[-10:]:
if 'model' in entry['metrics'] and 'request_duration' in entry['metrics']['model']:
recent_latencies.append(entry['metrics']['model']['request_duration'])
return np.mean(recent_latencies) if recent_latencies else 0.0
def _calculate_throughput(self) -> float:
"""计算吞吐量"""
# 简化处理,实际需要根据处理的token数量计算
return 100.0
def _check_alerts(self, metrics: Dict[str, Any]) -> List[Dict[str, Any]]:
"""检查告警"""
alerts = []
# 系统资源告警
if 'system' in metrics:
system_metrics = metrics['system']
# CPU使用率告警
if system_metrics.get('cpu_percent', 0) > self.alert_thresholds.get('cpu_threshold', 80):
alerts.append({
'type': 'high_cpu_usage',
'severity': 'warning',
'message': f"High CPU usage: {system_metrics['cpu_percent']:.1f}%",
'timestamp': time.time()
})
# 内存使用率告警
if system_metrics.get('memory_percent', 0) > self.alert_thresholds.get('memory_threshold', 85):
alerts.append({
'type': 'high_memory_usage',
'severity': 'critical',
'message': f"High memory usage: {system_metrics['memory_percent']:.1f}%",
'timestamp': time.time()
})
# GPU内存告警
if 'gpu_memory_used' in system_metrics:
gpu_memory_percent = (system_metrics['gpu_memory_used'] /
system_metrics.get('gpu_memory_total', 1)) * 100
if gpu_memory_percent > self.alert_thresholds.get('gpu_memory_threshold', 90):
alerts.append({
'type': 'high_gpu_memory',
'severity': 'critical',
'message': f"High GPU memory usage: {gpu_memory_percent:.1f}%",
'timestamp': time.time()
})
# 模型性能告警
if 'model' in metrics:
model_metrics = metrics['model']
# 高延迟告警
if 'request_duration' in model_metrics:
if model_metrics['request_duration'] > self.alert_thresholds.get('latency_threshold', 1.0):
alerts.append({
'type': 'high_latency',
'severity': 'warning',
'message': f"High request latency: {model_metrics['request_duration']:.3f}s",
'timestamp': time.time()
})
# 缓存命中率告警
if 'cache_stats' in model_metrics:
cache_stats = model_metrics['cache_stats']
total_cache_ops = cache_stats['hits'] + cache_stats['misses']
if total_cache_ops > 0:
hit_rate = cache_stats['hits'] / total_cache_ops
if hit_rate < self.alert_thresholds.get('cache_hit_rate_threshold', 0.8):
alerts.append({
'type': 'low_cache_hit_rate',
'severity': 'warning',
'message': f"Low cache hit rate: {hit_rate:.1%}",
'timestamp': time.time()
})
return alerts
async def _handle_alerts(self, alerts: List[Dict[str, Any]]):
"""处理告警"""
for alert in alerts:
severity = alert['severity']
message = alert['message']
if severity == 'critical':
print(f"🚨 CRITICAL ALERT: {message}")
# 这里可以发送紧急通知,如短信、电话等
elif severity == 'warning':
print(f"⚠️ WARNING ALERT: {message}")
# 这里可以发送普通通知,如邮件、Slack等
else:
print(f"ℹ️ INFO ALERT: {message}")
def _cleanup_old_metrics(self):
"""清理旧指标"""
max_history = self.config.get('max_metrics_history', 1000)
if len(self.metrics_history) > max_history:
self.metrics_history = self.metrics_history[-max_history:]
def get_performance_report(self) -> Dict[str, Any]:
"""获取性能报告"""
if not self.metrics_history:
return {'error': 'No metrics data available'}
recent_metrics = self.metrics_history[-100:] # 最近100个数据点
# 计算统计信息
report = {
'period': {
'start': recent_metrics[0]['timestamp'],
'end': recent_metrics[-1]['timestamp'],
'duration_minutes': (recent_metrics[-1]['timestamp'] - recent_metrics[0]['timestamp']) / 60
},
'system_performance': self._analyze_system_performance(recent_metrics),
'model_performance': self._analyze_model_performance(recent_metrics),
'business_metrics': self._analyze_business_metrics(recent_metrics),
'alerts_summary': self._summarize_alerts(),
'recommendations': self._generate_recommendations(recent_metrics)
}
return report
def _analyze_system_performance(self, metrics_history: List[Dict[str, Any]]) -> Dict[str, Any]:
"""分析系统性能"""
cpu_values = []
memory_values = []
gpu_memory_values = []
for entry in metrics_history:
if 'system' in entry['metrics']:
system_metrics = entry['metrics']['system']
cpu_values.append(system_metrics.get('cpu_percent', 0))
memory_values.append(system_metrics.get('memory_percent', 0))
if 'gpu_memory_used' in system_metrics and 'gpu_memory_total' in system_metrics:
gpu_memory_percent = (system_metrics['gpu_memory_used'] /
system_metrics['gpu_memory_total']) * 100
gpu_memory_values.append(gpu_memory_percent)
analysis = {
'cpu_usage': {
'avg': np.mean(cpu_values) if cpu_values else 0,
'max': np.max(cpu_values) if cpu_values else 0,
'min': np.min(cpu_values) if cpu_values else 0,
'status': 'normal' if np.mean(cpu_values) < 70 else 'high'
},
'memory_usage': {
'avg': np.mean(memory_values) if memory_values else 0,
'max': np.max(memory_values) if memory_values else 0,
'status': 'normal' if np.mean(memory_values) < 80 else 'high'
}
}
if gpu_memory_values:
analysis['gpu_memory_usage'] = {
'avg': np.mean(gpu_memory_values),
'max': np.max(gpu_memory_values),
'status': 'normal' if np.mean(gpu_memory_values) < 85 else 'high'
}
return analysis
def _analyze_model_performance(self, metrics_history: List[Dict[str, Any]]) -> Dict[str, Any]:
"""分析模型性能"""
latency_values = []
cache_hit_rates = []
for entry in metrics_history:
if 'model' in entry['metrics']:
model_metrics = entry['metrics']['model']
if 'request_duration' in model_metrics:
latency_values.append(model_metrics['request_duration'])
if 'cache_stats' in model_metrics:
cache_stats = model_metrics['cache_stats']
total_ops = cache_stats['hits'] + cache_stats['misses']
if total_ops > 0:
cache_hit_rates.append(cache_stats['hits'] / total_ops)
analysis = {}
if latency_values:
analysis['latency'] = {
'avg': np.mean(latency_values),
'p50': np.percentile(latency_values, 50),
'p95': np.percentile(latency_values, 95),
'p99': np.percentile(latency_values, 99),
'status': 'good' if np.mean(latency_values) < 0.5 else 'slow'
}
if cache_hit_rates:
analysis['cache_performance'] = {
'avg_hit_rate': np.mean(cache_hit_rates),
'min_hit_rate': np.min(cache_hit_rates),
'status': 'good' if np.mean(cache_hit_rates) > 0.8 else 'poor'
}
return analysis
def _analyze_business_metrics(self, metrics_history: List[Dict[str, Any]]) -> Dict[str, Any]:
"""分析业务指标"""
# 简化处理,实际需要更复杂的业务逻辑
return {
'request_rate': self._calculate_request_rate(),
'error_rate': self._calculate_error_rate(),
'throughput': self._calculate_throughput(),
'availability': 99.9 # 假设的可用性
}
def _summarize_alerts(self) -> Dict[str, Any]:
"""总结告警"""
if not self.alerts:
return {'total_alerts': 0, 'by_severity': {}, 'by_type': {}}
recent_alerts = [
alert for alert in self.alerts
if time.time() - alert['timestamp'] < 3600 # 最近1小时
]
by_severity = {}
by_type = {}
for alert in recent_alerts:
severity = alert['severity']
alert_type = alert['type']
by_severity[severity] = by_severity.get(severity, 0) + 1
by_type[alert_type] = by_type.get(alert_type, 0) + 1
return {
'total_alerts': len(recent_alerts),
'by_severity': by_severity,
'by_type': by_type
}
def _generate_recommendations(self, metrics_history: List[Dict[str, Any]]) -> List[str]:
"""生成建议"""
recommendations = []
# 基于系统性能的建议
system_analysis = self._analyze_system_performance(metrics_history)
if system_analysis['cpu_usage']['status'] == 'high':
recommendations.append("CPU使用率较高,建议优化模型推理或增加计算资源")
if system_analysis['memory_usage']['status'] == 'high':
recommendations.append("内存使用率较高,建议检查内存泄漏或增加内存")
# 基于模型性能的建议
model_analysis = self._analyze_model_performance(metrics_history)
if 'latency' in model_analysis and model_analysis['latency']['status'] == 'slow':
recommendations.append("模型延迟较高,建议优化批处理或启用缓存")
if 'cache_performance' in model_analysis and model_analysis['cache_performance']['status'] == 'poor':
recommendations.append("缓存命中率较低,建议优化缓存策略或增加缓存容量")
# 通用建议
recommendations.extend([
"定期审查监控阈值和告警规则",
"建立性能基线并跟踪趋势变化",
"准备容量规划以应对负载增长"
])
return recommendations
# 使用示例
async def demonstrate_deployment_monitoring():
"""演示部署监控"""
print("=== 部署监控与性能分析演示 ===")
# 创建监控配置
monitoring_config = {
'monitoring_interval': 5, # 5秒监控间隔
'max_metrics_history': 1000,
'alert_thresholds': {
'cpu_threshold': 75,
'memory_threshold': 80,
'gpu_memory_threshold': 85,
'latency_threshold': 0.5,
'cache_hit_rate_threshold': 0.75
}
}
# 创建性能监控器
monitor = DeploymentPerformanceMonitor(monitoring_config)
# 模拟部署实例
class MockDeployment:
def __init__(self):
self.metrics = {
'request_count': type('Metric', (), {'_value': type('Value', (), {'get': lambda: 100})()}),
'request_duration': type('Metric', (), {'_value': type('Value', (), {'get': lambda: 0.3})()}),
'active_requests': type('Metric', (), {'_value': type('Value', (), {'get': lambda: 5})()}),
}
self.optimizer = type('Optimizer', (), {'cache_stats': {'hits': 80, 'misses': 20}})()
self.batcher = type('Batcher', (), {'get_statistics': lambda: {'avg_batch_size': 4.5}})()
mock_deployment = MockDeployment()
# 开始监控(运行较短时间用于演示)
print("开始监控(演示模式,运行30秒)...")
monitoring_task = monitor.start_monitoring(mock_deployment)
# 运行监控一段时间
await asyncio.sleep(30)
# 停止监控
monitor.is_monitoring = False
await monitoring_task
# 生成性能报告
report = monitor.get_performance_report()
print(f"\n=== 性能报告 ===")
print(f"监控期间: {report['period']['duration_minutes']:.1f} 分钟")
print(f"\n系统性能:")
cpu_analysis = report['system_performance']['cpu_usage']
print(f" CPU使用率 - 平均: {cpu_analysis['avg']:.1f}%, 最大: {cpu_analysis['max']:.1f}%")
print(f" 状态: {cpu_analysis['status']}")
memory_analysis = report['system_performance']['memory_usage']
print(f" 内存使用率 - 平均: {memory_analysis['avg']:.1f}%, 最大: {memory_analysis['max']:.1f}%")
print(f" 状态: {memory_analysis['status']}")
print(f"\n模型性能:")
if 'latency' in report['model_performance']:
latency_analysis = report['model_performance']['latency']
print(f" 延迟 - 平均: {latency_analysis['avg']:.3f}s, P95: {latency_analysis['p95']:.3f}s")
print(f" 状态: {latency_analysis['status']}")
if 'cache_performance' in report['model_performance']:
cache_analysis = report['model_performance']['cache_performance']
print(f" 缓存命中率 - 平均: {cache_analysis['avg_hit_rate']:.1%}")
print(f" 状态: {cache_analysis['status']}")
print(f"\n业务指标:")
business_metrics = report['business_metrics']
print(f" 请求速率: {business_metrics['request_rate']:.1f} req/s")
print(f" 错误率: {business_metrics['error_rate']:.2%}")
print(f" 吞吐量: {business_metrics['throughput']:.1f} tokens/s")
print(f" 可用性: {business_metrics['availability']:.1f}%")
print(f"\n告警摘要:")
alerts_summary = report['alerts_summary']
print(f" 最近1小时告警数: {alerts_summary['total_alerts']}")
if alerts_summary['by_severity']:
print(f" 按严重程度: {alerts_summary['by_severity']}")
print(f"\n建议:")
for i, recommendation in enumerate(report['recommendations'], 1):
print(f" {i}. {recommendation}")
if __name__ == "__main__":
asyncio.run(demonstrate_deployment_monitoring())
更多推荐
所有评论(0)