english-to-chinese/code/translate_epub_v5(多线程版本).py

import os
import re
from bs4 import BeautifulSoup
import openai
import time
from tqdm import tqdm
import sqlite3
import json
from datetime import datetime
import logging
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
import asyncio
import aiohttp
from concurrent.futures import ThreadPoolExecutor
from functools import lru_cache
import hashlib
import yaml
from pathlib import Path
import multiprocessing
from multiprocessing import Pool, Manager, Lock
import queue

# 配置管理
class Config:
    def __init__(self, config_path='config.yaml'):
        self.config_path = config_path
        self.config = self.load_config()
        
        # 设置日志
        self.setup_logging()
        
        # 初始化OpenAI客户端
        self.setup_openai()
    
    def load_config(self):
        """加载配置文件"""
        if not os.path.exists(self.config_path):
            # 创建默认配置
            default_config = {
                'logging': {
                    'level': 'INFO',
                    'format': '%(asctime)s - %(levelname)s - %(message)s',
                    'file': 'translation.log'
                },
                'openai': {
                    'base_url': 'https://api.siliconflow.cn/v1',
                    'api_key': 'sk-',
                    'model_name': 'deepseek-ai/DeepSeek-R1',
                    'max_retries': 3,
                    'retry_delay': 2,
                    'timeout': 30,
                    'max_concurrent_requests': 5
                },
                'translation': {
                    'min_line_count': 1,
                    'max_line_count': 5,
                    'initial_line_count': 2,
                    'error_threshold': 3,
                    'success_threshold': 5,
                    'error_cooldown': 60,
                    'cache_size': 1000
                },
                'database': {
                    'path': 'translation_progress.db',
                    'pool_size': 5
                },
                'paths': {
                    'input_dir': '002/Ops',
                    'output_dir': '002/Ops_translated'
                }
            }
            
            # 保存默认配置
            with open(self.config_path, 'w', encoding='utf-8') as f:
                yaml.dump(default_config, f, allow_unicode=True)
            
            return default_config
        
        # 加载现有配置
        with open(self.config_path, 'r', encoding='utf-8') as f:
            return yaml.safe_load(f)
    
    def setup_logging(self):
        """设置日志"""
        logging.basicConfig(
            level=getattr(logging, self.config['logging']['level']),
            format=self.config['logging']['format'],
            handlers=[
                logging.FileHandler(self.config['logging']['file']),
                logging.StreamHandler()
            ]
        )
    
    def setup_openai(self):
        """设置OpenAI客户端"""
        self.client = openai.OpenAI(
            base_url=self.config['openai']['base_url'],
            api_key=self.config['openai']['api_key']
        )
    
    def get(self, *keys):
        """获取配置值"""
        value = self.config
        for key in keys:
            value = value[key]
        return value
    
    def update(self, updates):
        """更新配置"""
        def deep_update(d, u):
            for k, v in u.items():
                if isinstance(v, dict):
                    d[k] = deep_update(d.get(k, {}), v)
                else:
                    d[k] = v
            return d
        
        self.config = deep_update(self.config, updates)
        
        # 保存更新后的配置
        with open(self.config_path, 'w', encoding='utf-8') as f:
            yaml.dump(self.config, f, allow_unicode=True)
        
        # 重新设置日志和OpenAI客户端
        self.setup_logging()
        self.setup_openai()

# 创建全局的配置实例
config = Config()

# 更新全局变量
MODEL_CONFIG = {
    "model_name": config.get('openai', 'model_name'),
    "max_retries": config.get('openai', 'max_retries'),
    "retry_delay": config.get('openai', 'retry_delay'),
    "timeout": config.get('openai', 'timeout'),
    "max_concurrent_requests": config.get('openai', 'max_concurrent_requests'),
    "cache_size": config.get('translation', 'cache_size')
}

MIN_LINE_COUNT = config.get('translation', 'min_line_count')
MAX_LINE_COUNT = config.get('translation', 'max_line_count')
INITIAL_LINE_COUNT = config.get('translation', 'initial_line_count')
ERROR_THRESHOLD = config.get('translation', 'error_threshold')
SUCCESS_THRESHOLD = config.get('translation', 'success_threshold')

# 更新其他类的初始化参数
class LineCountManager:
    def __init__(self):
        self.current_line_count = INITIAL_LINE_COUNT
        self.consecutive_errors = 0
        self.consecutive_successes = 0
        self.last_error_time = None
        self.error_cooldown = config.get('translation', 'error_cooldown')
        self.version = f"1.0.{INITIAL_LINE_COUNT}"
        self.error_history = []
    
    def adjust_line_count(self, success):
        """根据翻译结果调整行数"""
        current_time = time.time()
        
        # 检查是否在冷却期内
        if self.last_error_time and (current_time - self.last_error_time) < self.error_cooldown:
            return self.current_line_count
        
        if success:
            self.consecutive_errors = 0
            self.consecutive_successes = 0  # 重置成功计数，但不增加行数
        else:
            self.consecutive_successes = 0
            self.consecutive_errors += 1
            self.last_error_time = current_time
            
            # 记录错误
            self.error_history.append({
                'time': current_time,
                'line_count': self.current_line_count
            })
            
            # 如果连续错误次数达到阈值，减少行数
            if self.consecutive_errors >= ERROR_THRESHOLD:
                if self.current_line_count > MIN_LINE_COUNT:
                    self.current_line_count -= 1
                    self.consecutive_errors = 0
                    self.version = f"1.0.{self.current_line_count}"
                    logging.warning(f"翻译连续失败，减少行数到 {self.current_line_count}，版本更新为 {self.version}")
        
        return self.current_line_count
    
    def get_error_stats(self):
        """获取错误统计信息"""
        if not self.error_history:
            return "无错误记录"
        
        recent_errors = [e for e in self.error_history if time.time() - e['time'] < 3600]  # 最近一小时的错误
        return {
            "总错误数": len(self.error_history),
            "最近一小时错误数": len(recent_errors),
            "当前行数": self.current_line_count,
            "连续错误": self.consecutive_errors,
            "连续成功": self.consecutive_successes
        }

class DatabaseManager:
    def __init__(self):
        self.db_path = config.get('database', 'path')
        self.conn = None
        self.init_db()
    
    def get_connection(self):
        """获取数据库连接"""
        if self.conn is None:
            self.conn = sqlite3.connect(self.db_path)
            self.conn.row_factory = sqlite3.Row
        return self.conn
    
    def close(self):
        """关闭数据库连接"""
        if self.conn:
            self.conn.close()
            self.conn = None
    
    def init_db(self):
        """初始化数据库"""
        conn = self.get_connection()
        c = conn.cursor()
        
        # 创建文件进度表
        c.execute('''
            CREATE TABLE IF NOT EXISTS file_progress (
                file_path TEXT PRIMARY KEY,
                total_lines INTEGER,
                processed_lines INTEGER,
                status TEXT,
                version TEXT,
                last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                error_count INTEGER DEFAULT 0,
                retry_count INTEGER DEFAULT 0
            )
        ''')
        
        # 创建翻译组进度表
        c.execute('''
            CREATE TABLE IF NOT EXISTS group_progress (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                file_path TEXT,
                group_index INTEGER,
                original_text TEXT,
                translated_text TEXT,
                status TEXT,
                version TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                error_count INTEGER DEFAULT 0,
                retry_count INTEGER DEFAULT 0,
                UNIQUE(file_path, group_index, version)
            )
        ''')
        
        # 创建错误日志表
        c.execute('''
            CREATE TABLE IF NOT EXISTS error_log (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                file_path TEXT,
                group_index INTEGER,
                error_type TEXT,
                error_message TEXT,
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                resolved_at TIMESTAMP,
                resolution TEXT
            )
        ''')
        
        conn.commit()
    
    def begin_transaction(self):
        """开始事务"""
        self.get_connection().execute('BEGIN TRANSACTION')
    
    def commit_transaction(self):
        """提交事务"""
        self.get_connection().commit()
    
    def rollback_transaction(self):
        """回滚事务"""
        self.get_connection().rollback()
    
    def get_file_progress(self, file_path):
        """获取文件翻译进度"""
        c = self.get_connection().cursor()
        c.execute('SELECT * FROM file_progress WHERE file_path = ?', (file_path,))
        return c.fetchone()
    
    def update_file_progress(self, file_path, total_lines, processed_lines, status):
        """更新文件翻译进度"""
        c = self.get_connection().cursor()
        c.execute('''
            INSERT OR REPLACE INTO file_progress 
            (file_path, total_lines, processed_lines, status, version, last_updated)
            VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
        ''', (file_path, total_lines, processed_lines, status, line_count_manager.version))
        self.get_connection().commit()
    
    def get_group_progress(self, file_path, group_index):
        """获取翻译组进度"""
        c = self.get_connection().cursor()
        c.execute('''
            SELECT * FROM group_progress 
            WHERE file_path = ? AND group_index = ? AND version = ?
        ''', (file_path, group_index, line_count_manager.version))
        return c.fetchone()
    
    def update_group_progress(self, file_path, group_index, original_text, translated_text, status):
        """更新翻译组进度"""
        c = self.get_connection().cursor()
        c.execute('''
            INSERT OR REPLACE INTO group_progress 
            (file_path, group_index, original_text, translated_text, status, version, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
        ''', (file_path, group_index, original_text, translated_text, status, line_count_manager.version))
        self.get_connection().commit()
    
    def log_error(self, file_path, group_index, error_type, error_message):
        """记录错误"""
        c = self.get_connection().cursor()
        c.execute('''
            INSERT INTO error_log 
            (file_path, group_index, error_type, error_message)
            VALUES (?, ?, ?, ?)
        ''', (file_path, group_index, error_type, error_message))
        self.get_connection().commit()
    
    def get_error_stats(self):
        """获取错误统计信息"""
        c = self.get_connection().cursor()
        c.execute('''
            SELECT 
                COUNT(*) as total_errors,
                COUNT(CASE WHEN resolved_at IS NULL THEN 1 END) as unresolved_errors,
                COUNT(CASE WHEN created_at > datetime('now', '-1 hour') THEN 1 END) as recent_errors
            FROM error_log
        ''')
        return c.fetchone()

class AsyncTranslationManager:
    def __init__(self):
        self.semaphore = asyncio.Semaphore(config.get('openai', 'max_concurrent_requests'))
        self.session = None

class TranslationCache:
    def __init__(self):
        self.cache = {}
        self.max_size = config.get('translation', 'cache_size')
        self.hits = 0
        self.misses = 0

# 创建全局实例
line_count_manager = LineCountManager()
db_manager = DatabaseManager()
async_translation_manager = AsyncTranslationManager()
translation_cache = TranslationCache()

# 添加版本控制
VERSION = "1.0.1" # 版本号，用于区分不同版本的翻译
line_count = 2 # 每组行数，越大越快，但越容易出错

class TranslationStats:
    def __init__(self):
        self.start_time = time.time()
        self.total_chars = 0
        self.translated_chars = 0
        self.total_requests = 0
        self.successful_requests = 0
        self.failed_requests = 0
    
    def update_stats(self, original_text, translated_text, success=True):
        self.total_chars += len(original_text)
        self.translated_chars += len(translated_text)
        self.total_requests += 1
        if success:
            self.successful_requests += 1
        else:
            self.failed_requests += 1
    
    def get_stats(self):
        elapsed_time = time.time() - self.start_time
        chars_per_second = self.translated_chars / elapsed_time if elapsed_time > 0 else 0
        success_rate = (self.successful_requests / self.total_requests * 100) if self.total_requests > 0 else 0
        
        return {
            "总字符数": self.total_chars,
            "已翻译字符数": self.translated_chars,
            "翻译速度": f"{chars_per_second:.2f} 字符/秒",
            "成功率": f"{success_rate:.1f}%",
            "总请求数": self.total_requests,
            "成功请求": self.successful_requests,
            "失败请求": self.failed_requests,
            "运行时间": f"{elapsed_time:.1f} 秒"
        }
    
    def to_dict(self):
        """返回可序列化的字典"""
        return {
            "total_chars": self.total_chars,
            "translated_chars": self.translated_chars,
            "total_requests": self.total_requests,
            "successful_requests": self.successful_requests,
            "failed_requests": self.failed_requests,
            "elapsed_time": time.time() - self.start_time
        }
    
    @classmethod
    def from_dict(cls, data):
        """从字典创建实例"""
        stats = cls()
        stats.total_chars = data.get("total_chars", 0)
        stats.translated_chars = data.get("translated_chars", 0)
        stats.total_requests = data.get("total_requests", 0)
        stats.successful_requests = data.get("successful_requests", 0)
        stats.failed_requests = data.get("failed_requests", 0)
        stats.start_time = time.time() - data.get("elapsed_time", 0)
        return stats

# 创建全局的统计对象
translation_stats = TranslationStats()

def get_completed_groups(conn, file_path):
    """获取已完成的翻译组"""
    c = conn.cursor()
    c.execute('''
        SELECT group_index, translated_text 
        FROM group_progress 
        WHERE file_path = ? AND status = 'completed' AND version = ?
        ORDER BY group_index
    ''', (file_path, line_count_manager.version))
    return c.fetchall()

# """ - 输出内容要求用代码块包裹起来
# ，只在必要时提供相应的语言注释
#  """
@retry(
    stop=stop_after_attempt(MODEL_CONFIG['max_retries']),
    wait=wait_exponential(multiplier=1, min=4, max=10),
    retry=retry_if_exception_type((openai.APIError, openai.APITimeoutError)),
    before_sleep=lambda retry_state: logging.warning(f"重试第 {retry_state.attempt_number} 次...")
)
def translate_text(text, file_name=None, context=None):
    """翻译文本，使用流式输出"""
    try:
        # 构建上下文信息
        context_info = ""
        if file_name:
            context_info += f"当前正在翻译文件：{file_name}\n"
        if context:
            context_info += f"上下文信息：{context}\n"
        
        messages = [
            {
                "role": "system",
                "content": f"""你是一个专业的翻译助手，专注于将文本翻译成中文。
- 你正在翻译一个完整的文档，请保持翻译风格的一致性。
- 翻译时要考虑上下文，确保翻译的连贯性。
- 对于专业术语，请保持统一的翻译。
- 保持原文的格式和结构。
- 不要添加任何额外的解释或说明。
- 只返回翻译后的内容，不要包含任何其他内容。

{context_info}"""
            },
            {
                "role": "user",
                "content": text
            }
        ]
        
        # 使用流式输出
        stream = config.client.chat.completions.create(
            model=MODEL_CONFIG['model_name'],
            messages=messages,
            timeout=MODEL_CONFIG['timeout'],
            stream=True  # 启用流式输出
        )
        
        # 收集流式输出的内容
        translated_text = ""
        for chunk in stream:
            if chunk.choices[0].delta.content is not None:
                content = chunk.choices[0].delta.content
                translated_text += content
                # 实时打印翻译内容
                print(content, end='', flush=True)
        
        print()  # 换行
        
        # 更新统计信息
        if hasattr(process_files_batch, 'process_stats'):
            process_files_batch.process_stats.update_stats(text, translated_text, True)
        
        return translated_text
        
    except Exception as e:
        logging.error(f"翻译出错: {str(e)}")
        
        # 更新统计信息
        if hasattr(process_files_batch, 'process_stats'):
            process_files_batch.process_stats.update_stats(text, "", False)
        
        raise

def process_html_file(file_path, conn):
    """处理HTML文件"""
    # 检查文件进度
    progress = db_manager.get_file_progress(file_path)
    
    try:
        # 尝试不同的编码方式读取文件
        encodings = ['utf-8', 'gbk', 'gb2312', 'latin1']
        content = None
        
        for encoding in encodings:
            try:
                with open(file_path, 'r', encoding=encoding) as f:
                    content = f.read()
                break
            except UnicodeDecodeError:
                continue
        
        if content is None:
            raise Exception(f"无法使用支持的编码读取文件: {file_path}")
    
        # 使用正则表达式提取body标签内的内容
        body_pattern = re.compile(r'<body[^>]*>(.*?)</body>', re.DOTALL)
        body_match = body_pattern.search(content)
        
        if not body_match:
            print(f"警告: {file_path} 中没有找到body标签")
            return
        
        body_content = body_match.group(1)
        
        # 按行分割内容，保留所有HTML标签行，但只翻译包含 <p class 的行
        lines = []
        for line in body_content.split('\n'):
            line = line.strip()
            if line and line.startswith('<'):
                lines.append(line)
        
        total_lines = len(lines)
        
        # 获取已完成的翻译组
        completed_groups = get_completed_groups(conn, file_path)
        completed_indices = {group[0] for group in completed_groups}
        
        # 计算已处理的进度
        if progress:
            print(f"文件 {file_path} 已处理进度: {progress[2]}/{progress[1]} 行 ({round(progress[2]*100/progress[1], 2)}%)")
        
        # 按组处理内容
        translated_lines = []
        try:
            with tqdm(range(0, len(lines), line_count_manager.current_line_count), 
                     desc=f"处理文件 {os.path.basename(file_path)}", 
                     unit="组") as pbar:
                for i in pbar:
                    group_index = i // line_count_manager.current_line_count
                    
                    # 检查是否已完成
                    if group_index in completed_indices:
                        # 使用已完成的翻译
                        for group in completed_groups:
                            if group[0] == group_index:
                                translated_lines.extend(group[1].split('\n'))
                                break
                        continue
                    
                    group = lines[i:i+line_count_manager.current_line_count]
                    if group:
                        # 保存原始文本
                        original_text = "\n".join(group)
                        
                        # 收集需要翻译的段落
                        paragraphs_to_translate = []
                        paragraph_indices = []
                        for idx, line in enumerate(group):
                            if '<p class' in line or line.startswith('<h'):
                                paragraphs_to_translate.append(line)
                                paragraph_indices.append(idx)
                        
                        # 如果有需要翻译的段落，进行翻译
                        if paragraphs_to_translate:
                            translated_paragraphs = []
                            for paragraph in paragraphs_to_translate:
                                print(f"\n翻译段落 {len(translated_paragraphs) + 1}/{len(paragraphs_to_translate)}:")
                                translated_paragraph = translate_text(paragraph)
                                translated_paragraphs.append(translated_paragraph)
                            
                            # 将翻译后的段落放回原位置
                            translated_group = group.copy()
                            for idx, translated in zip(paragraph_indices, translated_paragraphs):
                                translated_group[idx] = translated
                        else:
                            translated_group = group
                        
                        translated_text = "\n".join(translated_group)
                        
                        # 更新翻译组进度
                        db_manager.update_group_progress(file_path, group_index, original_text, translated_text, 'completed')
                        
                        # 分割翻译后的文本
                        translated_lines.extend(translated_group)
                        
                        # 更新文件进度
                        processed_lines = min((group_index + 1) * line_count_manager.current_line_count, total_lines)
                        db_manager.update_file_progress(file_path, total_lines, processed_lines, 'in_progress')
                        
                        # 显示当前统计信息
                        stats = translation_stats.get_stats()
                        pbar.set_postfix(stats)
                        
                        # 添加较小的延迟以避免API限制
                        time.sleep(0.1)  # 减少延迟时间
            
            # 替换原始内容
            if translated_lines:
                # 构建新的body内容
                new_body_content = []
                current_index = 0
                
                # 遍历原始内容，替换需要翻译的部分
                for line in body_content.split('\n'):
                    line = line.strip()
                    if not line:
                        new_body_content.append('')
                        continue
                        
                    if line.startswith('<'):
                        if ('<p class' in line or line.startswith('<h')) and current_index < len(translated_lines):
                            # 替换翻译后的内容
                            new_body_content.append(translated_lines[current_index])
                            current_index += 1
                        else:
                            # 保持原样
                            new_body_content.append(line)
                    else:
                        # 保持非HTML内容原样
                        new_body_content.append(line)
                
                # 将新内容重新组合
                new_body_content = '\n'.join(new_body_content)
                
                # 替换原始内容中的body部分
                new_content = content.replace(body_content, new_body_content)
                
                # 保存修改后的文件
                output_dir = config.get('paths', 'output_dir')
                os.makedirs(output_dir, exist_ok=True)
                output_path = os.path.join(output_dir, os.path.basename(file_path))
                
                with open(output_path, 'w', encoding='utf-8') as f:
                    f.write(new_content)
                
                # 更新完成状态
                db_manager.update_file_progress(file_path, total_lines, total_lines, 'completed')
                print(f"文件 {file_path} 翻译完成，已保存到 {output_path}")
                
                # 显示最终统计信息
                print("\n翻译统计信息:")
                for key, value in translation_stats.get_stats().items():
                    print(f"{key}: {value}")
                
        except KeyboardInterrupt:
            print("\n检测到中断，保存当前进度...")
            if 'processed_lines' in locals():
                db_manager.update_file_progress(file_path, total_lines, processed_lines, 'interrupted')
            # 显示中断时的统计信息
            print("\n中断时的统计信息:")
            for key, value in translation_stats.get_stats().items():
                print(f"{key}: {value}")
            raise
        except Exception as e:
            print(f"处理文件时出错: {str(e)}")
            if 'processed_lines' in locals():
                db_manager.update_file_progress(file_path, total_lines, processed_lines, 'error')
            raise
            
    except Exception as e:
        print(f"读取文件时出错: {str(e)}")
        return

def process_files_batch(file_batch, process_id):
    """处理一批文件的函数，用于多进程执行"""
    try:
        # 为每个进程创建独立的数据库连接
        process_db = DatabaseManager()
        conn = process_db.get_connection()
        
        # 创建进程级别的统计对象
        process_stats = TranslationStats()
        
        # 创建文件锁
        file_lock = Lock()
        
        def translate_with_stats(text, file_name, context=None):
            """包装翻译函数以收集统计信息"""
            with file_lock:  # 使用文件锁确保同一时间只有一个翻译请求
                return translate_text(text, file_name, context)
        
        for filename in tqdm(file_batch, desc=f"进程 {process_id} 处理文件", unit="文件"):
            file_path = os.path.join(config.get('paths', 'input_dir'), filename)
            
            try:
                # 尝试不同的编码方式读取文件
                encodings = ['utf-8', 'gbk', 'gb2312', 'latin1']
                content = None
                
                for encoding in encodings:
                    try:
                        with open(file_path, 'r', encoding=encoding) as f:
                            content = f.read()
                        break
                    except UnicodeDecodeError:
                        continue
                
                if content is None:
                    raise Exception(f"无法使用支持的编码读取文件: {file_path}")
            
                # 使用正则表达式提取body标签内的内容
                body_pattern = re.compile(r'<body[^>]*>(.*?)</body>', re.DOTALL)
                body_match = body_pattern.search(content)
                
                if not body_match:
                    print(f"警告: {file_path} 中没有找到body标签")
                    continue
                
                body_content = body_match.group(1)
                
                # 按行分割内容，保留所有HTML标签行，但只翻译包含 <p class 的行
                lines = []
                for line in body_content.split('\n'):
                    line = line.strip()
                    if line and line.startswith('<'):
                        lines.append(line)
                
                total_lines = len(lines)
                
                # 获取已完成的翻译组
                completed_groups = get_completed_groups(conn, file_path)
                completed_indices = {group[0] for group in completed_groups}
                
                # 计算已处理的进度
                progress = db_manager.get_file_progress(file_path)
                if progress:
                    print(f"文件 {file_path} 已处理进度: {progress[2]}/{progress[1]} 行 ({round(progress[2]*100/progress[1], 2)}%)")
                
                # 按组处理内容
                translated_lines = []
                with tqdm(range(0, len(lines), line_count_manager.current_line_count), 
                         desc=f"处理文件 {os.path.basename(file_path)}", 
                         unit="组") as pbar:
                    for i in pbar:
                        group_index = i // line_count_manager.current_line_count
                        
                        # 检查是否已完成
                        if group_index in completed_indices:
                            # 使用已完成的翻译
                            for group in completed_groups:
                                if group[0] == group_index:
                                    translated_lines.extend(group[1].split('\n'))
                                    break
                            continue
                        
                        group = lines[i:i+line_count_manager.current_line_count]
                        if group:
                            # 保存原始文本
                            original_text = "\n".join(group)
                            
                            # 收集需要翻译的段落
                            paragraphs_to_translate = []
                            paragraph_indices = []
                            for idx, line in enumerate(group):
                                if '<p class' in line or line.startswith('<h'):
                                    paragraphs_to_translate.append(line)
                                    paragraph_indices.append(idx)
                            
                            # 如果有需要翻译的段落，进行翻译
                            if paragraphs_to_translate:
                                translated_paragraphs = []
                                for paragraph in paragraphs_to_translate:
                                    print(f"\n翻译段落 {len(translated_paragraphs) + 1}/{len(paragraphs_to_translate)}:")
                                    # 获取上下文（前一段和后一段）
                                    context = ""
                                    if len(translated_lines) > 0:
                                        context += f"前文：{translated_lines[-1]}\n"
                                    if i + line_count_manager.current_line_count < len(lines):
                                        context += f"后文：{lines[i + line_count_manager.current_line_count]}"
                                    
                                    translated_paragraph = translate_with_stats(
                                        paragraph, 
                                        filename,
                                        context
                                    )
                                    translated_paragraphs.append(translated_paragraph)
                                
                                # 将翻译后的段落放回原位置
                                translated_group = group.copy()
                                for idx, translated in zip(paragraph_indices, translated_paragraphs):
                                    translated_group[idx] = translated
                            else:
                                translated_group = group
                            
                            translated_text = "\n".join(translated_group)
                            
                            # 更新翻译组进度
                            db_manager.update_group_progress(file_path, group_index, original_text, translated_text, 'completed')
                            
                            # 分割翻译后的文本
                            translated_lines.extend(translated_group)
                            
                            # 更新文件进度
                            processed_lines = min((group_index + 1) * line_count_manager.current_line_count, total_lines)
                            db_manager.update_file_progress(file_path, total_lines, processed_lines, 'in_progress')
                            
                            # 显示当前统计信息
                            stats = process_stats.get_stats()
                            pbar.set_postfix(stats)
                            
                            # 添加较小的延迟以避免API限制
                            time.sleep(0.1)  # 减少延迟时间
                
                # 替换原始内容
                if translated_lines:
                    # 构建新的body内容
                    new_body_content = []
                    current_index = 0
                    
                    # 遍历原始内容，替换需要翻译的部分
                    for line in body_content.split('\n'):
                        line = line.strip()
                        if not line:
                            new_body_content.append('')
                            continue
                            
                        if line.startswith('<'):
                            if ('<p class' in line or line.startswith('<h')) and current_index < len(translated_lines):
                                # 替换翻译后的内容
                                new_body_content.append(translated_lines[current_index])
                                current_index += 1
                            else:
                                # 保持原样
                                new_body_content.append(line)
                        else:
                            # 保持非HTML内容原样
                            new_body_content.append(line)
                    
                    # 将新内容重新组合
                    new_body_content = '\n'.join(new_body_content)
                    
                    # 替换原始内容中的body部分
                    new_content = content.replace(body_content, new_body_content)
                    
                    # 保存修改后的文件
                    output_dir = config.get('paths', 'output_dir')
                    os.makedirs(output_dir, exist_ok=True)
                    output_path = os.path.join(output_dir, os.path.basename(file_path))
                    
                    with open(output_path, 'w', encoding='utf-8') as f:
                        f.write(new_content)
                    
                    # 更新完成状态
                    db_manager.update_file_progress(file_path, total_lines, total_lines, 'completed')
                    print(f"文件 {file_path} 翻译完成，已保存到 {output_path}")
                    
                    # 显示最终统计信息
                    print("\n翻译统计信息:")
                    for key, value in process_stats.get_stats().items():
                        print(f"{key}: {value}")
                    
            except Exception as e:
                print(f"处理文件时出错: {str(e)}")
                if 'processed_lines' in locals():
                    db_manager.update_file_progress(file_path, total_lines, processed_lines, 'error')
                raise
        
        # 返回进程的统计信息
        return process_stats.to_dict()
            
    except Exception as e:
        logging.error(f"进程 {process_id} 发生错误: {str(e)}")
        return None
    finally:
        if 'process_db' in locals():
            process_db.close()

def aggregate_stats(stats_list):
    """聚合所有进程的统计信息"""
    aggregated_stats = {
        "total_chars": 0,
        "translated_chars": 0,
        "total_requests": 0,
        "successful_requests": 0,
        "failed_requests": 0,
        "elapsed_time": 0
    }
    
    for stats in stats_list:
        if not stats:
            continue
            
        for key in aggregated_stats:
            if key in stats:
                if key == "elapsed_time":
                    aggregated_stats[key] = max(aggregated_stats[key], stats[key])
                else:
                    aggregated_stats[key] += stats[key]
    
    # 创建统计对象并格式化输出
    final_stats = TranslationStats.from_dict(aggregated_stats)
    return final_stats.get_stats()

def main():
    # 设置进程数
    num_processes = 2
    
    # 获取输入目录中的所有HTML文件
    input_dir = config.get('paths', 'input_dir')
    html_files = [f for f in os.listdir(input_dir) if f.endswith('.html')]
    
    print(f"找到 {len(html_files)} 个HTML文件需要处理")
    print(f"开始时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    
    # 调整进程数，确保不超过文件数量
    num_processes = min(num_processes, len(html_files))
    
    # 将文件列表分成多个批次
    if num_processes > 0:
        batch_size = max(1, len(html_files) // num_processes)
        file_batches = [html_files[i:i + batch_size] for i in range(0, len(html_files), batch_size)]
    else:
        print("没有找到需要处理的HTML文件")
        return
    
    try:
        # 创建进程池
        with Pool(processes=num_processes) as pool:
            # 启动多个进程处理文件并收集结果
            results = []
            for i, batch in enumerate(file_batches):
                result = pool.apply_async(process_files_batch, args=(batch, i+1))
                results.append(result)
            
            # 等待所有进程完成并收集结果
            stats_list = [r.get() for r in results]
            
    except Exception as e:
        logging.error(f"进程池执行出错: {str(e)}")
        stats_list = []
    
    # 聚合统计信息
    final_stats = aggregate_stats(stats_list)
    
    print(f"\n结束时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print("\n最终统计信息:")
    for key, value in final_stats.items():
        print(f"{key}: {value}")

if __name__ == "__main__":
    # 设置多进程启动方法
    multiprocessing.set_start_method('spawn')
    # 设置日志格式
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s'
    )
    main()