feat: AI并行处理 + 修复邮件丢失
- _ai_processor 改用 ThreadPoolExecutor(5 workers) 并行调用AI - _tg_worker 每轮排空所有待发TG消息(原来每轮只取1条) - fetch_unseen_emails 单封FETCH失败不再丢弃整个批次,跳过继续
This commit is contained in:
58
main.py
58
main.py
@@ -5,6 +5,7 @@ import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from src.config import load_config
|
||||
from src.summarizer import poll_accounts, ai_process, tg_send_and_mark
|
||||
from src.tg_bot import poll_telegram
|
||||
@@ -64,21 +65,33 @@ def _email_poller(cfg):
|
||||
|
||||
|
||||
def _ai_processor(cfg):
|
||||
logger.info("AI 处理线程已启动")
|
||||
while _running:
|
||||
try:
|
||||
acct_idx, mail = _email_queue.get(timeout=1)
|
||||
except queue_module.Empty:
|
||||
continue
|
||||
try:
|
||||
info = ai_process(cfg, acct_idx, mail)
|
||||
if info:
|
||||
_tg_queue.put(info)
|
||||
except Exception as e:
|
||||
logger.error(f"AI 处理邮件失败: {e}", exc_info=True)
|
||||
finally:
|
||||
with _pending_lock:
|
||||
_pending_uids.discard(mail.uid)
|
||||
logger.info("AI 处理线程已启动 (并行 workers=5)")
|
||||
max_workers = 5
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
pending_futures = {}
|
||||
while _running:
|
||||
while len(pending_futures) < max_workers:
|
||||
try:
|
||||
acct_idx, mail = _email_queue.get_nowait()
|
||||
except queue_module.Empty:
|
||||
break
|
||||
future = executor.submit(ai_process, cfg, acct_idx, mail)
|
||||
pending_futures[future] = (acct_idx, mail)
|
||||
|
||||
completed = [f for f in pending_futures if f.done()]
|
||||
for future in completed:
|
||||
acct_idx, mail = pending_futures.pop(future)
|
||||
try:
|
||||
info = future.result()
|
||||
if info:
|
||||
_tg_queue.put(info)
|
||||
except Exception as e:
|
||||
logger.error(f"AI 处理邮件失败: {e}", exc_info=True)
|
||||
finally:
|
||||
with _pending_lock:
|
||||
_pending_uids.discard(mail.uid)
|
||||
|
||||
time.sleep(0.1 if pending_futures else 0.5)
|
||||
|
||||
|
||||
def _tg_worker(cfg):
|
||||
@@ -90,13 +103,14 @@ def _tg_worker(cfg):
|
||||
except Exception as e:
|
||||
logger.error(f"TG 轮询错误: {e}", exc_info=True)
|
||||
|
||||
try:
|
||||
info = _tg_queue.get_nowait()
|
||||
tg_send_and_mark(cfg, info)
|
||||
except queue_module.Empty:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"TG 发送失败: {e}", exc_info=True)
|
||||
while _running:
|
||||
try:
|
||||
info = _tg_queue.get_nowait()
|
||||
tg_send_and_mark(cfg, info)
|
||||
except queue_module.Empty:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"TG 发送失败: {e}", exc_info=True)
|
||||
|
||||
if _running:
|
||||
time.sleep(1)
|
||||
|
||||
Reference in New Issue
Block a user