Merge branch 'codex/fix-running-task-timeout-from-current'
This commit is contained in:
@@ -19,6 +19,7 @@ from app.services.task_service import (
|
||||
has_running_task,
|
||||
list_tasks,
|
||||
recover_running_tasks,
|
||||
recover_stale_running_tasks,
|
||||
)
|
||||
from app.templating import templates
|
||||
|
||||
@@ -83,6 +84,7 @@ def create_task_api(
|
||||
request: CreateTaskRequest,
|
||||
session: Session = Depends(get_db_session),
|
||||
) -> CreateTaskResponse:
|
||||
recover_stale_running_tasks(session)
|
||||
if has_running_task(session):
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=RUNNING_TASK_MESSAGE)
|
||||
|
||||
|
||||
@@ -87,6 +87,12 @@ class TikHubClient:
|
||||
)
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
if response.status_code == 401:
|
||||
raise PlatformAPIError(
|
||||
"TikHub 鉴权失败,请检查 TIKHUB_API_KEY 是否有效",
|
||||
error_type="auth_error",
|
||||
status_code=response.status_code,
|
||||
)
|
||||
if response.is_error:
|
||||
raise PlatformAPIError(
|
||||
f"External API returned HTTP {response.status_code}",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
@@ -121,12 +122,14 @@ def analyze_comments_with_retry(
|
||||
*,
|
||||
requester,
|
||||
max_retries: int = 3,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> list[AIAnalysisResult]:
|
||||
expected_ids = {str(comment["comment_id"]) for comment in comments}
|
||||
prompt = build_comment_prompt(comments)
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return parse_ai_comment_response(requester(prompt), expected_comment_ids=expected_ids)
|
||||
response_text = _request_with_timeout(requester, prompt, timeout_seconds=timeout_seconds)
|
||||
return parse_ai_comment_response(response_text, expected_comment_ids=expected_ids)
|
||||
except Exception:
|
||||
if attempt >= max_retries - 1:
|
||||
break
|
||||
@@ -143,6 +146,21 @@ def analyze_comments_with_retry(
|
||||
]
|
||||
|
||||
|
||||
def _request_with_timeout(requester, prompt: str, *, timeout_seconds: float | None) -> str:
|
||||
if timeout_seconds is None or timeout_seconds <= 0:
|
||||
return requester(prompt)
|
||||
|
||||
executor = ThreadPoolExecutor(max_workers=1)
|
||||
future = executor.submit(requester, prompt)
|
||||
try:
|
||||
return future.result(timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
future.cancel()
|
||||
raise TimeoutError("AI request timed out") from exc
|
||||
finally:
|
||||
executor.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
|
||||
class OpenAICompatibleAIClient:
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -21,6 +21,7 @@ from app.services.report_service import generate_hotspot_report, generate_item_r
|
||||
RUNNING_TASK_MESSAGE = "当前有正在运行的任务,请稍后再试"
|
||||
RESTART_ERROR_MESSAGE = "系统重启,任务被中断"
|
||||
STALE_PROGRESS_THRESHOLD_SECONDS = 10 * 60
|
||||
STALE_PROGRESS_ERROR_TYPE = "stale_progress_timeout"
|
||||
task_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
|
||||
@@ -44,6 +45,30 @@ def has_running_task(session: Session) -> bool:
|
||||
return session.scalar(select(Task.id).where(Task.status == "running").limit(1)) is not None
|
||||
|
||||
|
||||
def recover_stale_running_tasks(session: Session) -> int:
|
||||
now = ensure_utc_datetime(utc_now())
|
||||
tasks = list(session.scalars(select(Task).where(Task.status == "running")))
|
||||
recovered = 0
|
||||
for task in tasks:
|
||||
last_progress_at = ensure_utc_datetime(task.last_progress_at or task.started_at or task.created_at)
|
||||
if last_progress_at is None or now is None:
|
||||
continue
|
||||
seconds_since_progress = max(0, int((now - last_progress_at).total_seconds()))
|
||||
if seconds_since_progress < STALE_PROGRESS_THRESHOLD_SECONDS:
|
||||
continue
|
||||
_mark_task_failed(
|
||||
task,
|
||||
"system",
|
||||
STALE_PROGRESS_ERROR_TYPE,
|
||||
f"任务超过 {STALE_PROGRESS_THRESHOLD_SECONDS // 60} 分钟没有进度更新,已自动标记失败",
|
||||
)
|
||||
task.finished_at = task.finished_at or utc_now()
|
||||
recovered += 1
|
||||
if recovered:
|
||||
session.commit()
|
||||
return recovered
|
||||
|
||||
|
||||
def recover_running_tasks(session: Session) -> int:
|
||||
tasks = list(session.scalars(select(Task).where(Task.status == "running")))
|
||||
for task in tasks:
|
||||
@@ -256,6 +281,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
task = session.get(Task, task_id)
|
||||
if task is None:
|
||||
return
|
||||
task.started_at = task.started_at or utc_now()
|
||||
session.commit()
|
||||
try:
|
||||
platform = build_platform(task.platform)
|
||||
ai_requester, report_summary_provider = build_ai_dependencies()
|
||||
@@ -345,6 +372,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
|
||||
requester=ai_requester,
|
||||
max_retries=get_settings().ai_max_retries,
|
||||
timeout_seconds=get_settings().ai_timeout_seconds,
|
||||
)
|
||||
results_by_comment_id = {result.comment_id: result for result in ai_results}
|
||||
for comment in item_comments:
|
||||
@@ -390,9 +418,11 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
update_task_stage(task, "success")
|
||||
else:
|
||||
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
|
||||
task.finished_at = task.finished_at or utc_now()
|
||||
session.commit()
|
||||
except Exception as exc:
|
||||
_mark_task_failed(task, "system", "unexpected_error", str(exc))
|
||||
task.finished_at = task.finished_at or utc_now()
|
||||
session.commit()
|
||||
finally:
|
||||
if close_session:
|
||||
|
||||
Reference in New Issue
Block a user