435 lines
18 KiB
Python
435 lines
18 KiB
Python
import inspect
|
||
import json
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from collections.abc import Callable
|
||
from datetime import UTC, datetime
|
||
|
||
from sqlalchemy import func, select
|
||
from sqlalchemy.orm import Session, sessionmaker
|
||
|
||
from app.config import get_settings
|
||
from app.db import SessionLocal
|
||
from app.models import Comment, ContentItem, Hotspot, Report, Task, utc_now
|
||
from app.platforms.base import PlatformAPIError, TikHubClient
|
||
from app.platforms.douyin import DouyinPlatform
|
||
from app.platforms.xiaohongshu import XiaohongshuPlatform
|
||
from app.schemas import CreateTaskRequest
|
||
from app.services.ai_service import OpenAICompatibleAIClient, analyze_comments_with_retry, build_report_summary_prompt, calculate_analysis_status
|
||
from app.services.report_service import generate_hotspot_report, generate_item_report
|
||
|
||
|
||
RUNNING_TASK_MESSAGE = "当前有正在运行的任务，请稍后再试"
|
||
RESTART_ERROR_MESSAGE = "系统重启，任务被中断"
|
||
STALE_PROGRESS_THRESHOLD_SECONDS = 10 * 60
|
||
STALE_PROGRESS_ERROR_TYPE = "stale_progress_timeout"
|
||
task_executor = ThreadPoolExecutor(max_workers=1)
|
||
|
||
|
||
STAGE_LABELS = {
|
||
"queued": "等待启动",
|
||
"crawl_hotspots": "获取热点中",
|
||
"search_items": "搜索内容中",
|
||
"crawl_comments": "抓取评论中",
|
||
"ai_analysis": "AI 分析中",
|
||
"generate_reports": "生成报告中",
|
||
"success": "已完成",
|
||
"failed": "失败",
|
||
}
|
||
PLATFORM_LABELS = {
|
||
"xiaohongshu": "小红书",
|
||
"douyin": "抖音",
|
||
}
|
||
|
||
|
||
def has_running_task(session: Session) -> bool:
|
||
return session.scalar(select(Task.id).where(Task.status == "running").limit(1)) is not None
|
||
|
||
|
||
def recover_stale_running_tasks(session: Session) -> int:
|
||
now = ensure_utc_datetime(utc_now())
|
||
tasks = list(session.scalars(select(Task).where(Task.status == "running")))
|
||
recovered = 0
|
||
for task in tasks:
|
||
last_progress_at = ensure_utc_datetime(task.last_progress_at or task.started_at or task.created_at)
|
||
if last_progress_at is None or now is None:
|
||
continue
|
||
seconds_since_progress = max(0, int((now - last_progress_at).total_seconds()))
|
||
if seconds_since_progress < STALE_PROGRESS_THRESHOLD_SECONDS:
|
||
continue
|
||
_mark_task_failed(
|
||
task,
|
||
"system",
|
||
STALE_PROGRESS_ERROR_TYPE,
|
||
f"任务超过 {STALE_PROGRESS_THRESHOLD_SECONDS // 60} 分钟没有进度更新，已自动标记失败",
|
||
)
|
||
task.finished_at = task.finished_at or utc_now()
|
||
recovered += 1
|
||
if recovered:
|
||
session.commit()
|
||
return recovered
|
||
|
||
|
||
def recover_running_tasks(session: Session) -> int:
|
||
tasks = list(session.scalars(select(Task).where(Task.status == "running")))
|
||
for task in tasks:
|
||
task.status = "failed"
|
||
task.error_stage = "system"
|
||
task.error_type = "unexpected_restart"
|
||
task.error_message = RESTART_ERROR_MESSAGE
|
||
session.commit()
|
||
return len(tasks)
|
||
|
||
|
||
def build_platform(platform: str):
|
||
settings = get_settings()
|
||
client = TikHubClient(
|
||
base_url=settings.tikhub_base_url,
|
||
api_key=settings.tikhub_api_key,
|
||
timeout_seconds=settings.http_timeout_seconds,
|
||
max_retries=settings.http_max_retries,
|
||
)
|
||
if platform == "xiaohongshu":
|
||
return XiaohongshuPlatform(client)
|
||
if platform == "douyin":
|
||
return DouyinPlatform(client)
|
||
raise ValueError(f"Unsupported platform: {platform}")
|
||
|
||
|
||
def build_ai_client() -> OpenAICompatibleAIClient | None:
|
||
settings = get_settings()
|
||
if settings.ai_base_url and settings.ai_api_key and settings.ai_model:
|
||
return OpenAICompatibleAIClient(
|
||
base_url=settings.ai_base_url,
|
||
api_key=settings.ai_api_key,
|
||
model=settings.ai_model,
|
||
timeout_seconds=settings.ai_timeout_seconds,
|
||
)
|
||
return None
|
||
|
||
|
||
def build_ai_requester(client: OpenAICompatibleAIClient | None = None):
|
||
client = client if client is not None else build_ai_client()
|
||
if client is not None:
|
||
return lambda prompt: client.request(
|
||
prompt,
|
||
system_prompt="只返回 JSON Array，不输出 Markdown 或解释性自然语言。",
|
||
)
|
||
|
||
def fallback_requester(_prompt: str) -> str:
|
||
return "[]"
|
||
|
||
return fallback_requester
|
||
|
||
|
||
def build_report_summary_provider(client: OpenAICompatibleAIClient | None):
|
||
if client is None:
|
||
return None
|
||
|
||
def summarize_report(metrics: dict, typical: dict, *, word_limit: int = 200) -> str:
|
||
return client.request(
|
||
build_report_summary_prompt(metrics, typical, word_limit=word_limit),
|
||
system_prompt="只返回一段中文总结，不使用 Markdown。",
|
||
)
|
||
|
||
return summarize_report
|
||
|
||
|
||
def build_ai_dependencies():
|
||
client = build_ai_client()
|
||
requester = build_ai_requester(client)
|
||
return requester, build_report_summary_provider(client)
|
||
|
||
|
||
def create_task(session: Session, request: CreateTaskRequest, *, submit_background: bool = True) -> Task:
|
||
task = Task(
|
||
platform=request.platform,
|
||
status="running",
|
||
hotspot_limit=request.hotspot_limit,
|
||
item_limit_per_hotspot=request.item_limit_per_hotspot,
|
||
comment_limit_per_item=request.comment_limit_per_item,
|
||
total_items_count=0,
|
||
processed_items_count=0,
|
||
successful_items_count=0,
|
||
failed_items_count=0,
|
||
analysis_success_rate=0.0,
|
||
analysis_status="normal",
|
||
current_stage="queued",
|
||
last_progress_at=utc_now(),
|
||
)
|
||
session.add(task)
|
||
session.commit()
|
||
session.refresh(task)
|
||
if submit_background:
|
||
task_executor.submit(run_task, task.id)
|
||
return task
|
||
|
||
|
||
def list_tasks(session: Session) -> list[Task]:
|
||
tasks = list(session.scalars(select(Task).order_by(Task.created_at.desc())))
|
||
for task in tasks:
|
||
hydrate_task_progress(session, task)
|
||
return tasks
|
||
|
||
|
||
def get_task(session: Session, task_id: str) -> Task | None:
|
||
task = session.get(Task, task_id)
|
||
if task is not None:
|
||
hydrate_task_progress(session, task)
|
||
return task
|
||
|
||
|
||
def hydrate_task_progress(session: Session, task: Task) -> Task:
|
||
display_number = calculate_task_display_number(session, task)
|
||
task.display_number = display_number
|
||
task.display_id = str(display_number)
|
||
task.created_at_label = format_datetime_minute(task.created_at)
|
||
task.scale_label = f"{task.hotspot_limit}热点 × {task.item_limit_per_hotspot}内容 × {task.comment_limit_per_item}评论"
|
||
task.error_summary = build_task_error_summary(task)
|
||
task.hotspots_count = session.scalar(select(func.count(Hotspot.id)).where(Hotspot.task_id == task.id)) or 0
|
||
task.comments_count = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||
task.reports_count = session.scalar(select(func.count(Report.id)).where(Report.task_id == task.id)) or 0
|
||
task.is_demo = task.id.startswith("demo-")
|
||
task.last_progress_at = task.last_progress_at or task.created_at
|
||
stage_code = task.current_stage
|
||
if not stage_code and task.status == "success":
|
||
stage_code = "success"
|
||
elif not stage_code and task.status == "failed":
|
||
stage_code = "failed"
|
||
elif not stage_code and task.status == "running":
|
||
stage_code = "queued"
|
||
task.current_stage_label = STAGE_LABELS.get(stage_code or "", stage_code or "等待启动")
|
||
now = ensure_utc_datetime(utc_now())
|
||
running_start = ensure_utc_datetime(task.started_at or task.created_at)
|
||
running_end = ensure_utc_datetime(task.finished_at) or now
|
||
last_progress_at = ensure_utc_datetime(task.last_progress_at)
|
||
running_seconds = max(0, int((running_end - running_start).total_seconds()))
|
||
seconds_since_progress = max(0, int((now - last_progress_at).total_seconds())) if last_progress_at else None
|
||
task.running_seconds = running_seconds
|
||
task.seconds_since_last_progress = seconds_since_progress
|
||
task.running_duration_label = format_duration_zh(running_seconds)
|
||
task.last_progress_ago_label = f"{format_duration_zh(seconds_since_progress)}前" if seconds_since_progress is not None else "暂无记录"
|
||
task.stale_threshold_minutes = STALE_PROGRESS_THRESHOLD_SECONDS // 60
|
||
task.is_progress_stale = (
|
||
task.status == "running"
|
||
and seconds_since_progress is not None
|
||
and seconds_since_progress >= STALE_PROGRESS_THRESHOLD_SECONDS
|
||
)
|
||
return task
|
||
|
||
|
||
def calculate_task_display_number(session: Session, task: Task) -> int:
|
||
task_ids = list(session.scalars(select(Task.id).order_by(Task.created_at, Task.id)))
|
||
try:
|
||
return task_ids.index(task.id) + 1
|
||
except ValueError:
|
||
return len(task_ids) + 1
|
||
|
||
|
||
def format_datetime_minute(value: datetime | None) -> str:
|
||
value = ensure_utc_datetime(value)
|
||
return value.strftime("%Y-%m-%d %H:%M") if value else ""
|
||
|
||
|
||
def build_task_error_summary(task: Task) -> str | None:
|
||
if task.error_message:
|
||
return task.error_message
|
||
if task.error_stage == "system" and task.error_type == "unexpected_restart":
|
||
return RESTART_ERROR_MESSAGE
|
||
if task.error_stage or task.error_type:
|
||
return "任务失败，请查看后端日志"
|
||
return None
|
||
|
||
|
||
def format_duration_zh(total_seconds: int | None) -> str:
|
||
if total_seconds is None:
|
||
return "暂无记录"
|
||
total_seconds = max(0, int(total_seconds))
|
||
minutes = total_seconds // 60
|
||
hours = minutes // 60
|
||
days = hours // 24
|
||
if days > 0:
|
||
remaining_hours = hours % 24
|
||
return f"{days}天{remaining_hours}小时" if remaining_hours else f"{days}天"
|
||
if hours > 0:
|
||
remaining_minutes = minutes % 60
|
||
return f"{hours}小时{remaining_minutes}分钟" if remaining_minutes else f"{hours}小时"
|
||
if minutes > 0:
|
||
return f"{minutes}分钟"
|
||
return "刚刚"
|
||
|
||
|
||
def ensure_utc_datetime(value: datetime | None) -> datetime | None:
|
||
if value is None:
|
||
return None
|
||
if value.tzinfo is None:
|
||
return value.replace(tzinfo=UTC)
|
||
return value.astimezone(UTC)
|
||
|
||
|
||
def update_task_stage(task: Task, stage: str) -> None:
|
||
task.current_stage = stage
|
||
task.last_progress_at = utc_now()
|
||
|
||
|
||
def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionmaker = SessionLocal) -> None:
|
||
session = session_factory()
|
||
close_session = hasattr(session, "close")
|
||
try:
|
||
task = session.get(Task, task_id)
|
||
if task is None:
|
||
return
|
||
task.started_at = task.started_at or utc_now()
|
||
session.commit()
|
||
try:
|
||
platform = build_platform(task.platform)
|
||
ai_requester, report_summary_provider = build_ai_dependencies()
|
||
|
||
try:
|
||
update_task_stage(task, "crawl_hotspots")
|
||
session.commit()
|
||
hotspots = platform.fetch_hotspots(limit=task.hotspot_limit)
|
||
except PlatformAPIError as exc:
|
||
_mark_task_failed(task, "crawl_hotspots", exc.error_type, str(exc))
|
||
session.commit()
|
||
return
|
||
except Exception as exc:
|
||
_mark_task_failed(task, "crawl_hotspots", "api_error", str(exc))
|
||
session.commit()
|
||
return
|
||
|
||
for hotspot_data in hotspots:
|
||
hotspot = Hotspot(
|
||
task_id=task.id,
|
||
platform=task.platform,
|
||
source_hot_id=hotspot_data.source_hot_id,
|
||
rank=hotspot_data.rank,
|
||
title=hotspot_data.title,
|
||
heat_value=hotspot_data.heat_value,
|
||
raw_data=json.dumps(hotspot_data.raw_data or {}, ensure_ascii=False),
|
||
)
|
||
session.add(hotspot)
|
||
session.flush()
|
||
|
||
try:
|
||
update_task_stage(task, "search_items")
|
||
session.commit()
|
||
items = platform.search_items_by_hotspot(hotspot.title, limit=task.item_limit_per_hotspot)
|
||
except Exception as exc:
|
||
task.failed_items_count += task.item_limit_per_hotspot
|
||
task.error_stage = "crawl_items"
|
||
task.error_type = getattr(exc, "error_type", "api_error")
|
||
task.error_message = str(exc)
|
||
session.commit()
|
||
continue
|
||
|
||
task.total_items_count += len(items)
|
||
seen_item_ids: set[str] = set()
|
||
for item_data in items:
|
||
if item_data.source_item_id in seen_item_ids:
|
||
continue
|
||
seen_item_ids.add(item_data.source_item_id)
|
||
item = ContentItem(
|
||
task_id=task.id,
|
||
hotspot_id=hotspot.id,
|
||
platform=task.platform,
|
||
source_item_id=item_data.source_item_id,
|
||
item_type=item_data.item_type,
|
||
title=item_data.title,
|
||
summary=item_data.summary,
|
||
url=item_data.url,
|
||
status="pending",
|
||
raw_data=json.dumps(item_data.raw_data or {}, ensure_ascii=False),
|
||
)
|
||
session.add(item)
|
||
session.flush()
|
||
try:
|
||
update_task_stage(task, "crawl_comments")
|
||
session.commit()
|
||
comments = platform.fetch_comments(item.source_item_id, limit=task.comment_limit_per_item)
|
||
for comment_data in comments:
|
||
session.add(
|
||
Comment(
|
||
task_id=task.id,
|
||
hotspot_id=hotspot.id,
|
||
content_item_id=item.id,
|
||
platform=task.platform,
|
||
source_comment_id=comment_data.source_comment_id,
|
||
content=comment_data.content,
|
||
author=comment_data.author,
|
||
like_count=comment_data.like_count,
|
||
comment_time=comment_data.comment_time,
|
||
raw_data=json.dumps(comment_data.raw_data or {}, ensure_ascii=False),
|
||
)
|
||
)
|
||
session.flush()
|
||
item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||
update_task_stage(task, "ai_analysis")
|
||
session.commit()
|
||
ai_results = analyze_comments_with_retry(
|
||
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
|
||
requester=ai_requester,
|
||
max_retries=get_settings().ai_max_retries,
|
||
timeout_seconds=get_settings().ai_timeout_seconds,
|
||
)
|
||
results_by_comment_id = {result.comment_id: result for result in ai_results}
|
||
for comment in item_comments:
|
||
result = results_by_comment_id.get(comment.id)
|
||
if result is None:
|
||
comment.sentiment = "unknown"
|
||
comment.labels = "[]"
|
||
comment.reason = "ai_missing_result"
|
||
comment.ai_analysis_status = "failed"
|
||
continue
|
||
comment.sentiment = result.sentiment
|
||
comment.labels = json.dumps(result.labels, ensure_ascii=False)
|
||
comment.reason = result.reason
|
||
comment.ai_analysis_status = result.ai_analysis_status
|
||
item.status = "success"
|
||
task.successful_items_count += 1
|
||
update_task_stage(task, "generate_reports")
|
||
generate_item_report(session, item.id, summary_provider=report_summary_provider)
|
||
except Exception as exc:
|
||
item.status = "failed"
|
||
item.error_stage = "crawl_comments"
|
||
item.error_type = getattr(exc, "error_type", "api_error")
|
||
item.error_message = str(exc)
|
||
task.failed_items_count += 1
|
||
task.error_stage = item.error_stage
|
||
task.error_type = item.error_type
|
||
task.error_message = item.error_message
|
||
finally:
|
||
task.processed_items_count += 1
|
||
session.commit()
|
||
|
||
if task.successful_items_count > 0:
|
||
update_task_stage(task, "generate_reports")
|
||
total_comments = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||
success_comments = sum(1 for comment in task.comments if comment.ai_analysis_status == "success")
|
||
task.analysis_success_rate, task.analysis_status = calculate_analysis_status(
|
||
success_count=success_comments,
|
||
total_count=total_comments,
|
||
)
|
||
for hotspot in task.hotspots:
|
||
generate_hotspot_report(session, hotspot.id, summary_provider=report_summary_provider)
|
||
task.status = "success"
|
||
update_task_stage(task, "success")
|
||
else:
|
||
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
|
||
task.finished_at = task.finished_at or utc_now()
|
||
session.commit()
|
||
except Exception as exc:
|
||
_mark_task_failed(task, "system", "unexpected_error", str(exc))
|
||
task.finished_at = task.finished_at or utc_now()
|
||
session.commit()
|
||
finally:
|
||
if close_session:
|
||
session.close()
|
||
|
||
|
||
def _mark_task_failed(task: Task, stage: str, error_type: str, message: str) -> None:
|
||
task.status = "failed"
|
||
update_task_stage(task, "failed")
|
||
task.error_stage = stage
|
||
task.error_type = error_type
|
||
task.error_message = message
|