feat: 提交热榜评论分析工具 MVP 基线
This commit is contained in:
meijiali
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Service modules for task execution, crawling, AI analysis, reports, and export."""
|
||||
@@ -0,0 +1,140 @@
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
|
||||
Sentiment = Literal["positive", "negative", "neutral", "unknown"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AIAnalysisResult:
|
||||
comment_id: str
|
||||
sentiment: str
|
||||
labels: list[str]
|
||||
reason: str = ""
|
||||
ai_analysis_status: str = "success"
|
||||
|
||||
|
||||
class AIAnalysisItem(BaseModel):
|
||||
comment_id: str
|
||||
sentiment: Sentiment
|
||||
labels: list[str] = Field(default_factory=list, max_length=3)
|
||||
reason: str = ""
|
||||
|
||||
|
||||
def build_comment_prompt(comments: list[dict[str, str]]) -> str:
|
||||
payload = [
|
||||
{
|
||||
"comment_id": str(comment["comment_id"]),
|
||||
"content": str(comment.get("content", ""))[:150],
|
||||
}
|
||||
for comment in comments
|
||||
]
|
||||
return (
|
||||
"只返回 JSON Array，不输出 Markdown 或解释性自然语言。"
|
||||
"请原样回填输入中的 comment_id，不得修改或生成新 ID。"
|
||||
"sentiment 只能是 positive、negative、neutral、unknown；labels 最多 3 个简短中文短语。\n"
|
||||
f"{json.dumps(payload, ensure_ascii=False)}"
|
||||
)
|
||||
|
||||
|
||||
def parse_ai_comment_response(response_text: str, *, expected_comment_ids: set[str]) -> list[AIAnalysisResult]:
|
||||
try:
|
||||
raw = json.loads(response_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError("AI response is not valid JSON") from exc
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("AI response must be a JSON Array")
|
||||
|
||||
results = []
|
||||
for item in raw:
|
||||
try:
|
||||
parsed = AIAnalysisItem.model_validate(item)
|
||||
except ValidationError as exc:
|
||||
raise ValueError("AI response item does not match schema") from exc
|
||||
if parsed.comment_id not in expected_comment_ids:
|
||||
raise ValueError("AI response comment_id does not match input")
|
||||
results.append(
|
||||
AIAnalysisResult(
|
||||
comment_id=parsed.comment_id,
|
||||
sentiment=parsed.sentiment,
|
||||
labels=parsed.labels,
|
||||
reason=parsed.reason,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def calculate_analysis_status(*, success_count: int, total_count: int) -> tuple[float, str]:
|
||||
if total_count <= 0:
|
||||
return 0.0, "insufficient"
|
||||
rate = success_count / total_count
|
||||
return rate, "normal" if rate >= 0.8 else "insufficient"
|
||||
|
||||
|
||||
def analyze_comments_with_retry(
|
||||
comments: list[dict[str, str]],
|
||||
*,
|
||||
requester,
|
||||
max_retries: int = 3,
|
||||
) -> list[AIAnalysisResult]:
|
||||
expected_ids = {str(comment["comment_id"]) for comment in comments}
|
||||
prompt = build_comment_prompt(comments)
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return parse_ai_comment_response(requester(prompt), expected_comment_ids=expected_ids)
|
||||
except Exception:
|
||||
if attempt >= max_retries - 1:
|
||||
break
|
||||
time.sleep(2**attempt)
|
||||
return [
|
||||
AIAnalysisResult(
|
||||
comment_id=comment_id,
|
||||
sentiment="unknown",
|
||||
labels=[],
|
||||
reason="ai_parse_failed",
|
||||
ai_analysis_status="failed",
|
||||
)
|
||||
for comment_id in expected_ids
|
||||
]
|
||||
|
||||
|
||||
class OpenAICompatibleAIClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
timeout_seconds: int = 30,
|
||||
http_client: httpx.Client | None = None,
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self._http_client = http_client or httpx.Client(timeout=timeout_seconds)
|
||||
|
||||
def request(self, prompt: str) -> str:
|
||||
endpoint = f"{self.base_url}/chat/completions" if self.base_url.endswith("/v1") else f"{self.base_url}/v1/chat/completions"
|
||||
response = self._http_client.post(
|
||||
endpoint,
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "只返回 JSON Array，不输出 Markdown 或解释性自然语言。",
|
||||
},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return str(payload["choices"][0]["message"]["content"])
|
||||
@@ -0,0 +1,104 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Comment, ContentItem, Hotspot, Report
|
||||
|
||||
|
||||
def labels_to_text(labels_json: str | None) -> str:
|
||||
try:
|
||||
labels = json.loads(labels_json or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
return "，".join(str(label) for label in labels) if isinstance(labels, list) else ""
|
||||
|
||||
|
||||
def safe_csv_field(value) -> str:
|
||||
text = "" if value is None else str(value)
|
||||
text = text.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")
|
||||
if text.startswith(("=", "+", "-", "@")):
|
||||
return f"'{text}"
|
||||
return text
|
||||
|
||||
|
||||
def safe_filename(platform: str, task_id: str, hotspot_keyword: str, *, extension: str = "csv") -> str:
|
||||
keyword = hotspot_keyword[:20] + ("..." if len(hotspot_keyword) > 20 else "")
|
||||
name = f"{platform}_{task_id}_{keyword}"
|
||||
name = re.sub(r'[/\\:*?"<>|]+', "_", name)
|
||||
name = re.sub(r"_+", "_", name).strip("_")
|
||||
return f"{name}.{extension}"
|
||||
|
||||
|
||||
def export_item_comments_csv(session: Session, item_id: str) -> tuple[bytes, str]:
|
||||
item = session.get(ContentItem, item_id)
|
||||
if item is None:
|
||||
raise ValueError("content item not found")
|
||||
hotspot = session.get(Hotspot, item.hotspot_id)
|
||||
comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(["平台", "任务 ID", "热点 ID", "热点标题", "内容条目 ID", "内容条目标题", "评论 ID", "评论内容", "情绪倾向", "方向标签", "点赞数", "评论时间"])
|
||||
for comment in comments:
|
||||
writer.writerow(
|
||||
[
|
||||
item.platform,
|
||||
item.task_id,
|
||||
hotspot.id if hotspot else "",
|
||||
hotspot.title if hotspot else "",
|
||||
item.id,
|
||||
item.title or "",
|
||||
comment.source_comment_id or "",
|
||||
safe_csv_field(comment.content),
|
||||
comment.sentiment,
|
||||
labels_to_text(comment.labels),
|
||||
comment.like_count or 0,
|
||||
comment.comment_time or "",
|
||||
]
|
||||
)
|
||||
filename = safe_filename(item.platform, item.task_id, hotspot.title if hotspot else item.title or "comments")
|
||||
return output.getvalue().encode("utf-8-sig"), filename
|
||||
|
||||
|
||||
def export_hotspot_comments_csv(session: Session, hotspot_id: str) -> tuple[bytes, str]:
|
||||
hotspot = session.get(Hotspot, hotspot_id)
|
||||
if hotspot is None:
|
||||
raise ValueError("hotspot not found")
|
||||
comments = list(session.scalars(select(Comment).where(Comment.hotspot_id == hotspot.id)))
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(["平台", "任务 ID", "热点 ID", "热点标题", "内容条目 ID", "内容条目标题", "评论 ID", "评论内容", "情绪倾向", "方向标签", "点赞数", "评论时间"])
|
||||
for comment in comments:
|
||||
item = session.get(ContentItem, comment.content_item_id)
|
||||
writer.writerow(
|
||||
[
|
||||
comment.platform,
|
||||
comment.task_id,
|
||||
hotspot.id,
|
||||
hotspot.title,
|
||||
item.id if item else "",
|
||||
item.title if item else "",
|
||||
comment.source_comment_id or "",
|
||||
safe_csv_field(comment.content),
|
||||
comment.sentiment,
|
||||
labels_to_text(comment.labels),
|
||||
comment.like_count or 0,
|
||||
comment.comment_time or "",
|
||||
]
|
||||
)
|
||||
return output.getvalue().encode("utf-8-sig"), safe_filename(hotspot.platform, hotspot.task_id, hotspot.title)
|
||||
|
||||
|
||||
def export_report_markdown(session: Session, *, report_type: str, hotspot_id: str | None = None, item_id: str | None = None) -> tuple[str, str]:
|
||||
query = select(Report).where(Report.report_type == report_type)
|
||||
if hotspot_id:
|
||||
query = query.where(Report.hotspot_id == hotspot_id)
|
||||
if item_id:
|
||||
query = query.where(Report.content_item_id == item_id)
|
||||
report = session.scalar(query)
|
||||
if report is None:
|
||||
raise ValueError("report not found")
|
||||
return report.markdown_content or report.markdown, safe_filename("report", report.task_id, report.title, extension="md")
|
||||
@@ -0,0 +1,162 @@
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Callable
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Comment, ContentItem, Hotspot, Report
|
||||
|
||||
|
||||
DEFAULT_SUMMARY = "总结生成失败，请查看上方统计数据。"
|
||||
|
||||
|
||||
def _labels(value: str | None) -> list[str]:
|
||||
try:
|
||||
parsed = json.loads(value or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [str(label) for label in parsed] if isinstance(parsed, list) else []
|
||||
|
||||
|
||||
def build_comment_metrics(comments: list[Comment]) -> dict:
|
||||
sentiment_counts = Counter(comment.sentiment or "unknown" for comment in comments)
|
||||
label_counts = Counter(label for comment in comments for label in _labels(comment.labels))
|
||||
total = len(comments)
|
||||
|
||||
def sentiment_entry(name: str) -> dict:
|
||||
count = sentiment_counts.get(name, 0)
|
||||
return {"count": count, "pct": round((count / total * 100) if total else 0, 2)}
|
||||
|
||||
return {
|
||||
"sample_count": total,
|
||||
"sentiment": {
|
||||
"positive": sentiment_entry("positive"),
|
||||
"negative": sentiment_entry("negative"),
|
||||
"neutral": sentiment_entry("neutral"),
|
||||
"unknown": sentiment_entry("unknown"),
|
||||
},
|
||||
"top_labels": [{"name": name, "count": count} for name, count in label_counts.most_common(5)],
|
||||
}
|
||||
|
||||
|
||||
def select_typical_comments(comments: list[Comment], *, per_sentiment: int = 2) -> dict[str, list[dict]]:
|
||||
buckets: dict[str, list[Comment]] = defaultdict(list)
|
||||
for comment in comments:
|
||||
buckets[comment.sentiment or "unknown"].append(comment)
|
||||
|
||||
result = {}
|
||||
for sentiment, values in buckets.items():
|
||||
sorted_values = sorted(values, key=lambda c: (c.like_count or 0, c.comment_time or c.created_at), reverse=True)
|
||||
result[sentiment] = [
|
||||
{"content": comment.content, "like_count": comment.like_count or 0, "comment_id": comment.source_comment_id}
|
||||
for comment in sorted_values[:per_sentiment]
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
def generate_item_report(
|
||||
session: Session,
|
||||
content_item_id: str,
|
||||
*,
|
||||
summary_provider: Callable[[dict, dict], str] | None = None,
|
||||
) -> Report:
|
||||
item = session.get(ContentItem, content_item_id)
|
||||
if item is None:
|
||||
raise ValueError("content item not found")
|
||||
hotspot = session.get(Hotspot, item.hotspot_id)
|
||||
comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
metrics = build_comment_metrics(comments)
|
||||
typical = select_typical_comments(comments)
|
||||
summary = _safe_summary(summary_provider, metrics, typical, limit=200)
|
||||
markdown = _build_markdown(title=item.title or item.source_item_id, metrics=metrics, typical=typical, summary=summary, hotspot_title=hotspot.title if hotspot else "")
|
||||
report = Report(
|
||||
task_id=item.task_id,
|
||||
hotspot_id=item.hotspot_id,
|
||||
content_item_id=item.id,
|
||||
report_type="item",
|
||||
title=item.title or item.source_item_id,
|
||||
data=json.dumps(metrics, ensure_ascii=False),
|
||||
markdown=markdown,
|
||||
metrics_json=json.dumps(metrics, ensure_ascii=False),
|
||||
typical_comments_json=json.dumps(typical, ensure_ascii=False),
|
||||
summary=summary,
|
||||
markdown_content=markdown,
|
||||
)
|
||||
session.add(report)
|
||||
session.commit()
|
||||
session.refresh(report)
|
||||
return report
|
||||
|
||||
|
||||
def generate_hotspot_report(
|
||||
session: Session,
|
||||
hotspot_id: str,
|
||||
*,
|
||||
summary_provider: Callable[[dict, dict], str] | None = None,
|
||||
) -> Report:
|
||||
hotspot = session.get(Hotspot, hotspot_id)
|
||||
if hotspot is None:
|
||||
raise ValueError("hotspot not found")
|
||||
comments = list(session.scalars(select(Comment).where(Comment.hotspot_id == hotspot.id)))
|
||||
item_count = session.scalar(select(func.count(ContentItem.id)).where(ContentItem.hotspot_id == hotspot.id)) or 0
|
||||
metrics = build_comment_metrics(comments)
|
||||
metrics["item_count"] = item_count
|
||||
typical = select_typical_comments(comments)
|
||||
summary = _safe_summary(summary_provider, metrics, typical, limit=300)
|
||||
markdown = _build_markdown(title=hotspot.title, metrics=metrics, typical=typical, summary=summary)
|
||||
report = Report(
|
||||
task_id=hotspot.task_id,
|
||||
hotspot_id=hotspot.id,
|
||||
content_item_id=None,
|
||||
report_type="hotspot",
|
||||
title=hotspot.title,
|
||||
data=json.dumps(metrics, ensure_ascii=False),
|
||||
markdown=markdown,
|
||||
metrics_json=json.dumps(metrics, ensure_ascii=False),
|
||||
typical_comments_json=json.dumps(typical, ensure_ascii=False),
|
||||
summary=summary,
|
||||
markdown_content=markdown,
|
||||
)
|
||||
session.add(report)
|
||||
session.commit()
|
||||
session.refresh(report)
|
||||
return report
|
||||
|
||||
|
||||
def _safe_summary(summary_provider, metrics: dict, typical: dict, *, limit: int) -> str:
|
||||
if summary_provider is None:
|
||||
return DEFAULT_SUMMARY
|
||||
try:
|
||||
summary = summary_provider(metrics, typical)
|
||||
except Exception:
|
||||
return DEFAULT_SUMMARY
|
||||
return str(summary)[:limit]
|
||||
|
||||
|
||||
def _build_markdown(*, title: str, metrics: dict, typical: dict, summary: str, hotspot_title: str = "") -> str:
|
||||
lines = [
|
||||
f"# {title}",
|
||||
"",
|
||||
]
|
||||
if hotspot_title:
|
||||
lines.extend([f"- 所属热点：{hotspot_title}", ""])
|
||||
lines.extend(
|
||||
[
|
||||
f"- 样本评论数量：{metrics['sample_count']}",
|
||||
"",
|
||||
"## 情绪分布",
|
||||
]
|
||||
)
|
||||
for name in ("positive", "neutral", "negative", "unknown"):
|
||||
item = metrics["sentiment"][name]
|
||||
lines.append(f"- {name}: {item['count']} ({item['pct']}%)")
|
||||
lines.extend(["", "## Top 标签"])
|
||||
for label in metrics["top_labels"]:
|
||||
lines.append(f"- {label['name']}: {label['count']}")
|
||||
lines.extend(["", "## 典型评论"])
|
||||
for sentiment, comments in typical.items():
|
||||
for comment in comments:
|
||||
lines.append(f"- [{sentiment}] {comment['content']}")
|
||||
lines.extend(["", "## 总结", summary])
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,243 @@
|
||||
import inspect
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections.abc import Callable
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Comment, ContentItem, Hotspot, Task
|
||||
from app.platforms.base import PlatformAPIError, TikHubClient
|
||||
from app.platforms.douyin import DouyinPlatform
|
||||
from app.platforms.xiaohongshu import XiaohongshuPlatform
|
||||
from app.schemas import CreateTaskRequest
|
||||
from app.services.ai_service import OpenAICompatibleAIClient, analyze_comments_with_retry, calculate_analysis_status
|
||||
from app.services.report_service import generate_hotspot_report, generate_item_report
|
||||
|
||||
|
||||
RUNNING_TASK_MESSAGE = "当前有正在运行的任务，请稍后再试"
|
||||
RESTART_ERROR_MESSAGE = "系统重启，任务被中断"
|
||||
task_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
|
||||
def has_running_task(session: Session) -> bool:
|
||||
return session.scalar(select(Task.id).where(Task.status == "running").limit(1)) is not None
|
||||
|
||||
|
||||
def recover_running_tasks(session: Session) -> int:
|
||||
tasks = list(session.scalars(select(Task).where(Task.status == "running")))
|
||||
for task in tasks:
|
||||
task.status = "failed"
|
||||
task.error_stage = "system"
|
||||
task.error_type = "unexpected_restart"
|
||||
task.error_message = RESTART_ERROR_MESSAGE
|
||||
session.commit()
|
||||
return len(tasks)
|
||||
|
||||
|
||||
def build_platform(platform: str):
|
||||
settings = get_settings()
|
||||
client = TikHubClient(
|
||||
base_url=settings.tikhub_base_url,
|
||||
api_key=settings.tikhub_api_key,
|
||||
timeout_seconds=settings.http_timeout_seconds,
|
||||
max_retries=settings.http_max_retries,
|
||||
)
|
||||
if platform == "xiaohongshu":
|
||||
return XiaohongshuPlatform(client)
|
||||
if platform == "douyin":
|
||||
return DouyinPlatform(client)
|
||||
raise ValueError(f"Unsupported platform: {platform}")
|
||||
|
||||
|
||||
def build_ai_requester():
|
||||
settings = get_settings()
|
||||
if settings.ai_base_url and settings.ai_api_key and settings.ai_model:
|
||||
client = OpenAICompatibleAIClient(
|
||||
base_url=settings.ai_base_url,
|
||||
api_key=settings.ai_api_key,
|
||||
model=settings.ai_model,
|
||||
timeout_seconds=settings.ai_timeout_seconds,
|
||||
)
|
||||
return client.request
|
||||
|
||||
def fallback_requester(_prompt: str) -> str:
|
||||
return "[]"
|
||||
|
||||
return fallback_requester
|
||||
|
||||
|
||||
def create_task(session: Session, request: CreateTaskRequest, *, submit_background: bool = True) -> Task:
|
||||
task = Task(
|
||||
platform=request.platform,
|
||||
status="running",
|
||||
hotspot_limit=request.hotspot_limit,
|
||||
item_limit_per_hotspot=request.item_limit_per_hotspot,
|
||||
comment_limit_per_item=request.comment_limit_per_item,
|
||||
total_items_count=0,
|
||||
processed_items_count=0,
|
||||
successful_items_count=0,
|
||||
failed_items_count=0,
|
||||
analysis_success_rate=0.0,
|
||||
analysis_status="normal",
|
||||
)
|
||||
session.add(task)
|
||||
session.commit()
|
||||
session.refresh(task)
|
||||
if submit_background:
|
||||
task_executor.submit(run_task, task.id)
|
||||
return task
|
||||
|
||||
|
||||
def list_tasks(session: Session) -> list[Task]:
|
||||
return list(session.scalars(select(Task).order_by(Task.created_at.desc())))
|
||||
|
||||
|
||||
def get_task(session: Session, task_id: str) -> Task | None:
|
||||
return session.get(Task, task_id)
|
||||
|
||||
|
||||
def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionmaker = SessionLocal) -> None:
|
||||
session = session_factory()
|
||||
close_session = hasattr(session, "close")
|
||||
try:
|
||||
task = session.get(Task, task_id)
|
||||
if task is None:
|
||||
return
|
||||
try:
|
||||
platform = build_platform(task.platform)
|
||||
try:
|
||||
hotspots = platform.fetch_hotspots(limit=task.hotspot_limit)
|
||||
except PlatformAPIError as exc:
|
||||
_mark_task_failed(task, "crawl_hotspots", exc.error_type, str(exc))
|
||||
session.commit()
|
||||
return
|
||||
except Exception as exc:
|
||||
_mark_task_failed(task, "crawl_hotspots", "api_error", str(exc))
|
||||
session.commit()
|
||||
return
|
||||
|
||||
for hotspot_data in hotspots:
|
||||
hotspot = Hotspot(
|
||||
task_id=task.id,
|
||||
platform=task.platform,
|
||||
source_hot_id=hotspot_data.source_hot_id,
|
||||
rank=hotspot_data.rank,
|
||||
title=hotspot_data.title,
|
||||
heat_value=hotspot_data.heat_value,
|
||||
raw_data=json.dumps(hotspot_data.raw_data or {}, ensure_ascii=False),
|
||||
)
|
||||
session.add(hotspot)
|
||||
session.flush()
|
||||
|
||||
try:
|
||||
items = platform.search_items_by_hotspot(hotspot.title, limit=task.item_limit_per_hotspot)
|
||||
except Exception as exc:
|
||||
task.failed_items_count += task.item_limit_per_hotspot
|
||||
task.error_stage = "crawl_items"
|
||||
task.error_type = getattr(exc, "error_type", "api_error")
|
||||
task.error_message = str(exc)
|
||||
session.commit()
|
||||
continue
|
||||
|
||||
task.total_items_count += len(items)
|
||||
seen_item_ids: set[str] = set()
|
||||
for item_data in items:
|
||||
if item_data.source_item_id in seen_item_ids:
|
||||
continue
|
||||
seen_item_ids.add(item_data.source_item_id)
|
||||
item = ContentItem(
|
||||
task_id=task.id,
|
||||
hotspot_id=hotspot.id,
|
||||
platform=task.platform,
|
||||
source_item_id=item_data.source_item_id,
|
||||
item_type=item_data.item_type,
|
||||
title=item_data.title,
|
||||
summary=item_data.summary,
|
||||
url=item_data.url,
|
||||
status="pending",
|
||||
raw_data=json.dumps(item_data.raw_data or {}, ensure_ascii=False),
|
||||
)
|
||||
session.add(item)
|
||||
session.flush()
|
||||
try:
|
||||
comments = platform.fetch_comments(item.source_item_id, limit=task.comment_limit_per_item)
|
||||
for comment_data in comments:
|
||||
session.add(
|
||||
Comment(
|
||||
task_id=task.id,
|
||||
hotspot_id=hotspot.id,
|
||||
content_item_id=item.id,
|
||||
platform=task.platform,
|
||||
source_comment_id=comment_data.source_comment_id,
|
||||
content=comment_data.content,
|
||||
author=comment_data.author,
|
||||
like_count=comment_data.like_count,
|
||||
comment_time=comment_data.comment_time,
|
||||
raw_data=json.dumps(comment_data.raw_data or {}, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
ai_results = analyze_comments_with_retry(
|
||||
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
|
||||
requester=build_ai_requester(),
|
||||
max_retries=get_settings().ai_max_retries,
|
||||
)
|
||||
results_by_comment_id = {result.comment_id: result for result in ai_results}
|
||||
for comment in item_comments:
|
||||
result = results_by_comment_id.get(comment.id)
|
||||
if result is None:
|
||||
comment.sentiment = "unknown"
|
||||
comment.labels = "[]"
|
||||
comment.reason = "ai_missing_result"
|
||||
comment.ai_analysis_status = "failed"
|
||||
continue
|
||||
comment.sentiment = result.sentiment
|
||||
comment.labels = json.dumps(result.labels, ensure_ascii=False)
|
||||
comment.reason = result.reason
|
||||
comment.ai_analysis_status = result.ai_analysis_status
|
||||
item.status = "success"
|
||||
task.successful_items_count += 1
|
||||
generate_item_report(session, item.id)
|
||||
except Exception as exc:
|
||||
item.status = "failed"
|
||||
item.error_stage = "crawl_comments"
|
||||
item.error_type = getattr(exc, "error_type", "api_error")
|
||||
item.error_message = str(exc)
|
||||
task.failed_items_count += 1
|
||||
task.error_stage = item.error_stage
|
||||
task.error_type = item.error_type
|
||||
task.error_message = item.error_message
|
||||
finally:
|
||||
task.processed_items_count += 1
|
||||
session.commit()
|
||||
|
||||
if task.successful_items_count > 0:
|
||||
total_comments = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||||
success_comments = sum(1 for comment in task.comments if comment.ai_analysis_status == "success")
|
||||
task.analysis_success_rate, task.analysis_status = calculate_analysis_status(
|
||||
success_count=success_comments,
|
||||
total_count=total_comments,
|
||||
)
|
||||
for hotspot in task.hotspots:
|
||||
generate_hotspot_report(session, hotspot.id)
|
||||
task.status = "success"
|
||||
else:
|
||||
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
|
||||
session.commit()
|
||||
except Exception as exc:
|
||||
_mark_task_failed(task, "system", "unexpected_error", str(exc))
|
||||
session.commit()
|
||||
finally:
|
||||
if close_session:
|
||||
session.close()
|
||||
|
||||
|
||||
def _mark_task_failed(task: Task, stage: str, error_type: str, message: str) -> None:
|
||||
task.status = "failed"
|
||||
task.error_stage = stage
|
||||
task.error_type = error_type
|
||||
task.error_message = message
|
||||
Reference in New Issue
Block a user