feat: 提交热榜评论分析工具 MVP 基线
This commit is contained in:
meijiali
@@ -0,0 +1 @@
|
||||
"""Application package for the hot comments analysis tool."""
|
||||
@@ -0,0 +1,41 @@
|
||||
from functools import lru_cache
|
||||
|
||||
from pydantic import field_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
||||
|
||||
app_env: str = "development"
|
||||
database_url: str = "sqlite:///data/app.db"
|
||||
tikhub_api_key: str = ""
|
||||
tikhub_base_url: str = "https://api.tikhub.io"
|
||||
ai_provider: str = "openai-compatible"
|
||||
ai_base_url: str = ""
|
||||
ai_api_key: str = ""
|
||||
ai_model: str = ""
|
||||
ai_batch_size: int = 20
|
||||
ai_concurrency: int = 2
|
||||
ai_max_retries: int = 3
|
||||
ai_timeout_seconds: int = 30
|
||||
http_timeout_seconds: int = 20
|
||||
http_max_retries: int = 3
|
||||
crawl_page_interval_seconds: float = 1.5
|
||||
|
||||
hot_limit_min: int = 1
|
||||
hot_limit_max: int = 10
|
||||
item_limit_per_hot_min: int = 1
|
||||
item_limit_per_hot_max: int = 10
|
||||
comment_limit_per_item_min: int = 10
|
||||
comment_limit_per_item_max: int = 100
|
||||
|
||||
@field_validator("ai_concurrency", mode="after")
|
||||
@classmethod
|
||||
def cap_ai_concurrency(cls, value: int) -> int:
|
||||
return min(value, 3)
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
@@ -0,0 +1,45 @@
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
def create_sqlite_engine(database_url: str):
|
||||
if database_url.startswith("sqlite:///") and database_url != "sqlite:///:memory:":
|
||||
db_path = Path(database_url.removeprefix("sqlite:///"))
|
||||
if db_path.parent != Path("."):
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
connect_args = {"check_same_thread": False, "timeout": 10}
|
||||
engine = create_engine(database_url, connect_args=connect_args)
|
||||
engine.dialect.connect_args = connect_args
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def set_sqlite_pragma(dbapi_connection, _connection_record):
|
||||
cursor = dbapi_connection.cursor()
|
||||
cursor.execute("PRAGMA journal_mode=WAL")
|
||||
cursor.close()
|
||||
|
||||
return engine
|
||||
|
||||
|
||||
engine = create_sqlite_engine(get_settings().database_url)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
import app.models # noqa: F401
|
||||
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
|
||||
def get_db_session() -> Generator[Session]:
|
||||
with SessionLocal() as session:
|
||||
yield session
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException, Request, status
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.responses import Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal, get_db_session, init_db
|
||||
from app.models import Comment, ContentItem, Hotspot, Report
|
||||
from app.schemas import CreateTaskRequest, CreateTaskResponse, TaskResponse
|
||||
from app.services.export_service import export_hotspot_comments_csv, export_item_comments_csv, export_report_markdown
|
||||
from app.services.task_service import (
|
||||
RUNNING_TASK_MESSAGE,
|
||||
create_task,
|
||||
get_task,
|
||||
has_running_task,
|
||||
list_tasks,
|
||||
recover_running_tasks,
|
||||
)
|
||||
from app.templating import templates
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncGenerator[None]:
|
||||
init_db()
|
||||
with SessionLocal() as session:
|
||||
recover_running_tasks(session)
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title="热榜评论分析工具", lifespan=lifespan)
|
||||
app.mount("/static", StaticFiles(directory="app/static"), name="static")
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index_page(request: Request, session: Session = Depends(get_db_session)) -> HTMLResponse:
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"index.html",
|
||||
{"tasks": list_tasks(session)},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/tasks", response_model=CreateTaskResponse, status_code=status.HTTP_201_CREATED)
|
||||
def create_task_api(
|
||||
request: CreateTaskRequest,
|
||||
session: Session = Depends(get_db_session),
|
||||
) -> CreateTaskResponse:
|
||||
if has_running_task(session):
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=RUNNING_TASK_MESSAGE)
|
||||
|
||||
task = create_task(session, request)
|
||||
return CreateTaskResponse(task_id=task.id, status=task.status)
|
||||
|
||||
|
||||
@app.get("/api/tasks", response_model=list[TaskResponse])
|
||||
def list_tasks_api(session: Session = Depends(get_db_session)) -> list[TaskResponse]:
|
||||
return [TaskResponse.model_validate(task) for task in list_tasks(session)]
|
||||
|
||||
|
||||
@app.get("/api/tasks/{task_id}", response_model=TaskResponse)
|
||||
def get_task_api(task_id: str, session: Session = Depends(get_db_session)) -> TaskResponse:
|
||||
task = get_task(session, task_id)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="任务不存在")
|
||||
return TaskResponse.model_validate(task)
|
||||
|
||||
|
||||
@app.get("/tasks/{task_id}", response_class=HTMLResponse)
|
||||
def task_detail_page(task_id: str, request: Request, session: Session = Depends(get_db_session)) -> HTMLResponse:
|
||||
task = get_task(session, task_id)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="任务不存在")
|
||||
hotspots = list(session.scalars(select(Hotspot).where(Hotspot.task_id == task.id).order_by(Hotspot.rank)))
|
||||
return templates.TemplateResponse(request, "tasks/detail.html", {"task": task, "hotspots": hotspots})
|
||||
|
||||
|
||||
@app.get("/hotspots/{hotspot_id}/report", response_class=HTMLResponse)
|
||||
def hotspot_report_page(hotspot_id: str, request: Request, session: Session = Depends(get_db_session)) -> HTMLResponse:
|
||||
hotspot = session.get(Hotspot, hotspot_id)
|
||||
if hotspot is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="热点不存在")
|
||||
task = get_task(session, hotspot.task_id)
|
||||
report = session.scalar(select(Report).where(Report.hotspot_id == hotspot.id, Report.report_type == "hotspot"))
|
||||
return templates.TemplateResponse(request, "hotspots/report.html", {"task": task, "hotspot": hotspot, "report": report})
|
||||
|
||||
|
||||
@app.get("/items/{item_id}", response_class=HTMLResponse)
|
||||
def item_detail_page(item_id: str, request: Request, session: Session = Depends(get_db_session)) -> HTMLResponse:
|
||||
item = session.get(ContentItem, item_id)
|
||||
if item is None:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="内容条目不存在")
|
||||
hotspot = session.get(Hotspot, item.hotspot_id)
|
||||
task = get_task(session, item.task_id)
|
||||
report = session.scalar(select(Report).where(Report.content_item_id == item.id, Report.report_type == "item"))
|
||||
comments = list(
|
||||
session.scalars(
|
||||
select(Comment)
|
||||
.where(Comment.content_item_id == item.id)
|
||||
.order_by(Comment.like_count.desc().nullslast(), Comment.comment_time.desc().nullslast())
|
||||
.limit(100)
|
||||
)
|
||||
)
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"items/detail.html",
|
||||
{"task": task, "hotspot": hotspot, "item": item, "report": report, "comments": comments},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/export/items/{item_id}/comments.csv")
|
||||
def export_item_comments(item_id: str, session: Session = Depends(get_db_session)) -> Response:
|
||||
try:
|
||||
content, filename = export_item_comments_csv(session, item_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="导出数据不存在")
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/export/hotspots/{hotspot_id}/comments.csv")
|
||||
def export_hotspot_comments(hotspot_id: str, session: Session = Depends(get_db_session)) -> Response:
|
||||
try:
|
||||
content, filename = export_hotspot_comments_csv(session, hotspot_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="导出数据不存在")
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="text/csv; charset=utf-8",
|
||||
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/export/hotspots/{hotspot_id}.md")
|
||||
def export_hotspot_markdown(hotspot_id: str, session: Session = Depends(get_db_session)) -> Response:
|
||||
try:
|
||||
content, filename = export_report_markdown(session, report_type="hotspot", hotspot_id=hotspot_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="报告不存在")
|
||||
return Response(content=content, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"})
|
||||
|
||||
|
||||
@app.get("/api/export/items/{item_id}.md")
|
||||
def export_item_markdown(item_id: str, session: Session = Depends(get_db_session)) -> Response:
|
||||
try:
|
||||
content, filename = export_report_markdown(session, report_type="item", item_id=item_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="报告不存在")
|
||||
return Response(content=content, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"})
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, Float, ForeignKey, Index, Integer, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.db import Base
|
||||
|
||||
|
||||
def new_uuid() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
__table_args__ = (
|
||||
CheckConstraint("status IN ('running', 'success', 'failed')", name="ck_tasks_status"),
|
||||
CheckConstraint(
|
||||
"analysis_status IN ('normal', 'insufficient')",
|
||||
name="ck_tasks_analysis_status",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
platform: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running")
|
||||
analysis_status: Mapped[str] = mapped_column(String(16), nullable=False, default="normal")
|
||||
analysis_success_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
hotspot_limit: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
|
||||
item_limit_per_hotspot: Mapped[int] = mapped_column(Integer, nullable=False, default=5)
|
||||
comment_limit_per_item: Mapped[int] = mapped_column(Integer, nullable=False, default=50)
|
||||
total_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
processed_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
successful_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
error_stage: Mapped[str | None] = mapped_column(String(64))
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(nullable=False, default=utc_now)
|
||||
started_at: Mapped[datetime | None] = mapped_column()
|
||||
finished_at: Mapped[datetime | None] = mapped_column()
|
||||
|
||||
hotspots: Mapped[list["Hotspot"]] = relationship(back_populates="task")
|
||||
content_items: Mapped[list["ContentItem"]] = relationship(back_populates="task")
|
||||
comments: Mapped[list["Comment"]] = relationship(back_populates="task")
|
||||
reports: Mapped[list["Report"]] = relationship(back_populates="task")
|
||||
|
||||
|
||||
class Hotspot(Base):
|
||||
__tablename__ = "hotspots"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
task_id: Mapped[str] = mapped_column(ForeignKey("tasks.id"), nullable=False)
|
||||
platform: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
source_hot_id: Mapped[str | None] = mapped_column(String(128))
|
||||
rank: Mapped[int | None] = mapped_column(Integer)
|
||||
title: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
heat_value: Mapped[str | None] = mapped_column(String(128))
|
||||
raw_data: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
||||
created_at: Mapped[datetime] = mapped_column(nullable=False, default=utc_now)
|
||||
|
||||
task: Mapped[Task] = relationship(back_populates="hotspots")
|
||||
content_items: Mapped[list["ContentItem"]] = relationship(back_populates="hotspot")
|
||||
comments: Mapped[list["Comment"]] = relationship(back_populates="hotspot")
|
||||
reports: Mapped[list["Report"]] = relationship(back_populates="hotspot")
|
||||
|
||||
|
||||
class ContentItem(Base):
|
||||
__tablename__ = "content_items"
|
||||
__table_args__ = (
|
||||
Index("ix_content_items_task_hotspot", "task_id", "hotspot_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
task_id: Mapped[str] = mapped_column(ForeignKey("tasks.id"), nullable=False)
|
||||
hotspot_id: Mapped[str] = mapped_column(ForeignKey("hotspots.id"), nullable=False)
|
||||
platform: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
source_item_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
item_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
url: Mapped[str | None] = mapped_column(Text)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
|
||||
error_stage: Mapped[str | None] = mapped_column(String(64))
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
raw_data: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
||||
created_at: Mapped[datetime] = mapped_column(nullable=False, default=utc_now)
|
||||
|
||||
task: Mapped[Task] = relationship(back_populates="content_items")
|
||||
hotspot: Mapped[Hotspot] = relationship(back_populates="content_items")
|
||||
comments: Mapped[list["Comment"]] = relationship(back_populates="content_item")
|
||||
reports: Mapped[list["Report"]] = relationship(back_populates="content_item")
|
||||
|
||||
|
||||
class Comment(Base):
|
||||
__tablename__ = "comments"
|
||||
__table_args__ = (
|
||||
Index("ix_comments_task_content_item", "task_id", "content_item_id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
task_id: Mapped[str] = mapped_column(ForeignKey("tasks.id"), nullable=False)
|
||||
hotspot_id: Mapped[str] = mapped_column(ForeignKey("hotspots.id"), nullable=False)
|
||||
content_item_id: Mapped[str] = mapped_column(ForeignKey("content_items.id"), nullable=False)
|
||||
platform: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
source_comment_id: Mapped[str | None] = mapped_column(String(128))
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
author: Mapped[str | None] = mapped_column(Text)
|
||||
like_count: Mapped[int | None] = mapped_column(Integer)
|
||||
comment_time: Mapped[datetime | None] = mapped_column()
|
||||
sentiment: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown")
|
||||
labels: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
||||
reason: Mapped[str | None] = mapped_column(Text)
|
||||
ai_analysis_status: Mapped[str] = mapped_column(String(32), nullable=False, default="skipped")
|
||||
raw_data: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
||||
ai_raw_response: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(nullable=False, default=utc_now)
|
||||
|
||||
task: Mapped[Task] = relationship(back_populates="comments")
|
||||
hotspot: Mapped[Hotspot] = relationship(back_populates="comments")
|
||||
content_item: Mapped[ContentItem] = relationship(back_populates="comments")
|
||||
|
||||
|
||||
class Report(Base):
|
||||
__tablename__ = "reports"
|
||||
__table_args__ = (
|
||||
Index("ix_reports_task_report_type", "task_id", "report_type"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||
task_id: Mapped[str] = mapped_column(ForeignKey("tasks.id"), nullable=False)
|
||||
hotspot_id: Mapped[str | None] = mapped_column(ForeignKey("hotspots.id"))
|
||||
content_item_id: Mapped[str | None] = mapped_column(ForeignKey("content_items.id"))
|
||||
report_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
title: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
data: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
||||
markdown: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
metrics_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
||||
typical_comments_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
||||
summary: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
markdown_content: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
created_at: Mapped[datetime] = mapped_column(nullable=False, default=utc_now)
|
||||
|
||||
task: Mapped[Task] = relationship(back_populates="reports")
|
||||
hotspot: Mapped[Hotspot | None] = relationship(back_populates="reports")
|
||||
content_item: Mapped[ContentItem | None] = relationship(back_populates="reports")
|
||||
|
||||
|
||||
def create_report_record(
|
||||
*,
|
||||
task_id: str,
|
||||
report_type: str,
|
||||
title: str,
|
||||
hotspot_id: str | None = None,
|
||||
content_item_id: str | None = None,
|
||||
data: str = "{}",
|
||||
markdown: str = "",
|
||||
) -> Report:
|
||||
if report_type == "hotspot" and not hotspot_id:
|
||||
raise ValueError("hotspot report requires hotspot_id")
|
||||
if report_type == "content_item" and not content_item_id:
|
||||
raise ValueError("content item report requires content_item_id")
|
||||
return Report(
|
||||
task_id=task_id,
|
||||
report_type=report_type,
|
||||
title=title,
|
||||
hotspot_id=hotspot_id,
|
||||
content_item_id=content_item_id,
|
||||
data=data,
|
||||
markdown=markdown,
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass
|
||||
class HotspotData:
|
||||
source_hot_id: str | None
|
||||
title: str
|
||||
rank: int | None = None
|
||||
heat_value: str | None = None
|
||||
raw_data: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContentItemData:
|
||||
source_item_id: str
|
||||
item_type: str
|
||||
title: str | None = None
|
||||
summary: str | None = None
|
||||
url: str | None = None
|
||||
raw_data: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommentData:
|
||||
source_comment_id: str | None
|
||||
content: str
|
||||
author: str | None = None
|
||||
like_count: int | None = None
|
||||
comment_time: datetime | None = None
|
||||
raw_data: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class PlatformAPIError(Exception):
|
||||
def __init__(self, message: str, *, error_type: str, status_code: int | None = None):
|
||||
super().__init__(message)
|
||||
self.error_type = error_type
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class TikHubClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
timeout_seconds: int = 20,
|
||||
max_retries: int = 3,
|
||||
http_client: httpx.Client | None = None,
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.max_retries = max_retries
|
||||
self._http_client = http_client or httpx.Client(timeout=timeout_seconds)
|
||||
|
||||
def get(self, path: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return self._request("GET", path, params=params)
|
||||
|
||||
def post(self, path: str, *, json: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return self._request("POST", path, json=json)
|
||||
|
||||
def _request(self, method: str, path: str, **kwargs) -> dict[str, Any]:
|
||||
url = f"{self.base_url}{path}"
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
||||
last_status: int | None = None
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
response = self._http_client.request(method, url, headers=headers, **kwargs)
|
||||
except httpx.RequestError as exc:
|
||||
if attempt >= self.max_retries:
|
||||
raise PlatformAPIError("External API request failed", error_type="network_error") from exc
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
|
||||
last_status = response.status_code
|
||||
if response.status_code == 429:
|
||||
if attempt >= self.max_retries:
|
||||
raise PlatformAPIError(
|
||||
"External API rate limited",
|
||||
error_type="rate_limited",
|
||||
status_code=response.status_code,
|
||||
)
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
if response.is_error:
|
||||
raise PlatformAPIError(
|
||||
f"External API returned HTTP {response.status_code}",
|
||||
error_type="api_error",
|
||||
status_code=response.status_code,
|
||||
)
|
||||
return response.json()
|
||||
|
||||
raise PlatformAPIError("External API request failed", error_type="api_error", status_code=last_status)
|
||||
|
||||
|
||||
def parse_timestamp(value: Any) -> datetime | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
number = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if number > 10_000_000_000:
|
||||
number = number // 1000
|
||||
return datetime.fromtimestamp(number, tz=UTC)
|
||||
|
||||
|
||||
def first_present(data: dict[str, Any], *keys: str) -> Any:
|
||||
for key in keys:
|
||||
if key in data and data[key] not in (None, ""):
|
||||
return data[key]
|
||||
return None
|
||||
@@ -0,0 +1,99 @@
|
||||
from typing import Any
|
||||
|
||||
from app.platforms.base import CommentData, ContentItemData, HotspotData, TikHubClient, first_present, parse_timestamp
|
||||
|
||||
|
||||
class DouyinPlatform:
|
||||
def __init__(self, client: TikHubClient | None) -> None:
|
||||
self.client = client
|
||||
|
||||
def map_hotspots(self, payload: dict[str, Any], *, limit: int) -> list[HotspotData]:
|
||||
data = payload.get("data", {})
|
||||
items = data.get("word_list") or data.get("list") or data.get("item_list") or []
|
||||
result = []
|
||||
for index, item in enumerate(items[:limit], start=1):
|
||||
result.append(
|
||||
HotspotData(
|
||||
source_hot_id=str(item.get("query_id")) if item.get("query_id") is not None else None,
|
||||
title=str(first_present(item, "title", "sentence", "word") or ""),
|
||||
rank=item.get("rank") or index,
|
||||
heat_value=str(item.get("hot_score")) if item.get("hot_score") is not None else None,
|
||||
raw_data=item,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def map_items(self, payload: dict[str, Any], *, limit: int) -> list[ContentItemData]:
|
||||
data = payload.get("data", [])
|
||||
raw_items = data.get("business_data") or data.get("data") or data.get("items") or [] if isinstance(data, dict) else data
|
||||
result = []
|
||||
for item in raw_items[:limit]:
|
||||
nested = item.get("data", item) if isinstance(item, dict) else {}
|
||||
aweme = nested.get("aweme_info", nested) if isinstance(nested, dict) else {}
|
||||
aweme_id = aweme.get("aweme_id")
|
||||
if not aweme_id:
|
||||
continue
|
||||
result.append(
|
||||
ContentItemData(
|
||||
source_item_id=str(aweme_id),
|
||||
item_type="video",
|
||||
title=aweme.get("desc"),
|
||||
summary=aweme.get("desc"),
|
||||
url=aweme.get("share_url"),
|
||||
raw_data=aweme,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def map_comments(self, payload: dict[str, Any], *, limit: int) -> list[CommentData]:
|
||||
comments = payload.get("comments") or payload.get("data", {}).get("comments") or []
|
||||
result = []
|
||||
for comment in comments[:limit]:
|
||||
content = comment.get("text")
|
||||
if not content:
|
||||
continue
|
||||
result.append(
|
||||
CommentData(
|
||||
source_comment_id=first_present(comment, "comment_id", "cid"),
|
||||
content=str(content),
|
||||
author=self._author_name(comment),
|
||||
like_count=comment.get("digg_count"),
|
||||
comment_time=parse_timestamp(first_present(comment, "create_time", "create_time_str")),
|
||||
raw_data=comment,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def fetch_hotspots(self, *, limit: int) -> list[HotspotData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/douyin/creator/fetch_creator_hot_spot_billboard",
|
||||
params={"billboard_tag": 0, "hot_search_type": 1},
|
||||
)
|
||||
return self.map_hotspots(payload, limit=limit)
|
||||
|
||||
def search_items_by_hotspot(self, keyword: str, *, limit: int) -> list[ContentItemData]:
|
||||
payload = self.client.post(
|
||||
"/api/v1/douyin/search/fetch_video_search_v2",
|
||||
json={
|
||||
"keyword": keyword,
|
||||
"cursor": 0,
|
||||
"sort_type": "0",
|
||||
"publish_time": "0",
|
||||
"filter_duration": "0",
|
||||
"content_type": "1",
|
||||
"search_id": "",
|
||||
"backtrace": "",
|
||||
},
|
||||
)
|
||||
return self.map_items(payload, limit=limit)
|
||||
|
||||
def fetch_comments(self, source_item_id: str, *, limit: int) -> list[CommentData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/douyin/app/v3/fetch_video_comments",
|
||||
params={"aweme_id": source_item_id, "cursor": 0, "count": 20},
|
||||
)
|
||||
return self.map_comments(payload, limit=limit)
|
||||
|
||||
def _author_name(self, comment: dict[str, Any]) -> str | None:
|
||||
user = comment.get("user") or {}
|
||||
return user.get("nickname") or user.get("name") if isinstance(user, dict) else None
|
||||
@@ -0,0 +1,84 @@
|
||||
from typing import Any
|
||||
|
||||
from app.platforms.base import CommentData, ContentItemData, HotspotData, TikHubClient, first_present, parse_timestamp
|
||||
|
||||
|
||||
class XiaohongshuPlatform:
|
||||
def __init__(self, client: TikHubClient | None) -> None:
|
||||
self.client = client
|
||||
|
||||
def map_hotspots(self, payload: dict[str, Any], *, limit: int) -> list[HotspotData]:
|
||||
items = payload.get("data", {}).get("data", {}).get("items", [])
|
||||
hotspots = []
|
||||
for index, item in enumerate(items[:limit], start=1):
|
||||
hotspots.append(
|
||||
HotspotData(
|
||||
source_hot_id=str(item.get("id")) if item.get("id") is not None else None,
|
||||
title=str(item.get("title") or ""),
|
||||
rank=index,
|
||||
heat_value=str(item.get("score")) if item.get("score") is not None else None,
|
||||
raw_data=item,
|
||||
)
|
||||
)
|
||||
return hotspots
|
||||
|
||||
def map_items(self, payload: dict[str, Any], *, limit: int) -> list[ContentItemData]:
|
||||
raw_items = payload.get("data", {}).get("data", {}).get("items", [])
|
||||
notes = [item.get("note", item) for item in raw_items]
|
||||
notes.sort(key=lambda note: 0 if int(note.get("comments_count") or 0) > 0 else 1)
|
||||
result = []
|
||||
for note in notes[:limit]:
|
||||
note_id = note.get("id")
|
||||
if not note_id:
|
||||
continue
|
||||
result.append(
|
||||
ContentItemData(
|
||||
source_item_id=str(note_id),
|
||||
item_type="note",
|
||||
title=note.get("title"),
|
||||
summary=note.get("desc"),
|
||||
url=note.get("url"),
|
||||
raw_data=note,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def map_comments(self, payload: dict[str, Any], *, limit: int) -> list[CommentData]:
|
||||
comments = payload.get("data", {}).get("data", {}).get("comments", [])
|
||||
result = []
|
||||
for comment in comments[:limit]:
|
||||
content = first_present(comment, "content", "text")
|
||||
if not content:
|
||||
continue
|
||||
result.append(
|
||||
CommentData(
|
||||
source_comment_id=first_present(comment, "comment_id", "id"),
|
||||
content=str(content),
|
||||
author=self._author_name(comment),
|
||||
like_count=comment.get("like_count"),
|
||||
comment_time=parse_timestamp(first_present(comment, "create_time", "create_time_str")),
|
||||
raw_data=comment,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def fetch_hotspots(self, *, limit: int) -> list[HotspotData]:
|
||||
return self.map_hotspots(self.client.get("/api/v1/xiaohongshu/web_v2/fetch_hot_list"), limit=limit)
|
||||
|
||||
def search_items_by_hotspot(self, keyword: str, *, limit: int) -> list[ContentItemData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/xiaohongshu/app_v2/search_notes",
|
||||
params={"keyword": keyword, "page": 1, "sort": "general", "note_type": 0},
|
||||
)
|
||||
return self.map_items(payload, limit=limit)
|
||||
|
||||
def fetch_comments(self, source_item_id: str, *, limit: int) -> list[CommentData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/xiaohongshu/app_v2/get_note_comments",
|
||||
params={"note_id": source_item_id, "cursor": "", "index": 0, "pageArea": "UNFOLDED", "sort_strategy": "latest_v2"},
|
||||
)
|
||||
return self.map_comments(payload, limit=limit)
|
||||
|
||||
def _author_name(self, comment: dict[str, Any]) -> str | None:
|
||||
user = comment.get("user_info") or comment.get("user") or {}
|
||||
return user.get("nickname") or user.get("name") if isinstance(user, dict) else None
|
||||
@@ -0,0 +1,4 @@
|
||||
只返回 JSON Array，不输出 Markdown 或解释性自然语言。
|
||||
请原样回填输入中的 comment_id，不得修改或生成新 ID。
|
||||
sentiment 只能是 positive、negative、neutral、unknown。
|
||||
labels 使用 1 到 3 个简短中文短语；无法判断时 labels 可为空数组。
|
||||
@@ -0,0 +1,3 @@
|
||||
根据输入的情绪分布、Top 标签和典型评论，生成事实性中文总结。
|
||||
只输出纯文本，不使用 Markdown，不超过指定字数。
|
||||
总结失败时由后端使用默认文案兜底。
|
||||
@@ -0,0 +1,40 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import AliasChoices, BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class CreateTaskRequest(BaseModel):
|
||||
platform: Literal["xiaohongshu", "douyin"]
|
||||
hotspot_limit: int = Field(default=5, ge=1, le=10)
|
||||
item_limit_per_hotspot: int = Field(
|
||||
default=5,
|
||||
ge=1,
|
||||
le=10,
|
||||
validation_alias=AliasChoices("item_limit_per_hotspot", "item_limit"),
|
||||
)
|
||||
comment_limit_per_item: int = Field(default=50, ge=10, le=100)
|
||||
|
||||
|
||||
class TaskResponse(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
task_id: str = Field(validation_alias="id", serialization_alias="task_id")
|
||||
platform: str
|
||||
status: str
|
||||
analysis_status: str
|
||||
analysis_success_rate: float
|
||||
hotspot_limit: int
|
||||
item_limit_per_hotspot: int
|
||||
comment_limit_per_item: int
|
||||
total_items_count: int
|
||||
processed_items_count: int
|
||||
successful_items_count: int
|
||||
failed_items_count: int
|
||||
error_message: str | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class CreateTaskResponse(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Service modules for task execution, crawling, AI analysis, reports, and export."""
|
||||
@@ -0,0 +1,140 @@
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
|
||||
Sentiment = Literal["positive", "negative", "neutral", "unknown"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AIAnalysisResult:
|
||||
comment_id: str
|
||||
sentiment: str
|
||||
labels: list[str]
|
||||
reason: str = ""
|
||||
ai_analysis_status: str = "success"
|
||||
|
||||
|
||||
class AIAnalysisItem(BaseModel):
|
||||
comment_id: str
|
||||
sentiment: Sentiment
|
||||
labels: list[str] = Field(default_factory=list, max_length=3)
|
||||
reason: str = ""
|
||||
|
||||
|
||||
def build_comment_prompt(comments: list[dict[str, str]]) -> str:
|
||||
payload = [
|
||||
{
|
||||
"comment_id": str(comment["comment_id"]),
|
||||
"content": str(comment.get("content", ""))[:150],
|
||||
}
|
||||
for comment in comments
|
||||
]
|
||||
return (
|
||||
"只返回 JSON Array，不输出 Markdown 或解释性自然语言。"
|
||||
"请原样回填输入中的 comment_id，不得修改或生成新 ID。"
|
||||
"sentiment 只能是 positive、negative、neutral、unknown；labels 最多 3 个简短中文短语。\n"
|
||||
f"{json.dumps(payload, ensure_ascii=False)}"
|
||||
)
|
||||
|
||||
|
||||
def parse_ai_comment_response(response_text: str, *, expected_comment_ids: set[str]) -> list[AIAnalysisResult]:
|
||||
try:
|
||||
raw = json.loads(response_text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError("AI response is not valid JSON") from exc
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("AI response must be a JSON Array")
|
||||
|
||||
results = []
|
||||
for item in raw:
|
||||
try:
|
||||
parsed = AIAnalysisItem.model_validate(item)
|
||||
except ValidationError as exc:
|
||||
raise ValueError("AI response item does not match schema") from exc
|
||||
if parsed.comment_id not in expected_comment_ids:
|
||||
raise ValueError("AI response comment_id does not match input")
|
||||
results.append(
|
||||
AIAnalysisResult(
|
||||
comment_id=parsed.comment_id,
|
||||
sentiment=parsed.sentiment,
|
||||
labels=parsed.labels,
|
||||
reason=parsed.reason,
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def calculate_analysis_status(*, success_count: int, total_count: int) -> tuple[float, str]:
|
||||
if total_count <= 0:
|
||||
return 0.0, "insufficient"
|
||||
rate = success_count / total_count
|
||||
return rate, "normal" if rate >= 0.8 else "insufficient"
|
||||
|
||||
|
||||
def analyze_comments_with_retry(
|
||||
comments: list[dict[str, str]],
|
||||
*,
|
||||
requester,
|
||||
max_retries: int = 3,
|
||||
) -> list[AIAnalysisResult]:
|
||||
expected_ids = {str(comment["comment_id"]) for comment in comments}
|
||||
prompt = build_comment_prompt(comments)
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return parse_ai_comment_response(requester(prompt), expected_comment_ids=expected_ids)
|
||||
except Exception:
|
||||
if attempt >= max_retries - 1:
|
||||
break
|
||||
time.sleep(2**attempt)
|
||||
return [
|
||||
AIAnalysisResult(
|
||||
comment_id=comment_id,
|
||||
sentiment="unknown",
|
||||
labels=[],
|
||||
reason="ai_parse_failed",
|
||||
ai_analysis_status="failed",
|
||||
)
|
||||
for comment_id in expected_ids
|
||||
]
|
||||
|
||||
|
||||
class OpenAICompatibleAIClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
timeout_seconds: int = 30,
|
||||
http_client: httpx.Client | None = None,
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self._http_client = http_client or httpx.Client(timeout=timeout_seconds)
|
||||
|
||||
def request(self, prompt: str) -> str:
|
||||
endpoint = f"{self.base_url}/chat/completions" if self.base_url.endswith("/v1") else f"{self.base_url}/v1/chat/completions"
|
||||
response = self._http_client.post(
|
||||
endpoint,
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "只返回 JSON Array，不输出 Markdown 或解释性自然语言。",
|
||||
},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return str(payload["choices"][0]["message"]["content"])
|
||||
@@ -0,0 +1,104 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Comment, ContentItem, Hotspot, Report
|
||||
|
||||
|
||||
def labels_to_text(labels_json: str | None) -> str:
|
||||
try:
|
||||
labels = json.loads(labels_json or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
return "，".join(str(label) for label in labels) if isinstance(labels, list) else ""
|
||||
|
||||
|
||||
def safe_csv_field(value) -> str:
|
||||
text = "" if value is None else str(value)
|
||||
text = text.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")
|
||||
if text.startswith(("=", "+", "-", "@")):
|
||||
return f"'{text}"
|
||||
return text
|
||||
|
||||
|
||||
def safe_filename(platform: str, task_id: str, hotspot_keyword: str, *, extension: str = "csv") -> str:
|
||||
keyword = hotspot_keyword[:20] + ("..." if len(hotspot_keyword) > 20 else "")
|
||||
name = f"{platform}_{task_id}_{keyword}"
|
||||
name = re.sub(r'[/\\:*?"<>|]+', "_", name)
|
||||
name = re.sub(r"_+", "_", name).strip("_")
|
||||
return f"{name}.{extension}"
|
||||
|
||||
|
||||
def export_item_comments_csv(session: Session, item_id: str) -> tuple[bytes, str]:
|
||||
item = session.get(ContentItem, item_id)
|
||||
if item is None:
|
||||
raise ValueError("content item not found")
|
||||
hotspot = session.get(Hotspot, item.hotspot_id)
|
||||
comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(["平台", "任务 ID", "热点 ID", "热点标题", "内容条目 ID", "内容条目标题", "评论 ID", "评论内容", "情绪倾向", "方向标签", "点赞数", "评论时间"])
|
||||
for comment in comments:
|
||||
writer.writerow(
|
||||
[
|
||||
item.platform,
|
||||
item.task_id,
|
||||
hotspot.id if hotspot else "",
|
||||
hotspot.title if hotspot else "",
|
||||
item.id,
|
||||
item.title or "",
|
||||
comment.source_comment_id or "",
|
||||
safe_csv_field(comment.content),
|
||||
comment.sentiment,
|
||||
labels_to_text(comment.labels),
|
||||
comment.like_count or 0,
|
||||
comment.comment_time or "",
|
||||
]
|
||||
)
|
||||
filename = safe_filename(item.platform, item.task_id, hotspot.title if hotspot else item.title or "comments")
|
||||
return output.getvalue().encode("utf-8-sig"), filename
|
||||
|
||||
|
||||
def export_hotspot_comments_csv(session: Session, hotspot_id: str) -> tuple[bytes, str]:
|
||||
hotspot = session.get(Hotspot, hotspot_id)
|
||||
if hotspot is None:
|
||||
raise ValueError("hotspot not found")
|
||||
comments = list(session.scalars(select(Comment).where(Comment.hotspot_id == hotspot.id)))
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(["平台", "任务 ID", "热点 ID", "热点标题", "内容条目 ID", "内容条目标题", "评论 ID", "评论内容", "情绪倾向", "方向标签", "点赞数", "评论时间"])
|
||||
for comment in comments:
|
||||
item = session.get(ContentItem, comment.content_item_id)
|
||||
writer.writerow(
|
||||
[
|
||||
comment.platform,
|
||||
comment.task_id,
|
||||
hotspot.id,
|
||||
hotspot.title,
|
||||
item.id if item else "",
|
||||
item.title if item else "",
|
||||
comment.source_comment_id or "",
|
||||
safe_csv_field(comment.content),
|
||||
comment.sentiment,
|
||||
labels_to_text(comment.labels),
|
||||
comment.like_count or 0,
|
||||
comment.comment_time or "",
|
||||
]
|
||||
)
|
||||
return output.getvalue().encode("utf-8-sig"), safe_filename(hotspot.platform, hotspot.task_id, hotspot.title)
|
||||
|
||||
|
||||
def export_report_markdown(session: Session, *, report_type: str, hotspot_id: str | None = None, item_id: str | None = None) -> tuple[str, str]:
|
||||
query = select(Report).where(Report.report_type == report_type)
|
||||
if hotspot_id:
|
||||
query = query.where(Report.hotspot_id == hotspot_id)
|
||||
if item_id:
|
||||
query = query.where(Report.content_item_id == item_id)
|
||||
report = session.scalar(query)
|
||||
if report is None:
|
||||
raise ValueError("report not found")
|
||||
return report.markdown_content or report.markdown, safe_filename("report", report.task_id, report.title, extension="md")
|
||||
@@ -0,0 +1,162 @@
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Callable
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Comment, ContentItem, Hotspot, Report
|
||||
|
||||
|
||||
DEFAULT_SUMMARY = "总结生成失败，请查看上方统计数据。"
|
||||
|
||||
|
||||
def _labels(value: str | None) -> list[str]:
|
||||
try:
|
||||
parsed = json.loads(value or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [str(label) for label in parsed] if isinstance(parsed, list) else []
|
||||
|
||||
|
||||
def build_comment_metrics(comments: list[Comment]) -> dict:
|
||||
sentiment_counts = Counter(comment.sentiment or "unknown" for comment in comments)
|
||||
label_counts = Counter(label for comment in comments for label in _labels(comment.labels))
|
||||
total = len(comments)
|
||||
|
||||
def sentiment_entry(name: str) -> dict:
|
||||
count = sentiment_counts.get(name, 0)
|
||||
return {"count": count, "pct": round((count / total * 100) if total else 0, 2)}
|
||||
|
||||
return {
|
||||
"sample_count": total,
|
||||
"sentiment": {
|
||||
"positive": sentiment_entry("positive"),
|
||||
"negative": sentiment_entry("negative"),
|
||||
"neutral": sentiment_entry("neutral"),
|
||||
"unknown": sentiment_entry("unknown"),
|
||||
},
|
||||
"top_labels": [{"name": name, "count": count} for name, count in label_counts.most_common(5)],
|
||||
}
|
||||
|
||||
|
||||
def select_typical_comments(comments: list[Comment], *, per_sentiment: int = 2) -> dict[str, list[dict]]:
|
||||
buckets: dict[str, list[Comment]] = defaultdict(list)
|
||||
for comment in comments:
|
||||
buckets[comment.sentiment or "unknown"].append(comment)
|
||||
|
||||
result = {}
|
||||
for sentiment, values in buckets.items():
|
||||
sorted_values = sorted(values, key=lambda c: (c.like_count or 0, c.comment_time or c.created_at), reverse=True)
|
||||
result[sentiment] = [
|
||||
{"content": comment.content, "like_count": comment.like_count or 0, "comment_id": comment.source_comment_id}
|
||||
for comment in sorted_values[:per_sentiment]
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
def generate_item_report(
|
||||
session: Session,
|
||||
content_item_id: str,
|
||||
*,
|
||||
summary_provider: Callable[[dict, dict], str] | None = None,
|
||||
) -> Report:
|
||||
item = session.get(ContentItem, content_item_id)
|
||||
if item is None:
|
||||
raise ValueError("content item not found")
|
||||
hotspot = session.get(Hotspot, item.hotspot_id)
|
||||
comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
metrics = build_comment_metrics(comments)
|
||||
typical = select_typical_comments(comments)
|
||||
summary = _safe_summary(summary_provider, metrics, typical, limit=200)
|
||||
markdown = _build_markdown(title=item.title or item.source_item_id, metrics=metrics, typical=typical, summary=summary, hotspot_title=hotspot.title if hotspot else "")
|
||||
report = Report(
|
||||
task_id=item.task_id,
|
||||
hotspot_id=item.hotspot_id,
|
||||
content_item_id=item.id,
|
||||
report_type="item",
|
||||
title=item.title or item.source_item_id,
|
||||
data=json.dumps(metrics, ensure_ascii=False),
|
||||
markdown=markdown,
|
||||
metrics_json=json.dumps(metrics, ensure_ascii=False),
|
||||
typical_comments_json=json.dumps(typical, ensure_ascii=False),
|
||||
summary=summary,
|
||||
markdown_content=markdown,
|
||||
)
|
||||
session.add(report)
|
||||
session.commit()
|
||||
session.refresh(report)
|
||||
return report
|
||||
|
||||
|
||||
def generate_hotspot_report(
|
||||
session: Session,
|
||||
hotspot_id: str,
|
||||
*,
|
||||
summary_provider: Callable[[dict, dict], str] | None = None,
|
||||
) -> Report:
|
||||
hotspot = session.get(Hotspot, hotspot_id)
|
||||
if hotspot is None:
|
||||
raise ValueError("hotspot not found")
|
||||
comments = list(session.scalars(select(Comment).where(Comment.hotspot_id == hotspot.id)))
|
||||
item_count = session.scalar(select(func.count(ContentItem.id)).where(ContentItem.hotspot_id == hotspot.id)) or 0
|
||||
metrics = build_comment_metrics(comments)
|
||||
metrics["item_count"] = item_count
|
||||
typical = select_typical_comments(comments)
|
||||
summary = _safe_summary(summary_provider, metrics, typical, limit=300)
|
||||
markdown = _build_markdown(title=hotspot.title, metrics=metrics, typical=typical, summary=summary)
|
||||
report = Report(
|
||||
task_id=hotspot.task_id,
|
||||
hotspot_id=hotspot.id,
|
||||
content_item_id=None,
|
||||
report_type="hotspot",
|
||||
title=hotspot.title,
|
||||
data=json.dumps(metrics, ensure_ascii=False),
|
||||
markdown=markdown,
|
||||
metrics_json=json.dumps(metrics, ensure_ascii=False),
|
||||
typical_comments_json=json.dumps(typical, ensure_ascii=False),
|
||||
summary=summary,
|
||||
markdown_content=markdown,
|
||||
)
|
||||
session.add(report)
|
||||
session.commit()
|
||||
session.refresh(report)
|
||||
return report
|
||||
|
||||
|
||||
def _safe_summary(summary_provider, metrics: dict, typical: dict, *, limit: int) -> str:
|
||||
if summary_provider is None:
|
||||
return DEFAULT_SUMMARY
|
||||
try:
|
||||
summary = summary_provider(metrics, typical)
|
||||
except Exception:
|
||||
return DEFAULT_SUMMARY
|
||||
return str(summary)[:limit]
|
||||
|
||||
|
||||
def _build_markdown(*, title: str, metrics: dict, typical: dict, summary: str, hotspot_title: str = "") -> str:
|
||||
lines = [
|
||||
f"# {title}",
|
||||
"",
|
||||
]
|
||||
if hotspot_title:
|
||||
lines.extend([f"- 所属热点：{hotspot_title}", ""])
|
||||
lines.extend(
|
||||
[
|
||||
f"- 样本评论数量：{metrics['sample_count']}",
|
||||
"",
|
||||
"## 情绪分布",
|
||||
]
|
||||
)
|
||||
for name in ("positive", "neutral", "negative", "unknown"):
|
||||
item = metrics["sentiment"][name]
|
||||
lines.append(f"- {name}: {item['count']} ({item['pct']}%)")
|
||||
lines.extend(["", "## Top 标签"])
|
||||
for label in metrics["top_labels"]:
|
||||
lines.append(f"- {label['name']}: {label['count']}")
|
||||
lines.extend(["", "## 典型评论"])
|
||||
for sentiment, comments in typical.items():
|
||||
for comment in comments:
|
||||
lines.append(f"- [{sentiment}] {comment['content']}")
|
||||
lines.extend(["", "## 总结", summary])
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,243 @@
|
||||
import inspect
|
||||
import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections.abc import Callable
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Comment, ContentItem, Hotspot, Task
|
||||
from app.platforms.base import PlatformAPIError, TikHubClient
|
||||
from app.platforms.douyin import DouyinPlatform
|
||||
from app.platforms.xiaohongshu import XiaohongshuPlatform
|
||||
from app.schemas import CreateTaskRequest
|
||||
from app.services.ai_service import OpenAICompatibleAIClient, analyze_comments_with_retry, calculate_analysis_status
|
||||
from app.services.report_service import generate_hotspot_report, generate_item_report
|
||||
|
||||
|
||||
RUNNING_TASK_MESSAGE = "当前有正在运行的任务，请稍后再试"
|
||||
RESTART_ERROR_MESSAGE = "系统重启，任务被中断"
|
||||
task_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
|
||||
def has_running_task(session: Session) -> bool:
|
||||
return session.scalar(select(Task.id).where(Task.status == "running").limit(1)) is not None
|
||||
|
||||
|
||||
def recover_running_tasks(session: Session) -> int:
|
||||
tasks = list(session.scalars(select(Task).where(Task.status == "running")))
|
||||
for task in tasks:
|
||||
task.status = "failed"
|
||||
task.error_stage = "system"
|
||||
task.error_type = "unexpected_restart"
|
||||
task.error_message = RESTART_ERROR_MESSAGE
|
||||
session.commit()
|
||||
return len(tasks)
|
||||
|
||||
|
||||
def build_platform(platform: str):
|
||||
settings = get_settings()
|
||||
client = TikHubClient(
|
||||
base_url=settings.tikhub_base_url,
|
||||
api_key=settings.tikhub_api_key,
|
||||
timeout_seconds=settings.http_timeout_seconds,
|
||||
max_retries=settings.http_max_retries,
|
||||
)
|
||||
if platform == "xiaohongshu":
|
||||
return XiaohongshuPlatform(client)
|
||||
if platform == "douyin":
|
||||
return DouyinPlatform(client)
|
||||
raise ValueError(f"Unsupported platform: {platform}")
|
||||
|
||||
|
||||
def build_ai_requester():
|
||||
settings = get_settings()
|
||||
if settings.ai_base_url and settings.ai_api_key and settings.ai_model:
|
||||
client = OpenAICompatibleAIClient(
|
||||
base_url=settings.ai_base_url,
|
||||
api_key=settings.ai_api_key,
|
||||
model=settings.ai_model,
|
||||
timeout_seconds=settings.ai_timeout_seconds,
|
||||
)
|
||||
return client.request
|
||||
|
||||
def fallback_requester(_prompt: str) -> str:
|
||||
return "[]"
|
||||
|
||||
return fallback_requester
|
||||
|
||||
|
||||
def create_task(session: Session, request: CreateTaskRequest, *, submit_background: bool = True) -> Task:
|
||||
task = Task(
|
||||
platform=request.platform,
|
||||
status="running",
|
||||
hotspot_limit=request.hotspot_limit,
|
||||
item_limit_per_hotspot=request.item_limit_per_hotspot,
|
||||
comment_limit_per_item=request.comment_limit_per_item,
|
||||
total_items_count=0,
|
||||
processed_items_count=0,
|
||||
successful_items_count=0,
|
||||
failed_items_count=0,
|
||||
analysis_success_rate=0.0,
|
||||
analysis_status="normal",
|
||||
)
|
||||
session.add(task)
|
||||
session.commit()
|
||||
session.refresh(task)
|
||||
if submit_background:
|
||||
task_executor.submit(run_task, task.id)
|
||||
return task
|
||||
|
||||
|
||||
def list_tasks(session: Session) -> list[Task]:
|
||||
return list(session.scalars(select(Task).order_by(Task.created_at.desc())))
|
||||
|
||||
|
||||
def get_task(session: Session, task_id: str) -> Task | None:
|
||||
return session.get(Task, task_id)
|
||||
|
||||
|
||||
def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionmaker = SessionLocal) -> None:
|
||||
session = session_factory()
|
||||
close_session = hasattr(session, "close")
|
||||
try:
|
||||
task = session.get(Task, task_id)
|
||||
if task is None:
|
||||
return
|
||||
try:
|
||||
platform = build_platform(task.platform)
|
||||
try:
|
||||
hotspots = platform.fetch_hotspots(limit=task.hotspot_limit)
|
||||
except PlatformAPIError as exc:
|
||||
_mark_task_failed(task, "crawl_hotspots", exc.error_type, str(exc))
|
||||
session.commit()
|
||||
return
|
||||
except Exception as exc:
|
||||
_mark_task_failed(task, "crawl_hotspots", "api_error", str(exc))
|
||||
session.commit()
|
||||
return
|
||||
|
||||
for hotspot_data in hotspots:
|
||||
hotspot = Hotspot(
|
||||
task_id=task.id,
|
||||
platform=task.platform,
|
||||
source_hot_id=hotspot_data.source_hot_id,
|
||||
rank=hotspot_data.rank,
|
||||
title=hotspot_data.title,
|
||||
heat_value=hotspot_data.heat_value,
|
||||
raw_data=json.dumps(hotspot_data.raw_data or {}, ensure_ascii=False),
|
||||
)
|
||||
session.add(hotspot)
|
||||
session.flush()
|
||||
|
||||
try:
|
||||
items = platform.search_items_by_hotspot(hotspot.title, limit=task.item_limit_per_hotspot)
|
||||
except Exception as exc:
|
||||
task.failed_items_count += task.item_limit_per_hotspot
|
||||
task.error_stage = "crawl_items"
|
||||
task.error_type = getattr(exc, "error_type", "api_error")
|
||||
task.error_message = str(exc)
|
||||
session.commit()
|
||||
continue
|
||||
|
||||
task.total_items_count += len(items)
|
||||
seen_item_ids: set[str] = set()
|
||||
for item_data in items:
|
||||
if item_data.source_item_id in seen_item_ids:
|
||||
continue
|
||||
seen_item_ids.add(item_data.source_item_id)
|
||||
item = ContentItem(
|
||||
task_id=task.id,
|
||||
hotspot_id=hotspot.id,
|
||||
platform=task.platform,
|
||||
source_item_id=item_data.source_item_id,
|
||||
item_type=item_data.item_type,
|
||||
title=item_data.title,
|
||||
summary=item_data.summary,
|
||||
url=item_data.url,
|
||||
status="pending",
|
||||
raw_data=json.dumps(item_data.raw_data or {}, ensure_ascii=False),
|
||||
)
|
||||
session.add(item)
|
||||
session.flush()
|
||||
try:
|
||||
comments = platform.fetch_comments(item.source_item_id, limit=task.comment_limit_per_item)
|
||||
for comment_data in comments:
|
||||
session.add(
|
||||
Comment(
|
||||
task_id=task.id,
|
||||
hotspot_id=hotspot.id,
|
||||
content_item_id=item.id,
|
||||
platform=task.platform,
|
||||
source_comment_id=comment_data.source_comment_id,
|
||||
content=comment_data.content,
|
||||
author=comment_data.author,
|
||||
like_count=comment_data.like_count,
|
||||
comment_time=comment_data.comment_time,
|
||||
raw_data=json.dumps(comment_data.raw_data or {}, ensure_ascii=False),
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
ai_results = analyze_comments_with_retry(
|
||||
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
|
||||
requester=build_ai_requester(),
|
||||
max_retries=get_settings().ai_max_retries,
|
||||
)
|
||||
results_by_comment_id = {result.comment_id: result for result in ai_results}
|
||||
for comment in item_comments:
|
||||
result = results_by_comment_id.get(comment.id)
|
||||
if result is None:
|
||||
comment.sentiment = "unknown"
|
||||
comment.labels = "[]"
|
||||
comment.reason = "ai_missing_result"
|
||||
comment.ai_analysis_status = "failed"
|
||||
continue
|
||||
comment.sentiment = result.sentiment
|
||||
comment.labels = json.dumps(result.labels, ensure_ascii=False)
|
||||
comment.reason = result.reason
|
||||
comment.ai_analysis_status = result.ai_analysis_status
|
||||
item.status = "success"
|
||||
task.successful_items_count += 1
|
||||
generate_item_report(session, item.id)
|
||||
except Exception as exc:
|
||||
item.status = "failed"
|
||||
item.error_stage = "crawl_comments"
|
||||
item.error_type = getattr(exc, "error_type", "api_error")
|
||||
item.error_message = str(exc)
|
||||
task.failed_items_count += 1
|
||||
task.error_stage = item.error_stage
|
||||
task.error_type = item.error_type
|
||||
task.error_message = item.error_message
|
||||
finally:
|
||||
task.processed_items_count += 1
|
||||
session.commit()
|
||||
|
||||
if task.successful_items_count > 0:
|
||||
total_comments = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||||
success_comments = sum(1 for comment in task.comments if comment.ai_analysis_status == "success")
|
||||
task.analysis_success_rate, task.analysis_status = calculate_analysis_status(
|
||||
success_count=success_comments,
|
||||
total_count=total_comments,
|
||||
)
|
||||
for hotspot in task.hotspots:
|
||||
generate_hotspot_report(session, hotspot.id)
|
||||
task.status = "success"
|
||||
else:
|
||||
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
|
||||
session.commit()
|
||||
except Exception as exc:
|
||||
_mark_task_failed(task, "system", "unexpected_error", str(exc))
|
||||
session.commit()
|
||||
finally:
|
||||
if close_session:
|
||||
session.close()
|
||||
|
||||
|
||||
def _mark_task_failed(task: Task, stage: str, error_type: str, message: str) -> None:
|
||||
task.status = "failed"
|
||||
task.error_stage = stage
|
||||
task.error_type = error_type
|
||||
task.error_message = message
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
body {
|
||||
background: #f8f9fa;
|
||||
}
|
||||
|
||||
.card {
|
||||
border-radius: 8px;
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
function readInt(id) {
|
||||
return parseInt(document.getElementById(id).value, 10) || 0;
|
||||
}
|
||||
|
||||
function updateScaleHint() {
|
||||
const total = readInt("hotspot_limit") * readInt("item_limit_per_hotspot") * readInt("comment_limit_per_item");
|
||||
const target = document.getElementById("scale-calc");
|
||||
if (target) target.textContent = total.toLocaleString();
|
||||
}
|
||||
|
||||
function validateRange(input) {
|
||||
const value = readInt(input.id);
|
||||
const min = parseInt(input.min, 10);
|
||||
const max = parseInt(input.max, 10);
|
||||
const valid = value >= min && value <= max;
|
||||
input.classList.toggle("is-invalid", !valid);
|
||||
return valid;
|
||||
}
|
||||
|
||||
async function submitTask(event) {
|
||||
event.preventDefault();
|
||||
const btn = document.getElementById("submit-btn");
|
||||
const errorBox = document.getElementById("form-error");
|
||||
const inputs = Array.from(document.querySelectorAll(".scale-input"));
|
||||
if (!inputs.every(validateRange)) {
|
||||
errorBox.textContent = "参数超出范围，请检查配置。";
|
||||
errorBox.classList.remove("d-none");
|
||||
return;
|
||||
}
|
||||
|
||||
btn.disabled = true;
|
||||
btn.textContent = "提交中...";
|
||||
errorBox.classList.add("d-none");
|
||||
const payload = {
|
||||
platform: document.getElementById("platform").value,
|
||||
hotspot_limit: readInt("hotspot_limit"),
|
||||
item_limit_per_hotspot: readInt("item_limit_per_hotspot"),
|
||||
comment_limit_per_item: readInt("comment_limit_per_item"),
|
||||
};
|
||||
|
||||
try {
|
||||
const resp = await fetch("/api/tasks", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
if (resp.ok) {
|
||||
const data = await resp.json();
|
||||
window.location.href = `/tasks/${data.task_id}`;
|
||||
return;
|
||||
}
|
||||
const data = await resp.json();
|
||||
errorBox.textContent = resp.status === 422 ? "参数错误，请检查配置。" : (data.detail || "服务器错误，请稍后重试。");
|
||||
errorBox.classList.remove("d-none");
|
||||
} catch (_error) {
|
||||
errorBox.textContent = "网络异常，请检查连接后重试。";
|
||||
errorBox.classList.remove("d-none");
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = "开始抓取";
|
||||
}
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", () => {
|
||||
document.querySelectorAll(".scale-input").forEach((input) => {
|
||||
input.addEventListener("input", () => {
|
||||
validateRange(input);
|
||||
updateScaleHint();
|
||||
});
|
||||
});
|
||||
updateScaleHint();
|
||||
});
|
||||
|
||||
async function downloadExport(url, defaultFilename, event) {
|
||||
if (event) event.preventDefault();
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) {
|
||||
alert("导出失败，请稍后重试。");
|
||||
return;
|
||||
}
|
||||
const blob = await resp.blob();
|
||||
const disposition = resp.headers.get("Content-Disposition") || "";
|
||||
const match = disposition.match(/filename\*=UTF-8''([^;]+)/);
|
||||
const filename = match ? decodeURIComponent(match[1]) : defaultFilename;
|
||||
const blobUrl = URL.createObjectURL(blob);
|
||||
const link = document.createElement("a");
|
||||
link.href = blobUrl;
|
||||
link.download = filename;
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{% block title %}热榜评论分析工具{% endblock %}</title>
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link href="/static/app.css" rel="stylesheet">
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="/static/app.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="navbar navbar-expand-lg bg-body-tertiary border-bottom">
|
||||
<div class="container">
|
||||
<a class="navbar-brand" href="/">热榜评论分析工具</a>
|
||||
<div class="navbar-nav">
|
||||
<a class="nav-link" href="/">任务列表</a>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
<main class="container py-4">
|
||||
{% block breadcrumbs %}
|
||||
<nav aria-label="breadcrumb">
|
||||
<ol class="breadcrumb">
|
||||
<li class="breadcrumb-item active" aria-current="page">首页</li>
|
||||
</ol>
|
||||
</nav>
|
||||
{% endblock %}
|
||||
{% block content %}{% endblock %}
|
||||
</main>
|
||||
<footer class="border-top py-3">
|
||||
<div class="container text-muted small">内部演示工具 | 仅供学习参考</div>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,26 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{{ hotspot.title }} 汇总报告 - 热榜评论分析工具{% endblock %}
|
||||
{% block breadcrumbs %}
|
||||
<nav aria-label="breadcrumb"><ol class="breadcrumb">
|
||||
<li class="breadcrumb-item"><a href="/">首页</a></li>
|
||||
<li class="breadcrumb-item"><a href="/tasks/{{ task.id }}">任务 #{{ task.id }}</a></li>
|
||||
<li class="breadcrumb-item active">汇总报告</li>
|
||||
</ol></nav>
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h1 class="h3 mb-0">{{ hotspot.title }} 汇总报告</h1>
|
||||
<div class="d-flex gap-2">
|
||||
<a class="btn btn-outline-primary" href="/api/export/hotspots/{{ hotspot.id }}.md">导出 Markdown 报告</a>
|
||||
<a class="btn btn-outline-secondary" href="/api/export/hotspots/{{ hotspot.id }}/comments.csv">导出热点评论 CSV</a>
|
||||
</div>
|
||||
</div>
|
||||
{% if report %}
|
||||
<section class="card"><div class="card-body">
|
||||
<p class="lead">{{ report.summary }}</p>
|
||||
<pre class="bg-light p-3">{{ report.markdown_content }}</pre>
|
||||
</div></section>
|
||||
{% else %}
|
||||
<div class="alert alert-secondary">报告生成中，请稍候...</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,66 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}任务列表 - 热榜评论分析工具{% endblock %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h1 class="h3 mb-0">任务列表</h1>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button>
|
||||
</div>
|
||||
|
||||
<section class="card mb-4">
|
||||
<div class="card-header">创建抓取任务</div>
|
||||
<div class="card-body">
|
||||
<div id="form-error" class="alert alert-danger d-none" role="alert"></div>
|
||||
<form id="task-form" onsubmit="submitTask(event)">
|
||||
<div class="row g-3 align-items-end">
|
||||
<div class="col-md-3">
|
||||
<label class="form-label" for="platform">平台</label>
|
||||
<select class="form-select" id="platform" name="platform">
|
||||
<option value="xiaohongshu">小红书</option>
|
||||
<option value="douyin">抖音</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="col-md-2">
|
||||
<label class="form-label" for="hotspot_limit">热点数</label>
|
||||
<input class="form-control scale-input" id="hotspot_limit" name="hotspot_limit" type="number" min="1" max="10" value="5">
|
||||
</div>
|
||||
<div class="col-md-2">
|
||||
<label class="form-label" for="item_limit_per_hotspot">每热点内容</label>
|
||||
<input class="form-control scale-input" id="item_limit_per_hotspot" name="item_limit_per_hotspot" type="number" min="1" max="10" value="5">
|
||||
</div>
|
||||
<div class="col-md-2">
|
||||
<label class="form-label" for="comment_limit_per_item">每内容评论</label>
|
||||
<input class="form-control scale-input" id="comment_limit_per_item" name="comment_limit_per_item" type="number" min="10" max="100" value="50">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<button class="btn btn-primary w-100" id="submit-btn" type="submit">开始抓取</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-text text-muted mt-2" id="scale-hint">
|
||||
预计最多抓取：<strong id="scale-calc">1250</strong> 条评论（实际数量可能受平台返回数量、去重、失败、限流影响）
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card">
|
||||
<div class="card-header">历史任务</div>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>任务 ID</th>
|
||||
<th>平台</th>
|
||||
<th>创建时间</th>
|
||||
<th>配置规模</th>
|
||||
<th>进度</th>
|
||||
<th>状态</th>
|
||||
<th>操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="task-table-body">
|
||||
{% include "partials/task_rows.html" %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,34 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{{ item.title or item.source_item_id }} 详情 - 热榜评论分析工具{% endblock %}
|
||||
{% block breadcrumbs %}
|
||||
<nav aria-label="breadcrumb"><ol class="breadcrumb">
|
||||
<li class="breadcrumb-item"><a href="/">首页</a></li>
|
||||
<li class="breadcrumb-item"><a href="/tasks/{{ task.id }}">任务 #{{ task.id }}</a></li>
|
||||
<li class="breadcrumb-item"><a href="/hotspots/{{ hotspot.id }}/report">热点 #{{ hotspot.rank }}：{{ hotspot.title }}</a></li>
|
||||
<li class="breadcrumb-item active">{{ item.title or item.source_item_id }}</li>
|
||||
</ol></nav>
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h1 class="h3 mb-0">{{ item.title or item.source_item_id }}</h1>
|
||||
<div class="d-flex gap-2">
|
||||
<a class="btn btn-outline-primary" href="/api/export/items/{{ item.id }}.md">导出 Markdown 报告</a>
|
||||
<a class="btn btn-outline-secondary" href="/api/export/items/{{ item.id }}/comments.csv">导出评论 CSV</a>
|
||||
</div>
|
||||
</div>
|
||||
{% if report %}
|
||||
<section class="card mb-4"><div class="card-body">
|
||||
<p class="lead">{{ report.summary }}</p>
|
||||
<pre class="bg-light p-3">{{ report.markdown_content }}</pre>
|
||||
</div></section>
|
||||
{% endif %}
|
||||
<section class="card"><div class="card-header">评论明细</div><div class="table-responsive">
|
||||
<table class="table mb-0"><thead><tr><th>评论内容</th><th>情绪</th><th>标签</th><th>点赞</th></tr></thead><tbody>
|
||||
{% for comment in comments %}
|
||||
<tr><td>{{ comment.content }}</td><td>{{ comment.sentiment }}</td><td>{% for label in comment.labels | from_json %}<span class="badge bg-secondary me-1">{{ label }}</span>{% else %}-{% endfor %}</td><td>{{ comment.like_count or 0 }}</td></tr>
|
||||
{% else %}
|
||||
<tr><td colspan="4" class="text-center text-muted">暂无评论数据（该内容无评论或评论抓取为空）</td></tr>
|
||||
{% endfor %}
|
||||
</tbody></table>
|
||||
</div></section>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,7 @@
|
||||
{% macro label_tags(labels_json) %}
|
||||
{% for label in labels_json | from_json %}
|
||||
<span class="badge bg-secondary me-1">{{ label }}</span>
|
||||
{% else %}
|
||||
<span class="text-muted">-</span>
|
||||
{% endfor %}
|
||||
{% endmacro %}
|
||||
@@ -0,0 +1,10 @@
|
||||
{% macro sentiment_badge(sentiment) %}
|
||||
{% set config = {
|
||||
"positive": ("正向", "bg-success"),
|
||||
"neutral": ("中性", "bg-warning text-dark"),
|
||||
"negative": ("负向", "bg-danger"),
|
||||
"unknown": ("未知", "bg-secondary"),
|
||||
} %}
|
||||
{% set label, style = config.get(sentiment, ("未知", "bg-secondary")) %}
|
||||
<span class="badge {{ style }}">{{ label }}</span>
|
||||
{% endmacro %}
|
||||
@@ -0,0 +1,4 @@
|
||||
{% macro status_badge(status) %}
|
||||
{% set label, style = status_badge_config(status) %}
|
||||
<span class="badge {{ style }}">{{ label }}</span>
|
||||
{% endmacro %}
|
||||
@@ -0,0 +1,21 @@
|
||||
{% for task in tasks %}
|
||||
{% set status_label, status_class = status_badge_config(task.status) %}
|
||||
<tr>
|
||||
<td><code>{{ task.id }}</code></td>
|
||||
<td>{{ task.platform | platform_label }}</td>
|
||||
<td>{{ task.created_at }}</td>
|
||||
<td><small class="text-muted">热点 {{ task.hotspot_limit }} / 内容 {{ task.item_limit_per_hotspot }} / 评论 {{ task.comment_limit_per_item }}</small></td>
|
||||
<td>{{ task.successful_items_count }} / {{ task.total_items_count }}</td>
|
||||
<td>
|
||||
<span class="badge {{ status_class }}">{{ status_label }}</span>
|
||||
{% if task.error_stage or task.error_type %}
|
||||
<br><small class="text-muted">{{ task.error_stage }} / {{ task.error_type }}</small>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td><a class="btn btn-sm btn-outline-primary" href="/tasks/{{ task.id }}">查看</a></td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr>
|
||||
<td colspan="7" class="text-center text-muted py-4">还没有任何任务，请在上方创建第一个任务</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,47 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}任务 #{{ task.id }} - 热榜评论分析工具{% endblock %}
|
||||
{% block breadcrumbs %}
|
||||
<nav aria-label="breadcrumb"><ol class="breadcrumb">
|
||||
<li class="breadcrumb-item"><a href="/">首页</a></li>
|
||||
<li class="breadcrumb-item active">任务 #{{ task.id }}</li>
|
||||
</ol></nav>
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h1 class="h3 mb-0">任务 #{{ task.id }}</h1>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button>
|
||||
</div>
|
||||
<section class="card mb-4"><div class="card-body">
|
||||
{% set label, cls = status_badge_config(task.status) %}
|
||||
<p>平台：{{ task.platform | platform_label }} <span class="badge {{ cls }}">{{ label }}</span></p>
|
||||
<p>成功 {{ task.successful_items_count }} / 共 {{ task.total_items_count }} 条内容</p>
|
||||
{% if task.error_message %}<div class="alert alert-danger">{{ task.error_stage }} / {{ task.error_type }}：{{ task.error_message }}</div>{% endif %}
|
||||
</div></section>
|
||||
{% if task.status == "running" and not hotspots %}
|
||||
<div class="text-center text-muted py-5"><div class="spinner-border text-warning mb-3"></div><p>正在抓取热点数据，请稍候...</p></div>
|
||||
{% endif %}
|
||||
<div class="accordion" id="hotspot-list">
|
||||
{% for hotspot in hotspots %}
|
||||
<div class="accordion-item">
|
||||
<h2 class="accordion-header">
|
||||
<button class="accordion-button {% if not loop.first %}collapsed{% endif %}" type="button" data-bs-toggle="collapse" data-bs-target="#hotspot-{{ hotspot.id }}">
|
||||
热点 #{{ hotspot.rank }}：{{ hotspot.title }}
|
||||
</button>
|
||||
</h2>
|
||||
<div id="hotspot-{{ hotspot.id }}" class="accordion-collapse collapse {% if loop.first %}show{% endif %}" data-bs-parent="#hotspot-list">
|
||||
<div class="accordion-body">
|
||||
<a class="btn btn-sm btn-outline-primary mb-2" href="/hotspots/{{ hotspot.id }}/report">查看热点级汇总报告</a>
|
||||
<ul class="list-group">
|
||||
{% for item in hotspot.content_items %}
|
||||
<li class="list-group-item d-flex justify-content-between">
|
||||
<span>{{ item.title or item.summary or item.source_item_id }} <small class="text-muted">{{ item.status }}</small></span>
|
||||
<a class="btn btn-sm btn-outline-secondary" href="/items/{{ item.id }}">查看详情</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,33 @@
|
||||
import json
|
||||
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
|
||||
templates = Jinja2Templates(directory="app/templates")
|
||||
|
||||
|
||||
def status_badge_config(status: str) -> tuple[str, str]:
|
||||
config = {
|
||||
"pending": ("等待中", "bg-secondary"),
|
||||
"running": ("运行中", "bg-warning text-dark"),
|
||||
"success": ("已完成", "bg-success"),
|
||||
"failed": ("失败", "bg-danger"),
|
||||
}
|
||||
return config.get(status, ("未知", "bg-secondary"))
|
||||
|
||||
|
||||
def platform_label(platform: str) -> str:
|
||||
return {"xiaohongshu": "小红书", "douyin": "抖音"}.get(platform, platform)
|
||||
|
||||
|
||||
def from_json_filter(value: str | None) -> list:
|
||||
try:
|
||||
parsed = json.loads(value) if value else []
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return []
|
||||
return parsed if isinstance(parsed, list) else []
|
||||
|
||||
|
||||
templates.env.globals["status_badge_config"] = status_badge_config
|
||||
templates.env.filters["platform_label"] = platform_label
|
||||
templates.env.filters["from_json"] = from_json_filter
|
||||
Reference in New Issue
Block a user