feat: 完善 MVP-2 演示和进度体验
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from shutil import copy2
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Engine, create_engine, event
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
@@ -30,6 +32,16 @@ def create_sqlite_engine(database_url: str):
|
||||
return engine
|
||||
|
||||
|
||||
def sqlite_file_path(target_engine: Engine = None) -> Path | None:
|
||||
target_engine = target_engine or engine
|
||||
if target_engine.url.get_backend_name() != "sqlite":
|
||||
return None
|
||||
database = target_engine.url.database
|
||||
if not database or database == ":memory:":
|
||||
return None
|
||||
return Path(database)
|
||||
|
||||
|
||||
engine = create_sqlite_engine(get_settings().database_url)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
@@ -63,10 +75,39 @@ def checkpoint_sqlite_wal(target_engine: Engine = engine) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def ensure_sqlite_schema_compat(target_engine: Engine = engine) -> None:
|
||||
if target_engine.url.get_backend_name() != "sqlite":
|
||||
return
|
||||
with target_engine.begin() as connection:
|
||||
task_columns = {row[1] for row in connection.exec_driver_sql("PRAGMA table_info(tasks)").all()}
|
||||
if "current_stage" not in task_columns:
|
||||
connection.exec_driver_sql("ALTER TABLE tasks ADD COLUMN current_stage VARCHAR(64)")
|
||||
if "last_progress_at" not in task_columns:
|
||||
connection.exec_driver_sql("ALTER TABLE tasks ADD COLUMN last_progress_at DATETIME")
|
||||
|
||||
|
||||
def backup_sqlite_files(target_engine: Engine = engine) -> list[Path]:
|
||||
db_path = sqlite_file_path(target_engine)
|
||||
if db_path is None:
|
||||
return []
|
||||
backup_dir = db_path.parent / "corrupt-backups"
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
copied: list[Path] = []
|
||||
for source in [db_path, db_path.with_name(f"{db_path.name}-wal"), db_path.with_name(f"{db_path.name}-shm")]:
|
||||
if not source.exists():
|
||||
continue
|
||||
target = backup_dir / f"{source.name}.{timestamp}.bak"
|
||||
copy2(source, target)
|
||||
copied.append(target)
|
||||
return copied
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
import app.models # noqa: F401
|
||||
|
||||
Base.metadata.create_all(engine)
|
||||
ensure_sqlite_schema_compat(engine)
|
||||
check_database_integrity(engine)
|
||||
checkpoint_sqlite_wal(engine)
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal, init_db
|
||||
from app.models import Comment, ContentItem, Hotspot, Report, Task, utc_now
|
||||
|
||||
|
||||
FIXTURE_PATH = Path(__file__).resolve().parent / "fixtures" / "demo_seed.json"
|
||||
|
||||
|
||||
def seed_demo_data(session: Session, fixture_path: Path = FIXTURE_PATH) -> str:
|
||||
data = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
task_data = data["task"]
|
||||
task_id = task_data["id"]
|
||||
existing = session.scalar(select(Task.id).where(Task.id == task_id))
|
||||
if existing:
|
||||
return task_id
|
||||
|
||||
task = Task(
|
||||
id=task_id,
|
||||
platform=task_data["platform"],
|
||||
status="success",
|
||||
current_stage="success",
|
||||
last_progress_at=utc_now(),
|
||||
hotspot_limit=task_data["hotspot_limit"],
|
||||
item_limit_per_hotspot=task_data["item_limit_per_hotspot"],
|
||||
comment_limit_per_item=task_data["comment_limit_per_item"],
|
||||
total_items_count=len(data["items"]),
|
||||
processed_items_count=len(data["items"]),
|
||||
successful_items_count=len(data["items"]),
|
||||
failed_items_count=0,
|
||||
analysis_success_rate=1.0,
|
||||
analysis_status="normal",
|
||||
)
|
||||
session.add(task)
|
||||
|
||||
for hotspot_data in data["hotspots"]:
|
||||
session.add(
|
||||
Hotspot(
|
||||
id=hotspot_data["id"],
|
||||
task_id=task_id,
|
||||
platform=task.platform,
|
||||
rank=hotspot_data["rank"],
|
||||
title=hotspot_data["title"],
|
||||
heat_value=hotspot_data.get("heat_value"),
|
||||
source_hot_id=None,
|
||||
raw_data="{}",
|
||||
)
|
||||
)
|
||||
|
||||
for item_data in data["items"]:
|
||||
session.add(
|
||||
ContentItem(
|
||||
id=item_data["id"],
|
||||
task_id=task_id,
|
||||
hotspot_id=item_data["hotspot_id"],
|
||||
platform=task.platform,
|
||||
source_item_id=f"demo-item-{item_data['id']}",
|
||||
item_type=item_data["item_type"],
|
||||
title=item_data["title"],
|
||||
summary=item_data.get("summary"),
|
||||
url=None,
|
||||
status="success",
|
||||
raw_data="{}",
|
||||
)
|
||||
)
|
||||
|
||||
for comment_data in data["comments"]:
|
||||
session.add(
|
||||
Comment(
|
||||
id=comment_data["id"],
|
||||
task_id=task_id,
|
||||
hotspot_id=comment_data["hotspot_id"],
|
||||
content_item_id=comment_data["content_item_id"],
|
||||
platform=task.platform,
|
||||
source_comment_id=None,
|
||||
content=comment_data["content"],
|
||||
author=None,
|
||||
like_count=comment_data.get("like_count", 0),
|
||||
sentiment=comment_data["sentiment"],
|
||||
labels=json.dumps(comment_data["labels"], ensure_ascii=False),
|
||||
reason=comment_data.get("reason"),
|
||||
ai_analysis_status="success",
|
||||
raw_data="{}",
|
||||
ai_raw_response=None,
|
||||
)
|
||||
)
|
||||
|
||||
for report_data in data["reports"]:
|
||||
session.add(
|
||||
Report(
|
||||
task_id=task_id,
|
||||
hotspot_id=report_data.get("hotspot_id"),
|
||||
content_item_id=report_data.get("content_item_id"),
|
||||
report_type=report_data["report_type"],
|
||||
title=report_data["title"],
|
||||
metrics_json=json.dumps(report_data["metrics"], ensure_ascii=False),
|
||||
typical_comments_json=json.dumps(report_data["typical_comments"], ensure_ascii=False),
|
||||
summary=report_data["summary"],
|
||||
markdown_content=report_data["markdown_content"],
|
||||
data="{}",
|
||||
markdown=report_data["markdown_content"],
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
return task_id
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
with SessionLocal() as session:
|
||||
task_id = seed_demo_data(session)
|
||||
print(f"Seeded demo task: {task_id}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"task": {
|
||||
"id": "demo-douyin-20260703",
|
||||
"platform": "douyin",
|
||||
"hotspot_limit": 1,
|
||||
"item_limit_per_hotspot": 2,
|
||||
"comment_limit_per_item": 10
|
||||
},
|
||||
"hotspots": [
|
||||
{
|
||||
"id": "demo-hotspot-1",
|
||||
"rank": 1,
|
||||
"title": "演示热点:夏季新品讨论",
|
||||
"heat_value": "demo"
|
||||
}
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"id": "demo-item-1",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"item_type": "video",
|
||||
"title": "新品开箱体验",
|
||||
"summary": "演示用脱敏内容条目"
|
||||
},
|
||||
{
|
||||
"id": "demo-item-2",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"item_type": "video",
|
||||
"title": "用户上手反馈",
|
||||
"summary": "演示用脱敏内容条目"
|
||||
}
|
||||
],
|
||||
"comments": [
|
||||
{
|
||||
"id": "demo-comment-1",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-1",
|
||||
"content": "这个颜色很清爽,夏天用看起来挺舒服。",
|
||||
"sentiment": "positive",
|
||||
"labels": ["外观种草", "季节场景"],
|
||||
"reason": "用户表达了对外观和使用场景的认可。",
|
||||
"like_count": 36
|
||||
},
|
||||
{
|
||||
"id": "demo-comment-2",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-1",
|
||||
"content": "价格如果能再低一点就好了,现在有点观望。",
|
||||
"sentiment": "neutral",
|
||||
"labels": ["价格观望", "购买决策"],
|
||||
"reason": "用户没有否定产品,但对价格仍有顾虑。",
|
||||
"like_count": 21
|
||||
},
|
||||
{
|
||||
"id": "demo-comment-3",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-2",
|
||||
"content": "看完真实上手比广告图可信,想看看长期使用反馈。",
|
||||
"sentiment": "positive",
|
||||
"labels": ["真实体验", "长期反馈"],
|
||||
"reason": "用户认可真实体验内容,但仍希望补充长期反馈。",
|
||||
"like_count": 18
|
||||
},
|
||||
{
|
||||
"id": "demo-comment-4",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-2",
|
||||
"content": "评论里好多人问链接,说明种草效果还是挺明显的。",
|
||||
"sentiment": "positive",
|
||||
"labels": ["求购买链接", "种草效果"],
|
||||
"reason": "用户从评论行为判断内容有转化潜力。",
|
||||
"like_count": 12
|
||||
}
|
||||
],
|
||||
"reports": [
|
||||
{
|
||||
"report_type": "hotspot",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"title": "演示热点:夏季新品讨论",
|
||||
"summary": "该热点评论以正向反馈为主,用户主要关注外观、真实体验、价格和购买链接。价格仍是部分用户从种草到购买之间的关键阻力。",
|
||||
"metrics": {
|
||||
"sample_count": 4,
|
||||
"item_count": 2,
|
||||
"sentiment": {
|
||||
"positive": {"count": 3, "pct": 75},
|
||||
"neutral": {"count": 1, "pct": 25},
|
||||
"negative": {"count": 0, "pct": 0},
|
||||
"unknown": {"count": 0, "pct": 0}
|
||||
},
|
||||
"top_labels": [
|
||||
{"name": "外观种草", "count": 1},
|
||||
{"name": "价格观望", "count": 1},
|
||||
{"name": "真实体验", "count": 1},
|
||||
{"name": "求购买链接", "count": 1}
|
||||
]
|
||||
},
|
||||
"typical_comments": {
|
||||
"positive": [{"content": "这个颜色很清爽,夏天用看起来挺舒服。", "like_count": 36}],
|
||||
"neutral": [{"content": "价格如果能再低一点就好了,现在有点观望。", "like_count": 21}],
|
||||
"negative": []
|
||||
},
|
||||
"markdown_content": "# 演示热点:夏季新品讨论\n\n## 总结\n\n该热点评论以正向反馈为主,用户主要关注外观、真实体验、价格和购买链接。价格仍是部分用户从种草到购买之间的关键阻力。"
|
||||
},
|
||||
{
|
||||
"report_type": "item",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-1",
|
||||
"title": "新品开箱体验",
|
||||
"summary": "该内容主要激发了外观和季节场景兴趣,同时价格仍影响部分用户的购买决策。",
|
||||
"metrics": {
|
||||
"sample_count": 2,
|
||||
"sentiment": {
|
||||
"positive": {"count": 1, "pct": 50},
|
||||
"neutral": {"count": 1, "pct": 50},
|
||||
"negative": {"count": 0, "pct": 0},
|
||||
"unknown": {"count": 0, "pct": 0}
|
||||
},
|
||||
"top_labels": [
|
||||
{"name": "外观种草", "count": 1},
|
||||
{"name": "价格观望", "count": 1}
|
||||
]
|
||||
},
|
||||
"typical_comments": {
|
||||
"positive": [{"content": "这个颜色很清爽,夏天用看起来挺舒服。", "like_count": 36}],
|
||||
"neutral": [{"content": "价格如果能再低一点就好了,现在有点观望。", "like_count": 21}],
|
||||
"negative": []
|
||||
},
|
||||
"markdown_content": "# 新品开箱体验\n\n## 总结\n\n该内容主要激发了外观和季节场景兴趣,同时价格仍影响部分用户的购买决策。"
|
||||
},
|
||||
{
|
||||
"report_type": "item",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-2",
|
||||
"title": "用户上手反馈",
|
||||
"summary": "该内容的评论更关注真实体验和后续转化,用户希望看到长期反馈,也表现出明显的购买链接需求。",
|
||||
"metrics": {
|
||||
"sample_count": 2,
|
||||
"sentiment": {
|
||||
"positive": {"count": 2, "pct": 100},
|
||||
"neutral": {"count": 0, "pct": 0},
|
||||
"negative": {"count": 0, "pct": 0},
|
||||
"unknown": {"count": 0, "pct": 0}
|
||||
},
|
||||
"top_labels": [
|
||||
{"name": "真实体验", "count": 1},
|
||||
{"name": "求购买链接", "count": 1}
|
||||
]
|
||||
},
|
||||
"typical_comments": {
|
||||
"positive": [{"content": "看完真实上手比广告图可信,想看看长期使用反馈。", "like_count": 18}],
|
||||
"neutral": [],
|
||||
"negative": []
|
||||
},
|
||||
"markdown_content": "# 用户上手反馈\n\n## 总结\n\n该内容的评论更关注真实体验和后续转化,用户希望看到长期反馈,也表现出明显的购买链接需求。"
|
||||
}
|
||||
]
|
||||
}
|
||||
+33
-2
@@ -1,15 +1,17 @@
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
import logging
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException, Request, status
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from fastapi.responses import Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import OperationalError, SQLAlchemyError
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal, get_db_session, init_db
|
||||
from app.db import SessionLocal, backup_sqlite_files, get_db_session, init_db
|
||||
from app.models import Comment, ContentItem, Hotspot, Report
|
||||
from app.schemas import CreateTaskRequest, CreateTaskResponse, TaskResponse
|
||||
from app.services.export_service import export_hotspot_comments_csv, export_item_comments_csv, export_report_markdown
|
||||
@@ -24,6 +26,25 @@ from app.services.task_service import (
|
||||
from app.templating import templates
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
DATABASE_UNAVAILABLE_MESSAGE = "数据库暂时不可用,请稍后重试或联系维护者恢复数据。"
|
||||
|
||||
|
||||
def handle_database_error(request: Request, exc: SQLAlchemyError):
|
||||
try:
|
||||
backup_sqlite_files()
|
||||
except Exception:
|
||||
logger.exception("Failed to back up SQLite files after database error")
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(status_code=503, content={"detail": DATABASE_UNAVAILABLE_MESSAGE})
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"errors/database_unavailable.html",
|
||||
{"message": DATABASE_UNAVAILABLE_MESSAGE, "detail": str(exc)},
|
||||
status_code=503,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncGenerator[None]:
|
||||
init_db()
|
||||
@@ -36,6 +57,16 @@ app = FastAPI(title="热榜评论分析工具", lifespan=lifespan)
|
||||
app.mount("/static", StaticFiles(directory="app/static"), name="static")
|
||||
|
||||
|
||||
@app.exception_handler(OperationalError)
|
||||
def operational_error_handler(request: Request, exc: OperationalError):
|
||||
return handle_database_error(request, exc)
|
||||
|
||||
|
||||
@app.exception_handler(SQLAlchemyError)
|
||||
def sqlalchemy_error_handler(request: Request, exc: SQLAlchemyError):
|
||||
return handle_database_error(request, exc)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
@@ -37,6 +37,8 @@ class Task(Base):
|
||||
processed_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
successful_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
current_stage: Mapped[str | None] = mapped_column(String(64))
|
||||
last_progress_at: Mapped[datetime | None] = mapped_column()
|
||||
error_stage: Mapped[str | None] = mapped_column(String(64))
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
@@ -31,6 +31,12 @@ class TaskResponse(BaseModel):
|
||||
processed_items_count: int
|
||||
successful_items_count: int
|
||||
failed_items_count: int
|
||||
current_stage: str | None = None
|
||||
current_stage_label: str | None = None
|
||||
last_progress_at: datetime | None = None
|
||||
comments_count: int = 0
|
||||
reports_count: int = 0
|
||||
is_demo: bool = False
|
||||
error_message: str | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Comment, ContentItem, Hotspot, Task
|
||||
from app.models import Comment, ContentItem, Hotspot, Report, Task, utc_now
|
||||
from app.platforms.base import PlatformAPIError, TikHubClient
|
||||
from app.platforms.douyin import DouyinPlatform
|
||||
from app.platforms.xiaohongshu import XiaohongshuPlatform
|
||||
@@ -22,6 +22,18 @@ RESTART_ERROR_MESSAGE = "系统重启,任务被中断"
|
||||
task_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
|
||||
STAGE_LABELS = {
|
||||
"queued": "等待启动",
|
||||
"crawl_hotspots": "获取热点中",
|
||||
"search_items": "搜索内容中",
|
||||
"crawl_comments": "抓取评论中",
|
||||
"ai_analysis": "AI 分析中",
|
||||
"generate_reports": "生成报告中",
|
||||
"success": "已完成",
|
||||
"failed": "失败",
|
||||
}
|
||||
|
||||
|
||||
def has_running_task(session: Session) -> bool:
|
||||
return session.scalar(select(Task.id).where(Task.status == "running").limit(1)) is not None
|
||||
|
||||
@@ -110,6 +122,8 @@ def create_task(session: Session, request: CreateTaskRequest, *, submit_backgrou
|
||||
failed_items_count=0,
|
||||
analysis_success_rate=0.0,
|
||||
analysis_status="normal",
|
||||
current_stage="queued",
|
||||
last_progress_at=utc_now(),
|
||||
)
|
||||
session.add(task)
|
||||
session.commit()
|
||||
@@ -120,11 +134,38 @@ def create_task(session: Session, request: CreateTaskRequest, *, submit_backgrou
|
||||
|
||||
|
||||
def list_tasks(session: Session) -> list[Task]:
|
||||
return list(session.scalars(select(Task).order_by(Task.created_at.desc())))
|
||||
tasks = list(session.scalars(select(Task).order_by(Task.created_at.desc())))
|
||||
for task in tasks:
|
||||
hydrate_task_progress(session, task)
|
||||
return tasks
|
||||
|
||||
|
||||
def get_task(session: Session, task_id: str) -> Task | None:
|
||||
return session.get(Task, task_id)
|
||||
task = session.get(Task, task_id)
|
||||
if task is not None:
|
||||
hydrate_task_progress(session, task)
|
||||
return task
|
||||
|
||||
|
||||
def hydrate_task_progress(session: Session, task: Task) -> Task:
|
||||
task.comments_count = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||||
task.reports_count = session.scalar(select(func.count(Report.id)).where(Report.task_id == task.id)) or 0
|
||||
task.is_demo = task.id.startswith("demo-")
|
||||
task.last_progress_at = task.last_progress_at or task.created_at
|
||||
stage_code = task.current_stage
|
||||
if not stage_code and task.status == "success":
|
||||
stage_code = "success"
|
||||
elif not stage_code and task.status == "failed":
|
||||
stage_code = "failed"
|
||||
elif not stage_code and task.status == "running":
|
||||
stage_code = "queued"
|
||||
task.current_stage_label = STAGE_LABELS.get(stage_code or "", stage_code or "等待启动")
|
||||
return task
|
||||
|
||||
|
||||
def update_task_stage(task: Task, stage: str) -> None:
|
||||
task.current_stage = stage
|
||||
task.last_progress_at = utc_now()
|
||||
|
||||
|
||||
def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionmaker = SessionLocal) -> None:
|
||||
@@ -139,6 +180,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
ai_requester, report_summary_provider = build_ai_dependencies()
|
||||
|
||||
try:
|
||||
update_task_stage(task, "crawl_hotspots")
|
||||
session.commit()
|
||||
hotspots = platform.fetch_hotspots(limit=task.hotspot_limit)
|
||||
except PlatformAPIError as exc:
|
||||
_mark_task_failed(task, "crawl_hotspots", exc.error_type, str(exc))
|
||||
@@ -163,6 +206,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
session.flush()
|
||||
|
||||
try:
|
||||
update_task_stage(task, "search_items")
|
||||
session.commit()
|
||||
items = platform.search_items_by_hotspot(hotspot.title, limit=task.item_limit_per_hotspot)
|
||||
except Exception as exc:
|
||||
task.failed_items_count += task.item_limit_per_hotspot
|
||||
@@ -193,6 +238,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
session.add(item)
|
||||
session.flush()
|
||||
try:
|
||||
update_task_stage(task, "crawl_comments")
|
||||
session.commit()
|
||||
comments = platform.fetch_comments(item.source_item_id, limit=task.comment_limit_per_item)
|
||||
for comment_data in comments:
|
||||
session.add(
|
||||
@@ -211,6 +258,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
)
|
||||
session.flush()
|
||||
item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
update_task_stage(task, "ai_analysis")
|
||||
session.commit()
|
||||
ai_results = analyze_comments_with_retry(
|
||||
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
|
||||
requester=ai_requester,
|
||||
@@ -231,6 +280,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
comment.ai_analysis_status = result.ai_analysis_status
|
||||
item.status = "success"
|
||||
task.successful_items_count += 1
|
||||
update_task_stage(task, "generate_reports")
|
||||
generate_item_report(session, item.id, summary_provider=report_summary_provider)
|
||||
except Exception as exc:
|
||||
item.status = "failed"
|
||||
@@ -246,6 +296,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
session.commit()
|
||||
|
||||
if task.successful_items_count > 0:
|
||||
update_task_stage(task, "generate_reports")
|
||||
total_comments = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||||
success_comments = sum(1 for comment in task.comments if comment.ai_analysis_status == "success")
|
||||
task.analysis_success_rate, task.analysis_status = calculate_analysis_status(
|
||||
@@ -255,6 +306,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
for hotspot in task.hotspots:
|
||||
generate_hotspot_report(session, hotspot.id, summary_provider=report_summary_provider)
|
||||
task.status = "success"
|
||||
update_task_stage(task, "success")
|
||||
else:
|
||||
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
|
||||
session.commit()
|
||||
@@ -268,6 +320,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
|
||||
def _mark_task_failed(task: Task, stage: str, error_type: str, message: str) -> None:
|
||||
task.status = "failed"
|
||||
update_task_stage(task, "failed")
|
||||
task.error_stage = stage
|
||||
task.error_type = error_type
|
||||
task.error_message = message
|
||||
|
||||
+95
-1
@@ -1,7 +1,101 @@
|
||||
body {
|
||||
background: #f8f9fa;
|
||||
background: #f5f7fb;
|
||||
color: #172033;
|
||||
}
|
||||
|
||||
.card {
|
||||
border-radius: 8px;
|
||||
border: 1px solid #dfe5ef;
|
||||
box-shadow: 0 10px 28px rgba(31, 42, 68, 0.06);
|
||||
}
|
||||
|
||||
.navbar {
|
||||
background: #fff !important;
|
||||
}
|
||||
|
||||
.hero-band {
|
||||
align-items: flex-end;
|
||||
background:
|
||||
linear-gradient(135deg, rgba(12, 22, 40, 0.92), rgba(29, 80, 108, 0.78)),
|
||||
url("https://images.unsplash.com/photo-1551288049-bebda4e38f71?auto=format&fit=crop&w=1600&q=80");
|
||||
background-position: center;
|
||||
background-size: cover;
|
||||
border-radius: 8px;
|
||||
color: #fff;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
min-height: 300px;
|
||||
padding: 42px;
|
||||
}
|
||||
|
||||
.hero-copy {
|
||||
max-width: 680px;
|
||||
}
|
||||
|
||||
.hero-copy h1 {
|
||||
font-size: 48px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.hero-copy p:not(.eyebrow) {
|
||||
color: rgba(255, 255, 255, 0.84);
|
||||
font-size: 18px;
|
||||
line-height: 1.7;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.hero-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
color: #71d4c7;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.metric-box {
|
||||
background: #f8fafc;
|
||||
border: 1px solid #e3e9f2;
|
||||
border-radius: 8px;
|
||||
height: 100%;
|
||||
padding: 14px;
|
||||
}
|
||||
|
||||
.status-panel {
|
||||
background: #fff;
|
||||
border: 1px solid #dfe5ef;
|
||||
border-radius: 8px;
|
||||
padding: 32px;
|
||||
}
|
||||
|
||||
.status-panel-danger {
|
||||
border-color: #f1b8b8;
|
||||
}
|
||||
|
||||
.task-progress {
|
||||
height: 10px;
|
||||
}
|
||||
|
||||
.report-progress {
|
||||
height: 10px;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.hero-band {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
min-height: 360px;
|
||||
padding: 28px;
|
||||
}
|
||||
|
||||
.hero-copy h1 {
|
||||
font-size: 36px;
|
||||
}
|
||||
}
|
||||
|
||||
+38
-16
@@ -114,21 +114,43 @@ function pollTaskListStatus() {
|
||||
|
||||
async function downloadExport(url, defaultFilename, event) {
|
||||
if (event) event.preventDefault();
|
||||
const resp = await fetch(url);
|
||||
if (!resp.ok) {
|
||||
alert("导出失败,请稍后重试。");
|
||||
return;
|
||||
const button = event ? event.currentTarget : null;
|
||||
const originalText = button ? button.textContent : "";
|
||||
if (button) {
|
||||
button.disabled = true;
|
||||
button.textContent = "下载中...";
|
||||
}
|
||||
try {
|
||||
const resp = await fetch(url, { cache: "no-store" });
|
||||
if (!resp.ok) {
|
||||
let message = "导出失败,请稍后重试。";
|
||||
try {
|
||||
const data = await resp.json();
|
||||
if (data.detail) message = data.detail;
|
||||
} catch (_error) {
|
||||
message = resp.status === 503 ? "数据库暂时不可用,请稍后重试。" : message;
|
||||
}
|
||||
alert(message);
|
||||
return;
|
||||
}
|
||||
const blob = await resp.blob();
|
||||
const disposition = resp.headers.get("Content-Disposition") || "";
|
||||
const match = disposition.match(/filename\*=UTF-8''([^;]+)/);
|
||||
const filename = match ? decodeURIComponent(match[1]) : defaultFilename;
|
||||
const blobUrl = URL.createObjectURL(blob);
|
||||
const link = document.createElement("a");
|
||||
link.href = blobUrl;
|
||||
link.download = filename;
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
} catch (_error) {
|
||||
alert("网络异常,导出未完成。");
|
||||
} finally {
|
||||
if (button) {
|
||||
button.disabled = false;
|
||||
button.textContent = originalText;
|
||||
}
|
||||
}
|
||||
const blob = await resp.blob();
|
||||
const disposition = resp.headers.get("Content-Disposition") || "";
|
||||
const match = disposition.match(/filename\*=UTF-8''([^;]+)/);
|
||||
const filename = match ? decodeURIComponent(match[1]) : defaultFilename;
|
||||
const blobUrl = URL.createObjectURL(blob);
|
||||
const link = document.createElement("a");
|
||||
link.href = blobUrl;
|
||||
link.download = filename;
|
||||
document.body.appendChild(link);
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}数据库暂时不可用 - 热榜评论分析工具{% endblock %}
|
||||
{% block content %}
|
||||
<section class="status-panel status-panel-danger">
|
||||
<div>
|
||||
<p class="eyebrow mb-2">数据恢复保护</p>
|
||||
<h1 class="h3 mb-3">数据库暂时不可用</h1>
|
||||
<p class="mb-3">{{ message }}</p>
|
||||
<p class="mb-0 text-muted">请先保留 data 目录,不要删除 app.db、app.db-wal 或 app.db-shm。系统会优先备份现有数据库文件,再进行诊断和恢复。</p>
|
||||
</div>
|
||||
</section>
|
||||
{% if detail %}
|
||||
<details class="mt-3">
|
||||
<summary class="text-muted">查看技术细节</summary>
|
||||
<pre class="mt-2 small">{{ detail }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -1,13 +1,20 @@
|
||||
{% extends "base.html" %}
|
||||
{% set has_running_tasks = tasks | selectattr("status", "equalto", "running") | list | length > 0 %}
|
||||
{% block title %}任务列表 - 热榜评论分析工具{% endblock %}
|
||||
{% block title %}热榜评论雷达 - 热榜评论分析工具{% endblock %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h1 class="h3 mb-0">任务列表</h1>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button>
|
||||
</div>
|
||||
<section class="hero-band mb-4">
|
||||
<div class="hero-copy">
|
||||
<p class="eyebrow">Hot Comment Radar</p>
|
||||
<h1>热榜评论雷达</h1>
|
||||
<p>从小红书和抖音热点出发,抓取真实评论,生成 AI 情绪、标签和可导出的分析报告。</p>
|
||||
</div>
|
||||
<div class="hero-actions">
|
||||
<a class="btn btn-light" href="#create-task">创建新任务</a>
|
||||
<a class="btn btn-outline-light" href="#task-history">查看 Demo 数据</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card mb-4">
|
||||
<section class="card mb-4" id="create-task">
|
||||
<div class="card-header">创建抓取任务</div>
|
||||
<div class="card-body">
|
||||
<div id="form-error" class="alert alert-danger d-none" role="alert"></div>
|
||||
@@ -43,8 +50,11 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card">
|
||||
<div class="card-header">历史任务</div>
|
||||
<section class="card" id="task-history">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<span>最近任务与 Demo 数据</span>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button>
|
||||
</div>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0">
|
||||
<thead>
|
||||
|
||||
@@ -3,7 +3,12 @@
|
||||
{% set percent = progress_percent(task.processed_items_count, task.total_items_count) %}
|
||||
{% set ai_percent = rate_percent(task.analysis_success_rate) %}
|
||||
<tr>
|
||||
<td><code>{{ task.id }}</code></td>
|
||||
<td>
|
||||
<code>{{ task.id }}</code>
|
||||
{% if task.is_demo %}
|
||||
<br><span class="badge text-bg-info">Demo 数据</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ task.platform | platform_label }}</td>
|
||||
<td>{{ task.created_at }}</td>
|
||||
<td><small class="text-muted">热点 {{ task.hotspot_limit }} / 内容 {{ task.item_limit_per_hotspot }} / 评论 {{ task.comment_limit_per_item }}</small></td>
|
||||
@@ -16,6 +21,8 @@
|
||||
<div class="progress-bar" style="width: {{ percent }}%"></div>
|
||||
</div>
|
||||
<div class="small text-muted mt-1">成功 {{ task.successful_items_count }} / 失败 {{ task.failed_items_count }}</div>
|
||||
<div class="small text-muted mt-1">阶段:{{ task.current_stage_label or "等待启动" }}</div>
|
||||
<div class="small text-muted mt-1">评论 {{ task.comments_count or 0 }} / 报告 {{ task.reports_count or 0 }}</div>
|
||||
<div class="small {% if task.analysis_status == 'insufficient' %}text-warning{% else %}text-muted{% endif %}">
|
||||
AI 成功率 {{ ai_percent }}%
|
||||
{% if task.analysis_status == "insufficient" %}
|
||||
|
||||
@@ -15,7 +15,11 @@
|
||||
{% set label, cls = status_badge_config(task.status) %}
|
||||
{% set percent = progress_percent(task.processed_items_count, task.total_items_count) %}
|
||||
{% set ai_percent = rate_percent(task.analysis_success_rate) %}
|
||||
<p>平台:{{ task.platform | platform_label }} <span class="badge {{ cls }}">{{ label }}</span></p>
|
||||
{% set target_comments = task.hotspot_limit * task.item_limit_per_hotspot * task.comment_limit_per_item %}
|
||||
<div class="d-flex flex-wrap justify-content-between gap-3 mb-3">
|
||||
<p class="mb-0">平台:{{ task.platform | platform_label }} <span class="badge {{ cls }}">{{ label }}</span>{% if task.is_demo %}<span class="badge text-bg-info ms-1">Demo 数据</span>{% endif %}</p>
|
||||
<p class="mb-0 text-muted">阶段:{{ task.current_stage_label or "等待启动" }}</p>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<div class="d-flex justify-content-between align-items-center gap-2">
|
||||
<span>已处理 {{ task.processed_items_count }} / 共 {{ task.total_items_count }} 条内容</span>
|
||||
@@ -26,6 +30,32 @@
|
||||
</div>
|
||||
<div class="small text-muted mt-2">成功 {{ task.successful_items_count }} / 失败 {{ task.failed_items_count }}</div>
|
||||
</div>
|
||||
<div class="row g-3 mb-3">
|
||||
<div class="col-md-4">
|
||||
<div class="metric-box">
|
||||
<div class="text-muted small">目标上限</div>
|
||||
<strong>{{ task.hotspot_limit }} 热点 × {{ task.item_limit_per_hotspot }} 内容 × {{ task.comment_limit_per_item }} 评论</strong>
|
||||
<div class="text-muted small">最多 {{ target_comments }} 条评论</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="metric-box">
|
||||
<div class="text-muted small">实际结果</div>
|
||||
<strong>实际评论 {{ task.comments_count or 0 }}</strong>
|
||||
<div class="text-muted small">报告 {{ task.reports_count or 0 }} / 内容 {{ task.total_items_count }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="metric-box">
|
||||
<div class="text-muted small">最近进度</div>
|
||||
<strong>{{ task.last_progress_at or task.created_at }}</strong>
|
||||
<div class="text-muted small">外部接口和 AI 响应较慢时,阶段可能短时间不变</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if task.status == "success" and (task.comments_count or 0) < target_comments %}
|
||||
<div class="alert alert-info">少于理论上限通常是内容本身评论不足或平台返回不足,不直接代表任务失败。若失败内容数大于 0,请结合失败原因判断。</div>
|
||||
{% endif %}
|
||||
<div class="mb-2">
|
||||
<span class="{% if task.analysis_status == 'insufficient' %}text-warning{% else %}text-muted{% endif %}">AI 成功率 {{ ai_percent }}%</span>
|
||||
{% if task.analysis_status == "insufficient" %}
|
||||
|
||||
Reference in New Issue
Block a user