feat: 完善 MVP-2 演示和进度体验
This commit is contained in:
meijiali
@@ -1,5 +1,7 @@
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from shutil import copy2
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Engine, create_engine, event
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
@@ -30,6 +32,16 @@ def create_sqlite_engine(database_url: str):
|
||||
return engine
|
||||
|
||||
|
||||
def sqlite_file_path(target_engine: Engine = None) -> Path | None:
|
||||
target_engine = target_engine or engine
|
||||
if target_engine.url.get_backend_name() != "sqlite":
|
||||
return None
|
||||
database = target_engine.url.database
|
||||
if not database or database == ":memory:":
|
||||
return None
|
||||
return Path(database)
|
||||
|
||||
|
||||
engine = create_sqlite_engine(get_settings().database_url)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
|
||||
|
||||
@@ -63,10 +75,39 @@ def checkpoint_sqlite_wal(target_engine: Engine = engine) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def ensure_sqlite_schema_compat(target_engine: Engine = engine) -> None:
|
||||
if target_engine.url.get_backend_name() != "sqlite":
|
||||
return
|
||||
with target_engine.begin() as connection:
|
||||
task_columns = {row[1] for row in connection.exec_driver_sql("PRAGMA table_info(tasks)").all()}
|
||||
if "current_stage" not in task_columns:
|
||||
connection.exec_driver_sql("ALTER TABLE tasks ADD COLUMN current_stage VARCHAR(64)")
|
||||
if "last_progress_at" not in task_columns:
|
||||
connection.exec_driver_sql("ALTER TABLE tasks ADD COLUMN last_progress_at DATETIME")
|
||||
|
||||
|
||||
def backup_sqlite_files(target_engine: Engine = engine) -> list[Path]:
|
||||
db_path = sqlite_file_path(target_engine)
|
||||
if db_path is None:
|
||||
return []
|
||||
backup_dir = db_path.parent / "corrupt-backups"
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
||||
copied: list[Path] = []
|
||||
for source in [db_path, db_path.with_name(f"{db_path.name}-wal"), db_path.with_name(f"{db_path.name}-shm")]:
|
||||
if not source.exists():
|
||||
continue
|
||||
target = backup_dir / f"{source.name}.{timestamp}.bak"
|
||||
copy2(source, target)
|
||||
copied.append(target)
|
||||
return copied
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
import app.models # noqa: F401
|
||||
|
||||
Base.metadata.create_all(engine)
|
||||
ensure_sqlite_schema_compat(engine)
|
||||
check_database_integrity(engine)
|
||||
checkpoint_sqlite_wal(engine)
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal, init_db
|
||||
from app.models import Comment, ContentItem, Hotspot, Report, Task, utc_now
|
||||
|
||||
|
||||
FIXTURE_PATH = Path(__file__).resolve().parent / "fixtures" / "demo_seed.json"
|
||||
|
||||
|
||||
def seed_demo_data(session: Session, fixture_path: Path = FIXTURE_PATH) -> str:
|
||||
data = json.loads(fixture_path.read_text(encoding="utf-8"))
|
||||
task_data = data["task"]
|
||||
task_id = task_data["id"]
|
||||
existing = session.scalar(select(Task.id).where(Task.id == task_id))
|
||||
if existing:
|
||||
return task_id
|
||||
|
||||
task = Task(
|
||||
id=task_id,
|
||||
platform=task_data["platform"],
|
||||
status="success",
|
||||
current_stage="success",
|
||||
last_progress_at=utc_now(),
|
||||
hotspot_limit=task_data["hotspot_limit"],
|
||||
item_limit_per_hotspot=task_data["item_limit_per_hotspot"],
|
||||
comment_limit_per_item=task_data["comment_limit_per_item"],
|
||||
total_items_count=len(data["items"]),
|
||||
processed_items_count=len(data["items"]),
|
||||
successful_items_count=len(data["items"]),
|
||||
failed_items_count=0,
|
||||
analysis_success_rate=1.0,
|
||||
analysis_status="normal",
|
||||
)
|
||||
session.add(task)
|
||||
|
||||
for hotspot_data in data["hotspots"]:
|
||||
session.add(
|
||||
Hotspot(
|
||||
id=hotspot_data["id"],
|
||||
task_id=task_id,
|
||||
platform=task.platform,
|
||||
rank=hotspot_data["rank"],
|
||||
title=hotspot_data["title"],
|
||||
heat_value=hotspot_data.get("heat_value"),
|
||||
source_hot_id=None,
|
||||
raw_data="{}",
|
||||
)
|
||||
)
|
||||
|
||||
for item_data in data["items"]:
|
||||
session.add(
|
||||
ContentItem(
|
||||
id=item_data["id"],
|
||||
task_id=task_id,
|
||||
hotspot_id=item_data["hotspot_id"],
|
||||
platform=task.platform,
|
||||
source_item_id=f"demo-item-{item_data['id']}",
|
||||
item_type=item_data["item_type"],
|
||||
title=item_data["title"],
|
||||
summary=item_data.get("summary"),
|
||||
url=None,
|
||||
status="success",
|
||||
raw_data="{}",
|
||||
)
|
||||
)
|
||||
|
||||
for comment_data in data["comments"]:
|
||||
session.add(
|
||||
Comment(
|
||||
id=comment_data["id"],
|
||||
task_id=task_id,
|
||||
hotspot_id=comment_data["hotspot_id"],
|
||||
content_item_id=comment_data["content_item_id"],
|
||||
platform=task.platform,
|
||||
source_comment_id=None,
|
||||
content=comment_data["content"],
|
||||
author=None,
|
||||
like_count=comment_data.get("like_count", 0),
|
||||
sentiment=comment_data["sentiment"],
|
||||
labels=json.dumps(comment_data["labels"], ensure_ascii=False),
|
||||
reason=comment_data.get("reason"),
|
||||
ai_analysis_status="success",
|
||||
raw_data="{}",
|
||||
ai_raw_response=None,
|
||||
)
|
||||
)
|
||||
|
||||
for report_data in data["reports"]:
|
||||
session.add(
|
||||
Report(
|
||||
task_id=task_id,
|
||||
hotspot_id=report_data.get("hotspot_id"),
|
||||
content_item_id=report_data.get("content_item_id"),
|
||||
report_type=report_data["report_type"],
|
||||
title=report_data["title"],
|
||||
metrics_json=json.dumps(report_data["metrics"], ensure_ascii=False),
|
||||
typical_comments_json=json.dumps(report_data["typical_comments"], ensure_ascii=False),
|
||||
summary=report_data["summary"],
|
||||
markdown_content=report_data["markdown_content"],
|
||||
data="{}",
|
||||
markdown=report_data["markdown_content"],
|
||||
)
|
||||
)
|
||||
|
||||
session.commit()
|
||||
return task_id
|
||||
|
||||
|
||||
def main() -> None:
|
||||
init_db()
|
||||
with SessionLocal() as session:
|
||||
task_id = seed_demo_data(session)
|
||||
print(f"Seeded demo task: {task_id}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,157 @@
|
||||
{
|
||||
"task": {
|
||||
"id": "demo-douyin-20260703",
|
||||
"platform": "douyin",
|
||||
"hotspot_limit": 1,
|
||||
"item_limit_per_hotspot": 2,
|
||||
"comment_limit_per_item": 10
|
||||
},
|
||||
"hotspots": [
|
||||
{
|
||||
"id": "demo-hotspot-1",
|
||||
"rank": 1,
|
||||
"title": "演示热点：夏季新品讨论",
|
||||
"heat_value": "demo"
|
||||
}
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"id": "demo-item-1",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"item_type": "video",
|
||||
"title": "新品开箱体验",
|
||||
"summary": "演示用脱敏内容条目"
|
||||
},
|
||||
{
|
||||
"id": "demo-item-2",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"item_type": "video",
|
||||
"title": "用户上手反馈",
|
||||
"summary": "演示用脱敏内容条目"
|
||||
}
|
||||
],
|
||||
"comments": [
|
||||
{
|
||||
"id": "demo-comment-1",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-1",
|
||||
"content": "这个颜色很清爽，夏天用看起来挺舒服。",
|
||||
"sentiment": "positive",
|
||||
"labels": ["外观种草", "季节场景"],
|
||||
"reason": "用户表达了对外观和使用场景的认可。",
|
||||
"like_count": 36
|
||||
},
|
||||
{
|
||||
"id": "demo-comment-2",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-1",
|
||||
"content": "价格如果能再低一点就好了，现在有点观望。",
|
||||
"sentiment": "neutral",
|
||||
"labels": ["价格观望", "购买决策"],
|
||||
"reason": "用户没有否定产品，但对价格仍有顾虑。",
|
||||
"like_count": 21
|
||||
},
|
||||
{
|
||||
"id": "demo-comment-3",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-2",
|
||||
"content": "看完真实上手比广告图可信，想看看长期使用反馈。",
|
||||
"sentiment": "positive",
|
||||
"labels": ["真实体验", "长期反馈"],
|
||||
"reason": "用户认可真实体验内容，但仍希望补充长期反馈。",
|
||||
"like_count": 18
|
||||
},
|
||||
{
|
||||
"id": "demo-comment-4",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-2",
|
||||
"content": "评论里好多人问链接，说明种草效果还是挺明显的。",
|
||||
"sentiment": "positive",
|
||||
"labels": ["求购买链接", "种草效果"],
|
||||
"reason": "用户从评论行为判断内容有转化潜力。",
|
||||
"like_count": 12
|
||||
}
|
||||
],
|
||||
"reports": [
|
||||
{
|
||||
"report_type": "hotspot",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"title": "演示热点：夏季新品讨论",
|
||||
"summary": "该热点评论以正向反馈为主，用户主要关注外观、真实体验、价格和购买链接。价格仍是部分用户从种草到购买之间的关键阻力。",
|
||||
"metrics": {
|
||||
"sample_count": 4,
|
||||
"item_count": 2,
|
||||
"sentiment": {
|
||||
"positive": {"count": 3, "pct": 75},
|
||||
"neutral": {"count": 1, "pct": 25},
|
||||
"negative": {"count": 0, "pct": 0},
|
||||
"unknown": {"count": 0, "pct": 0}
|
||||
},
|
||||
"top_labels": [
|
||||
{"name": "外观种草", "count": 1},
|
||||
{"name": "价格观望", "count": 1},
|
||||
{"name": "真实体验", "count": 1},
|
||||
{"name": "求购买链接", "count": 1}
|
||||
]
|
||||
},
|
||||
"typical_comments": {
|
||||
"positive": [{"content": "这个颜色很清爽，夏天用看起来挺舒服。", "like_count": 36}],
|
||||
"neutral": [{"content": "价格如果能再低一点就好了，现在有点观望。", "like_count": 21}],
|
||||
"negative": []
|
||||
},
|
||||
"markdown_content": "# 演示热点：夏季新品讨论\n\n## 总结\n\n该热点评论以正向反馈为主，用户主要关注外观、真实体验、价格和购买链接。价格仍是部分用户从种草到购买之间的关键阻力。"
|
||||
},
|
||||
{
|
||||
"report_type": "item",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-1",
|
||||
"title": "新品开箱体验",
|
||||
"summary": "该内容主要激发了外观和季节场景兴趣，同时价格仍影响部分用户的购买决策。",
|
||||
"metrics": {
|
||||
"sample_count": 2,
|
||||
"sentiment": {
|
||||
"positive": {"count": 1, "pct": 50},
|
||||
"neutral": {"count": 1, "pct": 50},
|
||||
"negative": {"count": 0, "pct": 0},
|
||||
"unknown": {"count": 0, "pct": 0}
|
||||
},
|
||||
"top_labels": [
|
||||
{"name": "外观种草", "count": 1},
|
||||
{"name": "价格观望", "count": 1}
|
||||
]
|
||||
},
|
||||
"typical_comments": {
|
||||
"positive": [{"content": "这个颜色很清爽，夏天用看起来挺舒服。", "like_count": 36}],
|
||||
"neutral": [{"content": "价格如果能再低一点就好了，现在有点观望。", "like_count": 21}],
|
||||
"negative": []
|
||||
},
|
||||
"markdown_content": "# 新品开箱体验\n\n## 总结\n\n该内容主要激发了外观和季节场景兴趣，同时价格仍影响部分用户的购买决策。"
|
||||
},
|
||||
{
|
||||
"report_type": "item",
|
||||
"hotspot_id": "demo-hotspot-1",
|
||||
"content_item_id": "demo-item-2",
|
||||
"title": "用户上手反馈",
|
||||
"summary": "该内容的评论更关注真实体验和后续转化，用户希望看到长期反馈，也表现出明显的购买链接需求。",
|
||||
"metrics": {
|
||||
"sample_count": 2,
|
||||
"sentiment": {
|
||||
"positive": {"count": 2, "pct": 100},
|
||||
"neutral": {"count": 0, "pct": 0},
|
||||
"negative": {"count": 0, "pct": 0},
|
||||
"unknown": {"count": 0, "pct": 0}
|
||||
},
|
||||
"top_labels": [
|
||||
{"name": "真实体验", "count": 1},
|
||||
{"name": "求购买链接", "count": 1}
|
||||
]
|
||||
},
|
||||
"typical_comments": {
|
||||
"positive": [{"content": "看完真实上手比广告图可信，想看看长期使用反馈。", "like_count": 18}],
|
||||
"neutral": [],
|
||||
"negative": []
|
||||
},
|
||||
"markdown_content": "# 用户上手反馈\n\n## 总结\n\n该内容的评论更关注真实体验和后续转化，用户希望看到长期反馈，也表现出明显的购买链接需求。"
|
||||
}
|
||||
]
|
||||
}
|
||||
+33
-2
@@ -1,15 +1,17 @@
|
||||
from collections.abc import AsyncGenerator
|
||||
from contextlib import asynccontextmanager
|
||||
import logging
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import Depends, FastAPI, HTTPException, Request, status
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from fastapi.responses import Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import OperationalError, SQLAlchemyError
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import SessionLocal, get_db_session, init_db
|
||||
from app.db import SessionLocal, backup_sqlite_files, get_db_session, init_db
|
||||
from app.models import Comment, ContentItem, Hotspot, Report
|
||||
from app.schemas import CreateTaskRequest, CreateTaskResponse, TaskResponse
|
||||
from app.services.export_service import export_hotspot_comments_csv, export_item_comments_csv, export_report_markdown
|
||||
@@ -24,6 +26,25 @@ from app.services.task_service import (
|
||||
from app.templating import templates
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
DATABASE_UNAVAILABLE_MESSAGE = "数据库暂时不可用，请稍后重试或联系维护者恢复数据。"
|
||||
|
||||
|
||||
def handle_database_error(request: Request, exc: SQLAlchemyError):
|
||||
try:
|
||||
backup_sqlite_files()
|
||||
except Exception:
|
||||
logger.exception("Failed to back up SQLite files after database error")
|
||||
if request.url.path.startswith("/api/"):
|
||||
return JSONResponse(status_code=503, content={"detail": DATABASE_UNAVAILABLE_MESSAGE})
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"errors/database_unavailable.html",
|
||||
{"message": DATABASE_UNAVAILABLE_MESSAGE, "detail": str(exc)},
|
||||
status_code=503,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncGenerator[None]:
|
||||
init_db()
|
||||
@@ -36,6 +57,16 @@ app = FastAPI(title="热榜评论分析工具", lifespan=lifespan)
|
||||
app.mount("/static", StaticFiles(directory="app/static"), name="static")
|
||||
|
||||
|
||||
@app.exception_handler(OperationalError)
|
||||
def operational_error_handler(request: Request, exc: OperationalError):
|
||||
return handle_database_error(request, exc)
|
||||
|
||||
|
||||
@app.exception_handler(SQLAlchemyError)
|
||||
def sqlalchemy_error_handler(request: Request, exc: SQLAlchemyError):
|
||||
return handle_database_error(request, exc)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
@@ -37,6 +37,8 @@ class Task(Base):
|
||||
processed_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
successful_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed_items_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
current_stage: Mapped[str | None] = mapped_column(String(64))
|
||||
last_progress_at: Mapped[datetime | None] = mapped_column()
|
||||
error_stage: Mapped[str | None] = mapped_column(String(64))
|
||||
error_type: Mapped[str | None] = mapped_column(String(64))
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
@@ -31,6 +31,12 @@ class TaskResponse(BaseModel):
|
||||
processed_items_count: int
|
||||
successful_items_count: int
|
||||
failed_items_count: int
|
||||
current_stage: str | None = None
|
||||
current_stage_label: str | None = None
|
||||
last_progress_at: datetime | None = None
|
||||
comments_count: int = 0
|
||||
reports_count: int = 0
|
||||
is_demo: bool = False
|
||||
error_message: str | None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.config import get_settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Comment, ContentItem, Hotspot, Task
|
||||
from app.models import Comment, ContentItem, Hotspot, Report, Task, utc_now
|
||||
from app.platforms.base import PlatformAPIError, TikHubClient
|
||||
from app.platforms.douyin import DouyinPlatform
|
||||
from app.platforms.xiaohongshu import XiaohongshuPlatform
|
||||
@@ -22,6 +22,18 @@ RESTART_ERROR_MESSAGE = "系统重启，任务被中断"
|
||||
task_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
|
||||
STAGE_LABELS = {
|
||||
"queued": "等待启动",
|
||||
"crawl_hotspots": "获取热点中",
|
||||
"search_items": "搜索内容中",
|
||||
"crawl_comments": "抓取评论中",
|
||||
"ai_analysis": "AI 分析中",
|
||||
"generate_reports": "生成报告中",
|
||||
"success": "已完成",
|
||||
"failed": "失败",
|
||||
}
|
||||
|
||||
|
||||
def has_running_task(session: Session) -> bool:
|
||||
return session.scalar(select(Task.id).where(Task.status == "running").limit(1)) is not None
|
||||
|
||||
@@ -110,6 +122,8 @@ def create_task(session: Session, request: CreateTaskRequest, *, submit_backgrou
|
||||
failed_items_count=0,
|
||||
analysis_success_rate=0.0,
|
||||
analysis_status="normal",
|
||||
current_stage="queued",
|
||||
last_progress_at=utc_now(),
|
||||
)
|
||||
session.add(task)
|
||||
session.commit()
|
||||
@@ -120,11 +134,38 @@ def create_task(session: Session, request: CreateTaskRequest, *, submit_backgrou
|
||||
|
||||
|
||||
def list_tasks(session: Session) -> list[Task]:
|
||||
return list(session.scalars(select(Task).order_by(Task.created_at.desc())))
|
||||
tasks = list(session.scalars(select(Task).order_by(Task.created_at.desc())))
|
||||
for task in tasks:
|
||||
hydrate_task_progress(session, task)
|
||||
return tasks
|
||||
|
||||
|
||||
def get_task(session: Session, task_id: str) -> Task | None:
|
||||
return session.get(Task, task_id)
|
||||
task = session.get(Task, task_id)
|
||||
if task is not None:
|
||||
hydrate_task_progress(session, task)
|
||||
return task
|
||||
|
||||
|
||||
def hydrate_task_progress(session: Session, task: Task) -> Task:
|
||||
task.comments_count = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||||
task.reports_count = session.scalar(select(func.count(Report.id)).where(Report.task_id == task.id)) or 0
|
||||
task.is_demo = task.id.startswith("demo-")
|
||||
task.last_progress_at = task.last_progress_at or task.created_at
|
||||
stage_code = task.current_stage
|
||||
if not stage_code and task.status == "success":
|
||||
stage_code = "success"
|
||||
elif not stage_code and task.status == "failed":
|
||||
stage_code = "failed"
|
||||
elif not stage_code and task.status == "running":
|
||||
stage_code = "queued"
|
||||
task.current_stage_label = STAGE_LABELS.get(stage_code or "", stage_code or "等待启动")
|
||||
return task
|
||||
|
||||
|
||||
def update_task_stage(task: Task, stage: str) -> None:
|
||||
task.current_stage = stage
|
||||
task.last_progress_at = utc_now()
|
||||
|
||||
|
||||
def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionmaker = SessionLocal) -> None:
|
||||
@@ -139,6 +180,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
ai_requester, report_summary_provider = build_ai_dependencies()
|
||||
|
||||
try:
|
||||
update_task_stage(task, "crawl_hotspots")
|
||||
session.commit()
|
||||
hotspots = platform.fetch_hotspots(limit=task.hotspot_limit)
|
||||
except PlatformAPIError as exc:
|
||||
_mark_task_failed(task, "crawl_hotspots", exc.error_type, str(exc))
|
||||
@@ -163,6 +206,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
session.flush()
|
||||
|
||||
try:
|
||||
update_task_stage(task, "search_items")
|
||||
session.commit()
|
||||
items = platform.search_items_by_hotspot(hotspot.title, limit=task.item_limit_per_hotspot)
|
||||
except Exception as exc:
|
||||
task.failed_items_count += task.item_limit_per_hotspot
|
||||
@@ -193,6 +238,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
session.add(item)
|
||||
session.flush()
|
||||
try:
|
||||
update_task_stage(task, "crawl_comments")
|
||||
session.commit()
|
||||
comments = platform.fetch_comments(item.source_item_id, limit=task.comment_limit_per_item)
|
||||
for comment_data in comments:
|
||||
session.add(
|
||||
@@ -211,6 +258,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
)
|
||||
session.flush()
|
||||
item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
|
||||
update_task_stage(task, "ai_analysis")
|
||||
session.commit()
|
||||
ai_results = analyze_comments_with_retry(
|
||||
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
|
||||
requester=ai_requester,
|
||||
@@ -231,6 +280,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
comment.ai_analysis_status = result.ai_analysis_status
|
||||
item.status = "success"
|
||||
task.successful_items_count += 1
|
||||
update_task_stage(task, "generate_reports")
|
||||
generate_item_report(session, item.id, summary_provider=report_summary_provider)
|
||||
except Exception as exc:
|
||||
item.status = "failed"
|
||||
@@ -246,6 +296,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
session.commit()
|
||||
|
||||
if task.successful_items_count > 0:
|
||||
update_task_stage(task, "generate_reports")
|
||||
total_comments = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||||
success_comments = sum(1 for comment in task.comments if comment.ai_analysis_status == "success")
|
||||
task.analysis_success_rate, task.analysis_status = calculate_analysis_status(
|
||||
@@ -255,6 +306,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
for hotspot in task.hotspots:
|
||||
generate_hotspot_report(session, hotspot.id, summary_provider=report_summary_provider)
|
||||
task.status = "success"
|
||||
update_task_stage(task, "success")
|
||||
else:
|
||||
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
|
||||
session.commit()
|
||||
@@ -268,6 +320,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
|
||||
|
||||
def _mark_task_failed(task: Task, stage: str, error_type: str, message: str) -> None:
|
||||
task.status = "failed"
|
||||
update_task_stage(task, "failed")
|
||||
task.error_stage = stage
|
||||
task.error_type = error_type
|
||||
task.error_message = message
|
||||
|
||||
+95
-1
@@ -1,7 +1,101 @@
|
||||
body {
|
||||
background: #f8f9fa;
|
||||
background: #f5f7fb;
|
||||
color: #172033;
|
||||
}
|
||||
|
||||
.card {
|
||||
border-radius: 8px;
|
||||
border: 1px solid #dfe5ef;
|
||||
box-shadow: 0 10px 28px rgba(31, 42, 68, 0.06);
|
||||
}
|
||||
|
||||
.navbar {
|
||||
background: #fff !important;
|
||||
}
|
||||
|
||||
.hero-band {
|
||||
align-items: flex-end;
|
||||
background:
|
||||
linear-gradient(135deg, rgba(12, 22, 40, 0.92), rgba(29, 80, 108, 0.78)),
|
||||
url("https://images.unsplash.com/photo-1551288049-bebda4e38f71?auto=format&fit=crop&w=1600&q=80");
|
||||
background-position: center;
|
||||
background-size: cover;
|
||||
border-radius: 8px;
|
||||
color: #fff;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
min-height: 300px;
|
||||
padding: 42px;
|
||||
}
|
||||
|
||||
.hero-copy {
|
||||
max-width: 680px;
|
||||
}
|
||||
|
||||
.hero-copy h1 {
|
||||
font-size: 48px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.hero-copy p:not(.eyebrow) {
|
||||
color: rgba(255, 255, 255, 0.84);
|
||||
font-size: 18px;
|
||||
line-height: 1.7;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.hero-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
color: #71d4c7;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.metric-box {
|
||||
background: #f8fafc;
|
||||
border: 1px solid #e3e9f2;
|
||||
border-radius: 8px;
|
||||
height: 100%;
|
||||
padding: 14px;
|
||||
}
|
||||
|
||||
.status-panel {
|
||||
background: #fff;
|
||||
border: 1px solid #dfe5ef;
|
||||
border-radius: 8px;
|
||||
padding: 32px;
|
||||
}
|
||||
|
||||
.status-panel-danger {
|
||||
border-color: #f1b8b8;
|
||||
}
|
||||
|
||||
.task-progress {
|
||||
height: 10px;
|
||||
}
|
||||
|
||||
.report-progress {
|
||||
height: 10px;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.hero-band {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
min-height: 360px;
|
||||
padding: 28px;
|
||||
}
|
||||
|
||||
.hero-copy h1 {
|
||||
font-size: 36px;
|
||||
}
|
||||
}
|
||||
|
||||
+24
-2
@@ -114,9 +114,23 @@ function pollTaskListStatus() {
|
||||
|
||||
async function downloadExport(url, defaultFilename, event) {
|
||||
if (event) event.preventDefault();
|
||||
const resp = await fetch(url);
|
||||
const button = event ? event.currentTarget : null;
|
||||
const originalText = button ? button.textContent : "";
|
||||
if (button) {
|
||||
button.disabled = true;
|
||||
button.textContent = "下载中...";
|
||||
}
|
||||
try {
|
||||
const resp = await fetch(url, { cache: "no-store" });
|
||||
if (!resp.ok) {
|
||||
alert("导出失败，请稍后重试。");
|
||||
let message = "导出失败，请稍后重试。";
|
||||
try {
|
||||
const data = await resp.json();
|
||||
if (data.detail) message = data.detail;
|
||||
} catch (_error) {
|
||||
message = resp.status === 503 ? "数据库暂时不可用，请稍后重试。" : message;
|
||||
}
|
||||
alert(message);
|
||||
return;
|
||||
}
|
||||
const blob = await resp.blob();
|
||||
@@ -131,4 +145,12 @@ async function downloadExport(url, defaultFilename, event) {
|
||||
link.click();
|
||||
document.body.removeChild(link);
|
||||
URL.revokeObjectURL(blobUrl);
|
||||
} catch (_error) {
|
||||
alert("网络异常，导出未完成。");
|
||||
} finally {
|
||||
if (button) {
|
||||
button.disabled = false;
|
||||
button.textContent = originalText;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}数据库暂时不可用 - 热榜评论分析工具{% endblock %}
|
||||
{% block content %}
|
||||
<section class="status-panel status-panel-danger">
|
||||
<div>
|
||||
<p class="eyebrow mb-2">数据恢复保护</p>
|
||||
<h1 class="h3 mb-3">数据库暂时不可用</h1>
|
||||
<p class="mb-3">{{ message }}</p>
|
||||
<p class="mb-0 text-muted">请先保留 data 目录，不要删除 app.db、app.db-wal 或 app.db-shm。系统会优先备份现有数据库文件，再进行诊断和恢复。</p>
|
||||
</div>
|
||||
</section>
|
||||
{% if detail %}
|
||||
<details class="mt-3">
|
||||
<summary class="text-muted">查看技术细节</summary>
|
||||
<pre class="mt-2 small">{{ detail }}</pre>
|
||||
</details>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -1,13 +1,20 @@
|
||||
{% extends "base.html" %}
|
||||
{% set has_running_tasks = tasks | selectattr("status", "equalto", "running") | list | length > 0 %}
|
||||
{% block title %}任务列表 - 热榜评论分析工具{% endblock %}
|
||||
{% block title %}热榜评论雷达 - 热榜评论分析工具{% endblock %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h1 class="h3 mb-0">任务列表</h1>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button>
|
||||
</div>
|
||||
<section class="hero-band mb-4">
|
||||
<div class="hero-copy">
|
||||
<p class="eyebrow">Hot Comment Radar</p>
|
||||
<h1>热榜评论雷达</h1>
|
||||
<p>从小红书和抖音热点出发，抓取真实评论，生成 AI 情绪、标签和可导出的分析报告。</p>
|
||||
</div>
|
||||
<div class="hero-actions">
|
||||
<a class="btn btn-light" href="#create-task">创建新任务</a>
|
||||
<a class="btn btn-outline-light" href="#task-history">查看 Demo 数据</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card mb-4">
|
||||
<section class="card mb-4" id="create-task">
|
||||
<div class="card-header">创建抓取任务</div>
|
||||
<div class="card-body">
|
||||
<div id="form-error" class="alert alert-danger d-none" role="alert"></div>
|
||||
@@ -43,8 +50,11 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card">
|
||||
<div class="card-header">历史任务</div>
|
||||
<section class="card" id="task-history">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<span>最近任务与 Demo 数据</span>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button>
|
||||
</div>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0">
|
||||
<thead>
|
||||
|
||||
@@ -3,7 +3,12 @@
|
||||
{% set percent = progress_percent(task.processed_items_count, task.total_items_count) %}
|
||||
{% set ai_percent = rate_percent(task.analysis_success_rate) %}
|
||||
<tr>
|
||||
<td><code>{{ task.id }}</code></td>
|
||||
<td>
|
||||
<code>{{ task.id }}</code>
|
||||
{% if task.is_demo %}
|
||||
<br><span class="badge text-bg-info">Demo 数据</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ task.platform | platform_label }}</td>
|
||||
<td>{{ task.created_at }}</td>
|
||||
<td><small class="text-muted">热点 {{ task.hotspot_limit }} / 内容 {{ task.item_limit_per_hotspot }} / 评论 {{ task.comment_limit_per_item }}</small></td>
|
||||
@@ -16,6 +21,8 @@
|
||||
<div class="progress-bar" style="width: {{ percent }}%"></div>
|
||||
</div>
|
||||
<div class="small text-muted mt-1">成功 {{ task.successful_items_count }} / 失败 {{ task.failed_items_count }}</div>
|
||||
<div class="small text-muted mt-1">阶段：{{ task.current_stage_label or "等待启动" }}</div>
|
||||
<div class="small text-muted mt-1">评论 {{ task.comments_count or 0 }} / 报告 {{ task.reports_count or 0 }}</div>
|
||||
<div class="small {% if task.analysis_status == 'insufficient' %}text-warning{% else %}text-muted{% endif %}">
|
||||
AI 成功率 {{ ai_percent }}%
|
||||
{% if task.analysis_status == "insufficient" %}
|
||||
|
||||
@@ -15,7 +15,11 @@
|
||||
{% set label, cls = status_badge_config(task.status) %}
|
||||
{% set percent = progress_percent(task.processed_items_count, task.total_items_count) %}
|
||||
{% set ai_percent = rate_percent(task.analysis_success_rate) %}
|
||||
<p>平台：{{ task.platform | platform_label }} <span class="badge {{ cls }}">{{ label }}</span></p>
|
||||
{% set target_comments = task.hotspot_limit * task.item_limit_per_hotspot * task.comment_limit_per_item %}
|
||||
<div class="d-flex flex-wrap justify-content-between gap-3 mb-3">
|
||||
<p class="mb-0">平台：{{ task.platform | platform_label }} <span class="badge {{ cls }}">{{ label }}</span>{% if task.is_demo %}<span class="badge text-bg-info ms-1">Demo 数据</span>{% endif %}</p>
|
||||
<p class="mb-0 text-muted">阶段：{{ task.current_stage_label or "等待启动" }}</p>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<div class="d-flex justify-content-between align-items-center gap-2">
|
||||
<span>已处理 {{ task.processed_items_count }} / 共 {{ task.total_items_count }} 条内容</span>
|
||||
@@ -26,6 +30,32 @@
|
||||
</div>
|
||||
<div class="small text-muted mt-2">成功 {{ task.successful_items_count }} / 失败 {{ task.failed_items_count }}</div>
|
||||
</div>
|
||||
<div class="row g-3 mb-3">
|
||||
<div class="col-md-4">
|
||||
<div class="metric-box">
|
||||
<div class="text-muted small">目标上限</div>
|
||||
<strong>{{ task.hotspot_limit }} 热点 × {{ task.item_limit_per_hotspot }} 内容 × {{ task.comment_limit_per_item }} 评论</strong>
|
||||
<div class="text-muted small">最多 {{ target_comments }} 条评论</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="metric-box">
|
||||
<div class="text-muted small">实际结果</div>
|
||||
<strong>实际评论 {{ task.comments_count or 0 }}</strong>
|
||||
<div class="text-muted small">报告 {{ task.reports_count or 0 }} / 内容 {{ task.total_items_count }}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-4">
|
||||
<div class="metric-box">
|
||||
<div class="text-muted small">最近进度</div>
|
||||
<strong>{{ task.last_progress_at or task.created_at }}</strong>
|
||||
<div class="text-muted small">外部接口和 AI 响应较慢时，阶段可能短时间不变</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% if task.status == "success" and (task.comments_count or 0) < target_comments %}
|
||||
<div class="alert alert-info">少于理论上限通常是内容本身评论不足或平台返回不足，不直接代表任务失败。若失败内容数大于 0，请结合失败原因判断。</div>
|
||||
{% endif %}
|
||||
<div class="mb-2">
|
||||
<span class="{% if task.analysis_status == 'insufficient' %}text-warning{% else %}text-muted{% endif %}">AI 成功率 {{ ai_percent }}%</span>
|
||||
{% if task.analysis_status == "insufficient" %}
|
||||
|
||||
@@ -98,3 +98,90 @@ curl -f http://localhost:8000/health
|
||||
docker compose up -d --build
|
||||
curl -f http://localhost:8000/health
|
||||
```
|
||||
|
||||
## 公网云服务器部署
|
||||
|
||||
第一版公网演示使用云服务器 + Docker Compose，不增加登录或密码。
|
||||
|
||||
上线前准备：
|
||||
|
||||
```bash
|
||||
git pull
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
在 `.env` 中填写真实 Key：
|
||||
|
||||
```text
|
||||
TIKHUB_API_KEY=
|
||||
AI_BASE_URL=
|
||||
AI_API_KEY=
|
||||
AI_MODEL=
|
||||
```
|
||||
|
||||
启动：
|
||||
|
||||
```bash
|
||||
docker compose up -d --build
|
||||
curl -f http://127.0.0.1:8000/health
|
||||
```
|
||||
|
||||
如果服务器安全组直接开放端口，公网入口为：
|
||||
|
||||
```text
|
||||
http://服务器公网 IP:8000
|
||||
```
|
||||
|
||||
也可以用 Nginx 反向代理到本机 `127.0.0.1:8000`。
|
||||
|
||||
注意：
|
||||
|
||||
- 第一版公网不加访问控制，任何知道地址的人都可以访问页面。
|
||||
- 创建任务会消耗真实 TikHub 和 AI Key。
|
||||
- 系统仍保持同一时间只允许一个 running 任务，避免多人同时触发造成成本和稳定性问题。
|
||||
|
||||
## 初始化 Demo 数据
|
||||
|
||||
公网演示建议先初始化脱敏 Demo 数据：
|
||||
|
||||
```bash
|
||||
docker compose exec app python -m app.demo_seed
|
||||
```
|
||||
|
||||
Demo 数据特点：
|
||||
|
||||
- 使用脱敏真实感评论文本。
|
||||
- 不保存作者昵称、平台原始评论 ID、原始内容 URL 或 raw sensitive data。
|
||||
- 可以和新创建的真实任务同时出现在任务列表中。
|
||||
|
||||
## 数据库异常备份与恢复
|
||||
|
||||
如果页面出现数据库不可用提示，先不要删除 `data` 目录。
|
||||
|
||||
系统会优先备份现有 SQLite 文件到：
|
||||
|
||||
```text
|
||||
data/corrupt-backups/
|
||||
```
|
||||
|
||||
手动诊断：
|
||||
|
||||
```bash
|
||||
docker compose exec app python - <<'PY'
|
||||
from app.db import engine
|
||||
from sqlalchemy import text
|
||||
with engine.connect() as c:
|
||||
print(c.execute(text("PRAGMA integrity_check")).fetchall())
|
||||
print(c.exec_driver_sql("PRAGMA wal_checkpoint(TRUNCATE)").fetchall())
|
||||
PY
|
||||
```
|
||||
|
||||
如果需要重新初始化演示数据：
|
||||
|
||||
```bash
|
||||
docker compose down
|
||||
mv data/app.db data/corrupt-backups/app.db.manual.bak
|
||||
rm -f data/app.db-wal data/app.db-shm
|
||||
docker compose up -d --build
|
||||
docker compose exec app python -m app.demo_seed
|
||||
```
|
||||
|
||||
@@ -900,3 +900,433 @@ WO-10 收尾与 P1 评估
|
||||
验收结论：小红书 1×1×10 跑通，页面显示已完成，报告摘要正常
|
||||
遗留问题：任务列表自动刷新和进度条仍待做
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 13. MVP-2 新问题与产品化工单
|
||||
|
||||
本节记录 MVP 演示候选版本之后，用户在网页人工验收阶段发现的新问题和新增需求。
|
||||
|
||||
当前原则：
|
||||
|
||||
- 不覆盖 WO-01 到 WO-10 的完成记录。
|
||||
- 新问题按 MVP-2 工单继续推进。
|
||||
- 阻塞主链路的问题优先修复。
|
||||
- 不清楚的产品决策必须先确认，不擅自替用户决定。
|
||||
- 每个工单完成后继续按“完成日期 / commit / 验证命令 / 验收结论 / 遗留问题”记录。
|
||||
|
||||
### MVP-2 工单总览
|
||||
|
||||
| 编号 | 工单 | 优先级 | 目标 |
|
||||
|---|---|---|---|
|
||||
| WO-11 | 全站 Internal Server Error 排查与兜底 | P0 | 解决点击按钮或刷新页面出现 500 的阻塞问题 |
|
||||
| WO-12 | 导出 Markdown / CSV 点击失效修复 | P0 | 恢复报告和评论导出主链路 |
|
||||
| WO-13 | 抓取任务进度透明化与耗时预期 | P0 | 降低长任务黑盒感，让用户知道任务是否真的在推进 |
|
||||
| WO-14 | Demo 数据保留与新任务并存体验 | P0 | 打开页面即可看 demo 数据，同时还能新跑完整流程 |
|
||||
| WO-15 | 公网部署方案与上线验收 | P0 | 从本机 Docker 演示推进到可公网访问的部署 |
|
||||
| WO-16 | UI 产品化改版 | P1 | 在不破坏主链路的前提下提升页面观感和演示质感 |
|
||||
| WO-17 | 默认规模数据量解释与展示优化 | P1 | 把“未达到 1250”解释为真实内容/评论不足，并在 UI 中清楚呈现 |
|
||||
|
||||
### WO-11 全站 Internal Server Error 排查与兜底
|
||||
|
||||
优先级：P0
|
||||
|
||||
背景：
|
||||
|
||||
- 用户反馈：每次点击一个按钮或刷新页面时，页面可能显示 `Internal Server Error`。
|
||||
- 这是主链路阻塞问题，必须优先定位。
|
||||
|
||||
目标：
|
||||
|
||||
- 找出导致 500 的具体路由、异常堆栈和触发条件。
|
||||
- 对可恢复异常提供友好页面或错误提示，不让普通点击直接暴露 500。
|
||||
- 确保任务详情页、热点报告页、内容详情页、导出入口、刷新入口都不会因缺失数据直接崩溃。
|
||||
|
||||
包含范围：
|
||||
|
||||
- 查看 Docker / Uvicorn 日志，记录 500 对应的异常堆栈。
|
||||
- 覆盖任务详情页刷新、任务列表刷新、热点报告页、内容详情页、导出入口等高频路由。
|
||||
- 对缺失任务、缺失热点、缺失内容、缺失报告、数据库读取异常等场景增加兜底。
|
||||
- 增加回归测试，覆盖已发现的 500 触发路径。
|
||||
|
||||
边界情况：
|
||||
|
||||
- 任务 ID 不存在。
|
||||
- 报告记录不存在。
|
||||
- 内容条目存在但评论为空。
|
||||
- 任务处于 `running`，相关热点 / 内容 / 报告尚未生成。
|
||||
- 数据库暂时不可读或记录字段为空。
|
||||
- 浏览器重复刷新或重复点击按钮。
|
||||
|
||||
验收标准：
|
||||
|
||||
- 已知触发 500 的页面和按钮不再返回 `Internal Server Error`。
|
||||
- 真实异常在服务端日志中可定位，前端展示友好提示。
|
||||
- `pytest tests/unit tests/integration -q` 通过。
|
||||
- Docker 环境下手动刷新任务详情页、报告页、内容页不出现 500。
|
||||
|
||||
建议 commit：
|
||||
|
||||
```text
|
||||
fix: 修复页面刷新和按钮点击的 500 错误
|
||||
```
|
||||
|
||||
待确认：
|
||||
|
||||
- 用户需要提供或复现最容易触发 500 的页面 URL 与按钮名称；如果无法提供，则开发时先从当前浏览器打开的任务详情页开始排查。
|
||||
|
||||
### WO-12 导出 Markdown / CSV 点击失效修复
|
||||
|
||||
优先级：P0
|
||||
|
||||
背景：
|
||||
|
||||
- 用户反馈：点击导出 Markdown 文档和导出 CSV 评论时，页面没有反应，像是按钮失效。
|
||||
|
||||
目标：
|
||||
|
||||
- 恢复热点报告 Markdown、内容报告 Markdown、热点评论 CSV、内容评论 CSV 的下载能力。
|
||||
- 当报告尚未生成或数据为空时，按钮必须给出明确提示，而不是“点不动”。
|
||||
|
||||
包含范围：
|
||||
|
||||
- 检查导出按钮的前端事件绑定、HTMX / 普通链接行为、下载路由返回头。
|
||||
- 检查导出路由是否返回正确的 `Content-Type` 和 `Content-Disposition`。
|
||||
- 检查浏览器端是否被 disabled 状态、JS 错误或 500 响应卡住。
|
||||
- 报告缺失时提供友好提示。
|
||||
- 评论为空时仍可下载只有表头的 CSV，或按用户决策改为提示无评论。
|
||||
|
||||
边界情况：
|
||||
|
||||
- 报告尚未生成。
|
||||
- 评论数量为 0。
|
||||
- 文件名包含中文或特殊字符。
|
||||
- 浏览器拦截下载。
|
||||
- 导出接口返回 404 / 500。
|
||||
- 用户连续点击导出按钮。
|
||||
|
||||
验收标准：
|
||||
|
||||
- 页面点击导出后浏览器能下载文件，或看到明确的不可导出原因。
|
||||
- CSV 可打开且含 UTF-8-BOM、公式注入防护、换行替换。
|
||||
- Markdown 内容与页面报告一致。
|
||||
- 导出失败不会造成整页 500。
|
||||
|
||||
建议 commit：
|
||||
|
||||
```text
|
||||
fix: 修复报告和评论导出点击失效
|
||||
```
|
||||
|
||||
待确认：
|
||||
|
||||
- 评论为空时，用户希望“下载只有表头的 CSV”，还是“按钮禁用并提示暂无评论”。
|
||||
|
||||
### WO-13 抓取任务进度透明化与耗时预期
|
||||
|
||||
优先级：P0
|
||||
|
||||
背景：
|
||||
|
||||
- 用户反馈：开始抓取后虽然显示运行中，但不知道具体需要多少时间，也不知道系统是否真的在抓取。
|
||||
- 当前体验仍有黑盒感，尤其是默认规模任务会等待较久。
|
||||
|
||||
目标：
|
||||
|
||||
- 让任务详情页清楚展示当前阶段、已完成数量、失败数量、最近更新时间和粗略耗时预期。
|
||||
- 用户能判断任务是否仍在推进、是否卡住、卡在哪个阶段。
|
||||
|
||||
包含范围：
|
||||
|
||||
- 后端记录或计算任务阶段：
|
||||
- 获取热点中
|
||||
- 搜索内容中
|
||||
- 抓取评论中
|
||||
- AI 分析中
|
||||
- 生成报告中
|
||||
- 已完成 / 已失败
|
||||
- 展示当前进度：
|
||||
- 热点：已获取 / 目标
|
||||
- 内容：已处理 / 总数
|
||||
- 评论：已抓取数量
|
||||
- AI：成功率 / 失败数
|
||||
- 报告：已生成数量
|
||||
- 展示任务开始时间、运行时长、最近更新时间。
|
||||
- 给出非承诺式耗时提示，例如“小规模通常较快，默认规模可能需要数分钟，取决于 TikHub 与 AI 响应速度”。
|
||||
- 如果一段时间没有进度更新，展示“可能仍在等待外部接口响应”的提示。
|
||||
|
||||
边界情况：
|
||||
|
||||
- 刚开始运行，热点还没入库。
|
||||
- 已抓到热点但还没抓到内容。
|
||||
- 某个内容失败但任务继续。
|
||||
- 外部接口慢但未超时。
|
||||
- AI 请求慢。
|
||||
- Docker 重启导致任务中断。
|
||||
|
||||
验收标准：
|
||||
|
||||
- 创建任务后，任务详情页能看到阶段和进度数字变化。
|
||||
- 用户不需要打开日志，也能知道任务大概处于哪个阶段。
|
||||
- 任务长时间无变化时有提示，不再只显示“正在抓取热点数据，请稍候...”。
|
||||
- 终态为 success / failed 后停止轮询。
|
||||
|
||||
建议 commit：
|
||||
|
||||
```text
|
||||
feat: 增强任务进度和运行阶段展示
|
||||
```
|
||||
|
||||
待确认：
|
||||
|
||||
- 是否需要显示“预计剩余时间”。如果需要，建议第一版只显示粗略区间，不做精确倒计时，避免误导。
|
||||
|
||||
### WO-14 Demo 数据保留与新任务并存体验
|
||||
|
||||
优先级：P0
|
||||
|
||||
背景：
|
||||
|
||||
- 用户希望别人打开后既能看到已经跑通的 demo 数据，又能自己创建任务跑完整流程。
|
||||
- 当前 `data/` 是本地运行数据，未纳入 Git，不能直接当作可交付 demo 数据方案。
|
||||
|
||||
目标：
|
||||
|
||||
- 设计一套可控的 demo 数据方案，让页面首次打开就有可展示内容。
|
||||
- 同时保留用户创建新任务的能力。
|
||||
|
||||
可选方案：
|
||||
|
||||
| 方案 | 描述 | 优点 | 风险 |
|
||||
|---|---|---|---|
|
||||
| A | 保留本机 `data/app.db` 作为本地演示数据库，不提交 Git | 最快，适合自己电脑演示 | 换机器或公网部署时不可复现 |
|
||||
| B | 提供脱敏 seed 数据脚本，部署时生成 demo 任务 | 可复现，适合交付和公网部署 | 需要额外开发 seed 脚本 |
|
||||
| C | 提供 demo JSON / fixture，首次启动导入 | 可控、可版本化 | 需要确认哪些真实数据可以脱敏保存 |
|
||||
|
||||
包含范围：
|
||||
|
||||
- 首页展示 demo 任务和真实新任务。
|
||||
- 标记 demo 数据来源，避免和真实新任务混淆。
|
||||
- 提供重置 demo 数据或清空本地任务的说明。
|
||||
- 确认不提交 API Key、用户隐私、敏感评论作者信息。
|
||||
|
||||
边界情况：
|
||||
|
||||
- demo 数据和新抓取任务混在同一个任务列表。
|
||||
- 用户删除或重置数据库后 demo 数据消失。
|
||||
- 公网部署时没有本地 data 目录。
|
||||
- 真实评论内容可能包含敏感信息。
|
||||
|
||||
验收标准：
|
||||
|
||||
- 新用户打开页面能看到可点击的 demo 任务。
|
||||
- 用户仍能创建新任务并进入运行中状态。
|
||||
- demo 数据不依赖真实 API Key。
|
||||
- demo 数据不包含敏感凭证。
|
||||
|
||||
建议 commit：
|
||||
|
||||
```text
|
||||
feat: 增加可复现 demo 数据
|
||||
```
|
||||
|
||||
待确认：
|
||||
|
||||
- demo 数据使用真实抓取结果脱敏，还是使用模拟数据。
|
||||
- 是否允许在公网演示中展示真实评论文本和作者昵称。
|
||||
- demo 数据是否需要提供“一键恢复”能力。
|
||||
|
||||
### WO-15 公网部署方案与上线验收
|
||||
|
||||
优先级：P0
|
||||
|
||||
背景：
|
||||
|
||||
- 用户明确需要公网部署，而不仅是本机 Docker 访问。
|
||||
|
||||
目标：
|
||||
|
||||
- 选择并落地公网部署方式，提供可访问 URL。
|
||||
- 确保环境变量、数据目录、端口、健康检查、重启策略清楚可靠。
|
||||
|
||||
包含范围：
|
||||
|
||||
- 确认部署目标：
|
||||
- 云服务器 Docker Compose
|
||||
- PaaS 平台
|
||||
- 内网穿透 / 临时演示链接
|
||||
- 配置环境变量和密钥管理。
|
||||
- 配置持久化数据目录。
|
||||
- 配置反向代理或公网端口。
|
||||
- 配置健康检查和重启策略。
|
||||
- 更新部署文档。
|
||||
|
||||
边界情况：
|
||||
|
||||
- API Key 不应暴露在仓库或页面。
|
||||
- 公网访问可能产生额外抓取成本。
|
||||
- 多人同时点击创建任务。
|
||||
- SQLite 在公网多人使用下的并发限制。
|
||||
- 没有登录权限时，任何知道地址的人都能创建抓取任务。
|
||||
|
||||
验收标准：
|
||||
|
||||
- 用户可以通过公网 URL 打开首页。
|
||||
- 公网环境能查看 demo 数据。
|
||||
- 公网环境能创建至少一个小规模任务。
|
||||
- 健康检查可访问。
|
||||
- 重启后数据不丢失，或文档明确说明数据生命周期。
|
||||
|
||||
建议 commit：
|
||||
|
||||
```text
|
||||
docs: 补充公网部署方案
|
||||
```
|
||||
|
||||
或如果包含实际部署配置：
|
||||
|
||||
```text
|
||||
chore: 增加公网部署配置
|
||||
```
|
||||
|
||||
待确认：
|
||||
|
||||
- 部署平台选择。
|
||||
- 是否需要访问密码 / 简单登录。
|
||||
- 是否允许公网用户直接消耗真实 TikHub 和 AI Key。
|
||||
- 是否需要限制同一时间只能运行一个任务。
|
||||
|
||||
### WO-16 UI 产品化改版
|
||||
|
||||
优先级：P1
|
||||
|
||||
背景：
|
||||
|
||||
- 用户明确希望后续修改 UI。
|
||||
- 当前页面主链路可用，但仍需要提升产品化观感。
|
||||
|
||||
目标：
|
||||
|
||||
- 在不破坏功能的前提下，让首页、任务列表、任务详情、报告页、内容详情页更适合演示。
|
||||
|
||||
包含范围：
|
||||
|
||||
- 首页信息架构优化。
|
||||
- 任务列表更像仪表盘。
|
||||
- 任务详情页突出进度、阶段、失败原因、AI 成功率。
|
||||
- 热点报告页和内容详情页优化阅读层次。
|
||||
- 按钮状态更清楚：可点击、加载中、禁用、失败。
|
||||
- 空状态更友好。
|
||||
- 移动端基础适配。
|
||||
|
||||
边界情况：
|
||||
|
||||
- 文本过长。
|
||||
- 评论列表很多。
|
||||
- 无报告 / 无评论 / AI 失败。
|
||||
- 默认规模任务 running 很久。
|
||||
- 导出按钮不可用。
|
||||
|
||||
验收标准：
|
||||
|
||||
- 页面不会出现文字重叠、按钮挤压、信息难以扫描。
|
||||
- 主要 CTA 明确。
|
||||
- 运行中和失败态清楚。
|
||||
- UI 改动不影响创建任务、查看报告和导出。
|
||||
|
||||
建议 commit：
|
||||
|
||||
```text
|
||||
feat: 优化 MVP 页面产品化体验
|
||||
```
|
||||
|
||||
待确认：
|
||||
|
||||
- UI 风格方向：更偏数据仪表盘、内部工具，还是偏演示型产品页面。
|
||||
- 是否需要提供简单品牌名 / Logo / 说明文案。
|
||||
|
||||
### WO-17 默认规模数据量解释与展示优化
|
||||
|
||||
优先级：P1
|
||||
|
||||
背景：
|
||||
|
||||
- 默认规模理论值是 `5×5×50=1250` 评论，但真实任务不一定达到。
|
||||
- 用户判断这更可能是笔记 / 视频本身评论不足，而不是接口错误。
|
||||
|
||||
目标：
|
||||
|
||||
- 在 UI 和文档中清楚解释“目标上限”和“实际返回”的区别。
|
||||
- 避免用户看到少于 1250 就误以为任务失败。
|
||||
|
||||
包含范围：
|
||||
|
||||
- 任务详情页展示：
|
||||
- 配置目标：5 热点 × 5 内容 × 50 评论
|
||||
- 实际结果：实际热点、实际内容、实际评论
|
||||
- 不足原因提示：平台内容 / 评论不足、接口返回不足、部分内容失败
|
||||
- 报告页展示样本数，避免将样本不足包装成完整全量分析。
|
||||
- 文档补充默认规模解释。
|
||||
|
||||
边界情况：
|
||||
|
||||
- 某个热点只有少量内容。
|
||||
- 某条内容本身评论不足 50。
|
||||
- API 返回空评论但内容存在。
|
||||
- 部分内容失败导致评论不足。
|
||||
|
||||
验收标准：
|
||||
|
||||
- 用户能区分“抓取上限”和“实际抓到数量”。
|
||||
- 默认规模任务少于 1250 时，页面给出合理说明。
|
||||
- 真实失败和自然不足有不同提示。
|
||||
|
||||
建议 commit：
|
||||
|
||||
```text
|
||||
feat: 展示默认规模目标与实际抓取差异
|
||||
```
|
||||
|
||||
待确认：
|
||||
|
||||
- 是否需要在报告里显示“样本不足，不代表完整舆情”的提示。
|
||||
|
||||
## 14. MVP-2 推荐执行顺序
|
||||
|
||||
建议先处理阻塞主链路和演示可信度的问题：
|
||||
|
||||
```text
|
||||
WO-11 全站 500 排查与兜底
|
||||
↓
|
||||
WO-12 导出点击失效修复
|
||||
↓
|
||||
WO-13 抓取进度透明化
|
||||
↓
|
||||
WO-14 Demo 数据方案
|
||||
↓
|
||||
WO-15 公网部署
|
||||
↓
|
||||
WO-16 UI 产品化改版
|
||||
↓
|
||||
WO-17 默认规模数据量解释优化
|
||||
```
|
||||
|
||||
原因：
|
||||
|
||||
- 500 和导出失效会直接破坏验收，优先级最高。
|
||||
- 进度透明化解决“黑盒运行”的核心体验问题。
|
||||
- Demo 数据和公网部署强相关，应该在部署前明确。
|
||||
- UI 改版适合在主链路稳定后进行，避免把 bug 和视觉修改混在一起。
|
||||
|
||||
## 15. MVP-2 待用户确认问题
|
||||
|
||||
以下问题需要用户确认后再进入对应工单开发：
|
||||
|
||||
1. 公网部署选择哪种方式：云服务器 Docker Compose、PaaS、还是临时内网穿透演示。
|
||||
2. 公网访问是否需要密码 / 简单登录。
|
||||
3. 公网用户是否允许直接消耗真实 TikHub 和 AI Key。
|
||||
4. Demo 数据使用真实抓取结果脱敏，还是使用模拟数据。
|
||||
5. 公网 demo 是否允许展示真实评论文本和作者昵称。
|
||||
6. 评论为空时，CSV 导出是下载只有表头的文件，还是按钮禁用并提示暂无评论。
|
||||
7. 任务进度是否需要显示预计剩余时间，还是只显示阶段、运行时长和最近更新时间。
|
||||
8. UI 风格方向：内部数据仪表盘，还是更偏演示型产品页面。
|
||||
|
||||
@@ -26,6 +26,10 @@ dev = [
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["."]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["app*"]
|
||||
exclude = ["data*", "docs*", "tests*"]
|
||||
|
||||
[tool.coverage.run]
|
||||
branch = true
|
||||
source = ["app"]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from app.models import Comment, ContentItem, Hotspot, Report, Task
|
||||
from tests.helpers import make_test_client
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
|
||||
def test_index_page_renders_task_form_empty_state_and_default_scale():
|
||||
@@ -354,3 +355,59 @@ def test_export_routes_return_csv_and_markdown():
|
||||
assert "热点总结" in hotspot_md.text
|
||||
assert item_md.status_code == 200
|
||||
assert "内容总结" in item_md.text
|
||||
|
||||
|
||||
def test_api_tasks_returns_service_unavailable_when_database_has_io_error(monkeypatch):
|
||||
def broken_list_tasks(_session):
|
||||
raise OperationalError("SELECT 1", {}, Exception("disk I/O error"))
|
||||
|
||||
monkeypatch.setattr("app.main.list_tasks", broken_list_tasks)
|
||||
|
||||
with make_test_client() as (client, _engine):
|
||||
response = client.get("/api/tasks")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert response.json()["detail"] == "数据库暂时不可用，请稍后重试或联系维护者恢复数据。"
|
||||
|
||||
|
||||
def test_index_page_shows_database_error_state_when_database_has_io_error(monkeypatch):
|
||||
def broken_list_tasks(_session):
|
||||
raise OperationalError("SELECT 1", {}, Exception("disk I/O error"))
|
||||
|
||||
monkeypatch.setattr("app.main.list_tasks", broken_list_tasks)
|
||||
|
||||
with make_test_client() as (client, _engine):
|
||||
response = client.get("/")
|
||||
|
||||
assert response.status_code == 503
|
||||
assert "数据库暂时不可用" in response.text
|
||||
assert "请先保留 data 目录" in response.text
|
||||
|
||||
|
||||
def test_task_api_includes_progress_counts_stage_and_demo_flag():
|
||||
with make_test_client() as (client, engine):
|
||||
seed_result_data(engine)
|
||||
|
||||
response = client.get("/api/tasks/task-result")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["current_stage"] in (None, "success")
|
||||
assert data["current_stage_label"] == "已完成"
|
||||
assert data["comments_count"] == 1
|
||||
assert data["reports_count"] == 2
|
||||
assert data["is_demo"] is False
|
||||
assert data["last_progress_at"] is not None
|
||||
|
||||
|
||||
def test_task_detail_page_explains_target_and_actual_counts():
|
||||
with make_test_client() as (client, engine):
|
||||
seed_result_data(engine)
|
||||
|
||||
response = client.get("/tasks/task-result")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "目标上限" in response.text
|
||||
assert "实际结果" in response.text
|
||||
assert "实际评论 1" in response.text
|
||||
assert "少于理论上限通常是内容本身评论不足或平台返回不足" in response.text
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import pytest
|
||||
|
||||
from app.db import check_database_integrity, checkpoint_sqlite_wal, create_sqlite_engine
|
||||
from app.db import check_database_integrity, checkpoint_sqlite_wal, create_sqlite_engine, ensure_sqlite_schema_compat
|
||||
|
||||
|
||||
def test_check_database_integrity_returns_ok_for_valid_sqlite_database():
|
||||
@@ -63,3 +63,20 @@ def test_checkpoint_sqlite_wal_skips_in_memory_database():
|
||||
assert checkpoint_sqlite_wal(engine) is False
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_ensure_sqlite_schema_compat_adds_progress_columns_to_existing_tasks_table(tmp_path):
|
||||
db_path = tmp_path / "legacy.db"
|
||||
engine = create_sqlite_engine(f"sqlite:///{db_path}")
|
||||
try:
|
||||
with engine.begin() as connection:
|
||||
connection.exec_driver_sql("CREATE TABLE tasks (id VARCHAR(36) PRIMARY KEY, platform VARCHAR(32) NOT NULL)")
|
||||
|
||||
ensure_sqlite_schema_compat(engine)
|
||||
|
||||
with engine.connect() as connection:
|
||||
columns = {row[1] for row in connection.exec_driver_sql("PRAGMA table_info(tasks)").all()}
|
||||
assert "current_stage" in columns
|
||||
assert "last_progress_at" in columns
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.demo_seed import seed_demo_data
|
||||
from app.models import Comment, ContentItem, Task
|
||||
from tests.helpers import make_test_client
|
||||
|
||||
|
||||
def test_seed_demo_data_is_idempotent_and_removes_sensitive_fields():
|
||||
with make_test_client() as (_client, engine):
|
||||
with Session(engine) as session:
|
||||
task_id = seed_demo_data(session)
|
||||
second_task_id = seed_demo_data(session)
|
||||
|
||||
task = session.get(Task, task_id)
|
||||
comments = session.query(Comment).all()
|
||||
items = session.query(ContentItem).all()
|
||||
|
||||
assert second_task_id == task_id
|
||||
assert task is not None
|
||||
assert task.id.startswith("demo-")
|
||||
assert task.status == "success"
|
||||
assert comments
|
||||
assert all(comment.author is None for comment in comments)
|
||||
assert all(comment.source_comment_id is None for comment in comments)
|
||||
assert all(comment.raw_data == "{}" for comment in comments)
|
||||
assert all(item.url is None for item in items)
|
||||
assert all(item.raw_data == "{}" for item in items)
|
||||
@@ -17,5 +17,8 @@ def test_user_guide_covers_startup_acceptance_exports_and_troubleshooting():
|
||||
"API Key 缺失",
|
||||
"任务长期 running",
|
||||
"重置本地数据库",
|
||||
"公网云服务器部署",
|
||||
"python -m app.demo_seed",
|
||||
"data/corrupt-backups",
|
||||
]:
|
||||
assert required in content
|
||||
|
||||
Reference in New Issue
Block a user