feat: 提交热榜评论分析工具 MVP 基线

2026-07-03 11:12:12 +08:00
parent 0231be6338
commit 9bad7b6e63
60 changed files with 4058 additions and 0 deletions
@@ -0,0 +1 @@
+"""Integration tests."""
@@ -0,0 +1,51 @@
+from sqlalchemy.orm import Session
+
+from app.platforms.base import PlatformAPIError
+from app.schemas import CreateTaskRequest
+from app.services.task_service import create_task, run_task
+from tests.helpers import make_test_client
+
+
+class FailingHotspotPlatform:
+    def fetch_hotspots(self, *, limit):
+        raise PlatformAPIError("hotspot failed", error_type="api_error", status_code=500)
+
+
+def test_hotspot_failure_marks_task_failed(monkeypatch):
+    with make_test_client() as (_client, engine):
+        monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FailingHotspotPlatform())
+        with Session(engine) as session:
+            task = create_task(
+                session,
+                CreateTaskRequest(platform="douyin", hotspot_limit=1, item_limit_per_hotspot=1, comment_limit_per_item=10),
+                submit_background=False,
+            )
+            task_id = task.id
+            run_task(task_id, session_factory=lambda: session)
+            task = session.get(type(task), task_id)
+
+        assert task.status == "failed"
+        assert task.error_stage == "crawl_hotspots"
+        assert task.error_type == "api_error"
+
+
+def test_unexpected_task_exception_marks_task_failed(monkeypatch):
+    def raise_unexpected_error(_platform):
+        raise RuntimeError("boom")
+
+    with make_test_client() as (_client, engine):
+        monkeypatch.setattr("app.services.task_service.build_platform", raise_unexpected_error)
+        with Session(engine) as session:
+            task = create_task(
+                session,
+                CreateTaskRequest(platform="douyin", hotspot_limit=1, item_limit_per_hotspot=1, comment_limit_per_item=10),
+                submit_background=False,
+            )
+            task_id = task.id
+            run_task(task_id, session_factory=lambda: session)
+            task = session.get(type(task), task_id)
+
+        assert task.status == "failed"
+        assert task.error_stage == "system"
+        assert task.error_type == "unexpected_error"
+        assert task.error_message == "boom"
@@ -0,0 +1,12 @@
+from fastapi.testclient import TestClient
+
+from app.main import app
+
+
+def test_health_returns_ok():
+    client = TestClient(app)
+
+    response = client.get("/health")
+
+    assert response.status_code == 200
+    assert response.json() == {"status": "ok"}
@@ -0,0 +1,146 @@
+from app.models import Comment, ContentItem, Hotspot, Report, Task
+from tests.helpers import make_test_client
+
+
+def test_index_page_renders_task_form_empty_state_and_default_scale():
+    with make_test_client() as (client, _engine):
+        response = client.get("/")
+
+    assert response.status_code == 200
+    assert "热榜评论分析工具" in response.text
+    assert 'name="platform"' in response.text
+    assert 'name="hotspot_limit"' in response.text
+    assert 'name="item_limit_per_hotspot"' in response.text
+    assert 'name="comment_limit_per_item"' in response.text
+    assert "1250" in response.text
+    assert 'onclick="window.location.href=\'/\'"' in response.text
+    assert "还没有任何任务" in response.text
+
+
+def test_index_page_lists_existing_tasks():
+    with make_test_client() as (client, engine):
+        from sqlalchemy.orm import Session
+
+        with Session(engine) as session:
+            session.add(
+                Task(
+                    id="task-visible",
+                    platform="douyin",
+                    status="running",
+                    hotspot_limit=3,
+                    item_limit_per_hotspot=4,
+                    comment_limit_per_item=50,
+                )
+            )
+            session.commit()
+
+        response = client.get("/")
+
+    assert response.status_code == 200
+    assert "task-visible" in response.text
+    assert "抖音" in response.text
+    assert "运行中" in response.text
+
+
+def seed_result_data(engine):
+    from sqlalchemy.orm import Session
+
+    with Session(engine) as session:
+        task = Task(id="task-result", platform="douyin", status="success", total_items_count=1, successful_items_count=1)
+        session.add(task)
+        hotspot = Hotspot(id="hot-result", task_id=task.id, platform="douyin", title="热点标题", rank=1, raw_data="{}")
+        session.add(hotspot)
+        item = ContentItem(
+            id="item-result",
+            task_id=task.id,
+            hotspot_id=hotspot.id,
+            platform="douyin",
+            source_item_id="v1",
+            item_type="video",
+            title="视频标题",
+            status="success",
+            raw_data="{}",
+        )
+        session.add(item)
+        session.add(
+            Comment(
+                id="comment-result",
+                task_id=task.id,
+                hotspot_id=hotspot.id,
+                content_item_id=item.id,
+                platform="douyin",
+                source_comment_id="c1",
+                content="评论内容",
+                sentiment="positive",
+                labels='["认可"]',
+                like_count=3,
+                raw_data="{}",
+            )
+        )
+        session.add(
+            Report(
+                task_id=task.id,
+                hotspot_id=hotspot.id,
+                report_type="hotspot",
+                title="热点标题",
+                metrics_json='{"sample_count":1,"sentiment":{"positive":{"count":1,"pct":100}},"top_labels":[{"name":"认可","count":1}]}',
+                typical_comments_json='{"positive":[{"content":"评论内容","like_count":3}]}',
+                summary="热点总结",
+                markdown_content="# 热点标题\n\n热点总结",
+                data="{}",
+                markdown="# 热点标题\n\n热点总结",
+            )
+        )
+        session.add(
+            Report(
+                task_id=task.id,
+                hotspot_id=hotspot.id,
+                content_item_id=item.id,
+                report_type="item",
+                title="视频标题",
+                metrics_json='{"sample_count":1,"sentiment":{"positive":{"count":1,"pct":100}},"top_labels":[{"name":"认可","count":1}]}',
+                typical_comments_json='{"positive":[{"content":"评论内容","like_count":3}]}',
+                summary="内容总结",
+                markdown_content="# 视频标题\n\n内容总结",
+                data="{}",
+                markdown="# 视频标题\n\n内容总结",
+            )
+        )
+        session.commit()
+
+
+def test_result_pages_render_seeded_data():
+    with make_test_client() as (client, engine):
+        seed_result_data(engine)
+
+        task_response = client.get("/tasks/task-result")
+        hotspot_response = client.get("/hotspots/hot-result/report")
+        item_response = client.get("/items/item-result")
+
+    assert task_response.status_code == 200
+    assert "热点标题" in task_response.text
+    assert 'onclick="window.location.href=\'/\'"' in task_response.text
+    assert hotspot_response.status_code == 200
+    assert "热点总结" in hotspot_response.text
+    assert item_response.status_code == 200
+    assert "评论内容" in item_response.text
+
+
+def test_export_routes_return_csv_and_markdown():
+    with make_test_client() as (client, engine):
+        seed_result_data(engine)
+
+        csv_response = client.get("/api/export/items/item-result/comments.csv")
+        hotspot_csv_response = client.get("/api/export/hotspots/hot-result/comments.csv")
+        hotspot_md = client.get("/api/export/hotspots/hot-result.md")
+        item_md = client.get("/api/export/items/item-result.md")
+
+    assert csv_response.status_code == 200
+    assert csv_response.content.startswith(b"\xef\xbb\xbf")
+    assert "评论内容".encode("utf-8") in csv_response.content
+    assert hotspot_csv_response.status_code == 200
+    assert "评论内容".encode("utf-8") in hotspot_csv_response.content
+    assert hotspot_md.status_code == 200
+    assert "热点总结" in hotspot_md.text
+    assert item_md.status_code == 200
+    assert "内容总结" in item_md.text
@@ -0,0 +1,118 @@
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine
+from sqlalchemy.pool import StaticPool
+
+from app.db import Base, get_db_session
+from app.main import app
+from app.models import Task
+
+
+def make_test_client():
+    engine = create_engine(
+        "sqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    Base.metadata.create_all(engine)
+
+    def override_db_session():
+        from sqlalchemy.orm import Session
+
+        with Session(engine) as session:
+            yield session
+
+    app.dependency_overrides[get_db_session] = override_db_session
+    return TestClient(app), engine
+
+
+def test_create_task_returns_task_id_and_running_status():
+    client, engine = make_test_client()
+    try:
+        response = client.post(
+            "/api/tasks",
+            json={
+                "platform": "xiaohongshu",
+                "hotspot_limit": 5,
+                "item_limit_per_hotspot": 5,
+                "comment_limit_per_item": 50,
+            },
+        )
+
+        assert response.status_code == 201
+        data = response.json()
+        assert data["task_id"]
+        assert data["status"] == "running"
+    finally:
+        engine.dispose()
+        app.dependency_overrides.clear()
+
+
+def test_create_task_rejects_invalid_platform_and_limits():
+    client, engine = make_test_client()
+    try:
+        response = client.post(
+            "/api/tasks",
+            json={
+                "platform": "weibo",
+                "hotspot_limit": 11,
+                "item_limit_per_hotspot": 0,
+                "comment_limit_per_item": 101,
+            },
+        )
+
+        assert response.status_code == 422
+    finally:
+        engine.dispose()
+        app.dependency_overrides.clear()
+
+
+def test_create_task_returns_400_when_running_task_exists():
+    client, engine = make_test_client()
+    try:
+        from sqlalchemy.orm import Session
+
+        with Session(engine) as session:
+            session.add(Task(platform="douyin", status="running"))
+            session.commit()
+
+        response = client.post(
+            "/api/tasks",
+            json={
+                "platform": "douyin",
+                "hotspot_limit": 5,
+                "item_limit_per_hotspot": 5,
+                "comment_limit_per_item": 50,
+            },
+        )
+
+        assert response.status_code == 400
+        assert response.json() == {"detail": "当前有正在运行的任务，请稍后再试"}
+    finally:
+        engine.dispose()
+        app.dependency_overrides.clear()
+
+
+def test_task_list_and_detail_return_created_tasks():
+    client, engine = make_test_client()
+    try:
+        created = client.post(
+            "/api/tasks",
+            json={
+                "platform": "douyin",
+                "hotspot_limit": 3,
+                "item_limit_per_hotspot": 4,
+                "comment_limit_per_item": 30,
+            },
+        ).json()
+
+        list_response = client.get("/api/tasks")
+        detail_response = client.get(f"/api/tasks/{created['task_id']}")
+
+        assert list_response.status_code == 200
+        assert list_response.json()[0]["task_id"] == created["task_id"]
+        assert detail_response.status_code == 200
+        assert detail_response.json()["platform"] == "douyin"
+        assert detail_response.json()["hotspot_limit"] == 3
+    finally:
+        engine.dispose()
+        app.dependency_overrides.clear()
@@ -0,0 +1,39 @@
+from sqlalchemy.orm import Session
+
+from app.models import Comment, Task
+from app.platforms.base import CommentData, ContentItemData, HotspotData
+from app.schemas import CreateTaskRequest
+from app.services.task_service import create_task, run_task
+from tests.helpers import make_test_client
+
+
+class FakeDouyinPlatform:
+    def fetch_hotspots(self, *, limit):
+        return [HotspotData(source_hot_id="d1", title="抖音热点", rank=1, heat_value="200", raw_data={})]
+
+    def search_items_by_hotspot(self, keyword, *, limit):
+        return [ContentItemData(source_item_id="v1", item_type="video", title="视频", raw_data={})]
+
+    def fetch_comments(self, source_item_id, *, limit):
+        return [CommentData(source_comment_id="dc1", content="抖音评论", raw_data={})]
+
+
+def test_douyin_task_flow_persists_comments(monkeypatch):
+    with make_test_client() as (_client, engine):
+        monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeDouyinPlatform())
+
+        with Session(engine) as session:
+            task = create_task(
+                session,
+                CreateTaskRequest(platform="douyin", hotspot_limit=1, item_limit_per_hotspot=1, comment_limit_per_item=10),
+                submit_background=False,
+            )
+            task_id = task.id
+
+        with Session(engine) as session:
+            run_task(task_id, session_factory=lambda: session)
+            task = session.get(Task, task_id)
+            comment = session.query(Comment).filter_by(task_id=task_id).one()
+
+        assert task.status == "success"
+        assert comment.content == "抖音评论"
@@ -0,0 +1,130 @@
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from app.models import Comment, ContentItem, Hotspot, Report, Task
+from app.schemas import CreateTaskRequest
+from app.services.task_service import create_task, run_task
+from tests.helpers import make_test_client
+
+
+class FakeXhsPlatform:
+    def fetch_hotspots(self, *, limit):
+        from app.platforms.base import HotspotData
+
+        return [HotspotData(source_hot_id="h1", title="热点一", rank=1, heat_value="100", raw_data={"id": "h1"})]
+
+    def search_items_by_hotspot(self, keyword, *, limit):
+        from app.platforms.base import ContentItemData
+
+        return [
+            ContentItemData(source_item_id="n1", item_type="note", title=f"{keyword} 笔记", raw_data={"id": "n1"})
+        ]
+
+    def fetch_comments(self, source_item_id, *, limit):
+        from app.platforms.base import CommentData
+
+        return [CommentData(source_comment_id="c1", content="评论一", like_count=5, raw_data={"id": "c1"})]
+
+
+def test_xiaohongshu_task_flow_persists_hotspots_items_and_comments(monkeypatch):
+    with make_test_client() as (_client, engine):
+        monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform())
+        monkeypatch.setattr(
+            "app.services.task_service.build_ai_requester",
+            lambda: (
+                lambda prompt: (
+                    '[{"comment_id":"'
+                    + ("c1" if "c1" in prompt else prompt.split('"comment_id": "')[1].split('"')[0])
+                    + '","sentiment":"positive","labels":["认可"],"reason":"喜欢"}]'
+                )
+            ),
+        )
+
+        with Session(engine) as session:
+            task = create_task(
+                session,
+                CreateTaskRequest(platform="xiaohongshu", hotspot_limit=1, item_limit_per_hotspot=1, comment_limit_per_item=10),
+                submit_background=False,
+            )
+            task_id = task.id
+
+        with Session(engine) as session:
+            run_task(task_id, session_factory=lambda: session)
+
+            persisted = session.get(Task, task_id)
+            assert persisted.status == "success"
+            assert persisted.processed_items_count == 1
+            assert persisted.successful_items_count == 1
+            assert persisted.analysis_status == "normal"
+            assert session.scalar(select(Hotspot).where(Hotspot.task_id == task_id)).title == "热点一"
+            assert session.scalar(select(ContentItem).where(ContentItem.task_id == task_id)).source_item_id == "n1"
+            comment = session.scalar(select(Comment).where(Comment.task_id == task_id))
+            assert comment.content == "评论一"
+            assert comment.ai_analysis_status == "success"
+            assert comment.sentiment == "positive"
+            assert comment.labels == '["认可"]'
+            assert comment.reason == "喜欢"
+            assert session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "item")) is not None
+            assert session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "hotspot")) is not None
+
+
+def test_xiaohongshu_task_flow_marks_comments_failed_when_ai_parse_fails(monkeypatch):
+    with make_test_client() as (_client, engine):
+        monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform())
+        monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: (lambda _prompt: "not-json"))
+        monkeypatch.setattr("app.services.ai_service.time.sleep", lambda _seconds: None)
+
+        with Session(engine) as session:
+            task = create_task(
+                session,
+                CreateTaskRequest(platform="xiaohongshu", hotspot_limit=1, item_limit_per_hotspot=1, comment_limit_per_item=10),
+                submit_background=False,
+            )
+            task_id = task.id
+
+        with Session(engine) as session:
+            run_task(task_id, session_factory=lambda: session)
+
+            persisted = session.get(Task, task_id)
+            comment = session.scalar(select(Comment).where(Comment.task_id == task_id))
+
+            assert persisted.status == "success"
+            assert persisted.analysis_status == "insufficient"
+            assert persisted.analysis_success_rate == 0.0
+            assert comment.ai_analysis_status == "failed"
+            assert comment.sentiment == "unknown"
+            assert comment.labels == "[]"
+            assert comment.reason == "ai_parse_failed"
+
+
+def test_xiaohongshu_task_flow_keeps_item_success_when_ai_request_raises(monkeypatch):
+    with make_test_client() as (_client, engine):
+        monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform())
+        monkeypatch.setattr("app.services.ai_service.time.sleep", lambda _seconds: None)
+
+        def failing_requester(_prompt):
+            raise RuntimeError("ai unauthorized")
+
+        monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: failing_requester)
+
+        with Session(engine) as session:
+            task = create_task(
+                session,
+                CreateTaskRequest(platform="xiaohongshu", hotspot_limit=1, item_limit_per_hotspot=1, comment_limit_per_item=10),
+                submit_background=False,
+            )
+            task_id = task.id
+
+        with Session(engine) as session:
+            run_task(task_id, session_factory=lambda: session)
+
+            persisted = session.get(Task, task_id)
+            comment = session.scalar(select(Comment).where(Comment.task_id == task_id))
+
+            assert persisted.status == "success"
+            assert persisted.successful_items_count == 1
+            assert persisted.failed_items_count == 0
+            assert persisted.analysis_status == "insufficient"
+            assert persisted.analysis_success_rate == 0.0
+            assert comment.ai_analysis_status == "failed"
+            assert comment.reason == "ai_parse_failed"
@@ -0,0 +1,40 @@
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session, sessionmaker
+from sqlalchemy.pool import StaticPool
+
+from app.db import Base
+from app.main import app
+from app.models import Task
+
+
+def test_lifespan_marks_running_tasks_failed_after_restart(monkeypatch):
+    engine = create_engine(
+        "sqlite:///:memory:",
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+    TestingSessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)
+    Base.metadata.create_all(engine)
+
+    with TestingSessionLocal() as session:
+        session.add(Task(platform="xiaohongshu", status="running"))
+        session.commit()
+
+    monkeypatch.setattr("app.main.SessionLocal", TestingSessionLocal, raising=False)
+    monkeypatch.setattr("app.main.init_db", lambda: None)
+
+    try:
+        with TestClient(app):
+            pass
+
+        with Session(engine) as session:
+            task = session.scalar(select(Task))
+
+        assert task is not None
+        assert task.status == "failed"
+        assert task.error_stage == "system"
+        assert task.error_type == "unexpected_restart"
+        assert task.error_message == "系统重启，任务被中断"
+    finally:
+        engine.dispose()