feat: 提交热榜评论分析工具 MVP 基线

tests/unit/__init__.py

@@ -0,0 +1 @@
+"""Unit tests."""

tests/unit/test_ai_schema.py

@@ -0,0 +1,119 @@
+import pytest
+import httpx
+
+from app.services.ai_service import (
+    AIAnalysisResult,
+    OpenAICompatibleAIClient,
+    analyze_comments_with_retry,
+    build_comment_prompt,
+    calculate_analysis_status,
+    parse_ai_comment_response,
+)
+
+
+def test_build_comment_prompt_contains_ids_and_truncates_content():
+    prompt = build_comment_prompt([{"comment_id": "c1", "content": "你" * 200}])
+
+    assert "c1" in prompt
+    assert "请原样回填输入中的 comment_id" in prompt
+    assert "你" * 150 in prompt
+    assert "你" * 151 not in prompt
+
+
+def test_parse_ai_comment_response_validates_array_sentiment_labels_and_ids():
+    result = parse_ai_comment_response(
+        '[{"comment_id":"c1","sentiment":"positive","labels":["质量好"],"reason":"认可"}]',
+        expected_comment_ids={"c1"},
+    )
+
+    assert result == [AIAnalysisResult(comment_id="c1", sentiment="positive", labels=["质量好"], reason="认可")]
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        "not-json",
+        '{"comment_id":"c1"}',
+        '[{"comment_id":"missing","sentiment":"positive","labels":[],"reason":""}]',
+        '[{"comment_id":"c1","sentiment":"happy","labels":[],"reason":""}]',
+        '[{"comment_id":"c1","sentiment":"positive","labels":["a","b","c","d"],"reason":""}]',
+    ],
+)
+def test_parse_ai_comment_response_rejects_invalid_payloads(payload):
+    with pytest.raises(ValueError):
+        parse_ai_comment_response(payload, expected_comment_ids={"c1"})
+
+
+def test_calculate_analysis_status_uses_eighty_percent_threshold():
+    assert calculate_analysis_status(success_count=8, total_count=10) == (0.8, "normal")
+    assert calculate_analysis_status(success_count=7, total_count=10) == (0.7, "insufficient")
+    assert calculate_analysis_status(success_count=0, total_count=0) == (0.0, "insufficient")
+
+
+def test_analyze_comments_with_retry_marks_batch_failed_after_three_parse_failures(monkeypatch):
+    sleeps = []
+    monkeypatch.setattr("app.services.ai_service.time.sleep", sleeps.append)
+
+    results = analyze_comments_with_retry(
+        [{"comment_id": "c1", "content": "内容"}],
+        requester=lambda _prompt: "not-json",
+        max_retries=3,
+    )
+
+    assert results[0].comment_id == "c1"
+    assert results[0].sentiment == "unknown"
+    assert results[0].ai_analysis_status == "failed"
+    assert results[0].reason == "ai_parse_failed"
+    assert sleeps == [1, 2]
+
+
+def test_openai_compatible_client_posts_chat_completion_and_returns_message_content():
+    captured = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        captured["url"] = str(request.url)
+        captured["authorization"] = request.headers.get("authorization")
+        captured["body"] = request.read().decode("utf-8")
+        return httpx.Response(
+            200,
+            json={"choices": [{"message": {"content": '[{"comment_id":"c1","sentiment":"positive","labels":["认可"],"reason":"喜欢"}]'}}]},
+        )
+
+    http_client = httpx.Client(transport=httpx.MockTransport(handler))
+    client = OpenAICompatibleAIClient(
+        base_url="https://ai.example.com",
+        api_key="test-ai-key",
+        model="test-model",
+        http_client=http_client,
+    )
+
+    result = client.request("prompt text")
+
+    assert result == '[{"comment_id":"c1","sentiment":"positive","labels":["认可"],"reason":"喜欢"}]'
+    assert captured["url"] == "https://ai.example.com/v1/chat/completions"
+    assert captured["authorization"] == "Bearer test-ai-key"
+    assert '"model":"test-model"' in captured["body"]
+    assert "prompt text" in captured["body"]
+
+
+def test_openai_compatible_client_accepts_base_url_that_already_includes_v1():
+    captured = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        captured["url"] = str(request.url)
+        return httpx.Response(
+            200,
+            json={"choices": [{"message": {"content": "[]"}}]},
+        )
+
+    http_client = httpx.Client(transport=httpx.MockTransport(handler))
+    client = OpenAICompatibleAIClient(
+        base_url="https://ai.example.com/v1",
+        api_key="test-ai-key",
+        model="test-model",
+        http_client=http_client,
+    )
+
+    client.request("prompt text")
+
+    assert captured["url"] == "https://ai.example.com/v1/chat/completions"

tests/unit/test_comment_pagination.py

@@ -0,0 +1,51 @@
+import httpx
+import pytest
+
+from app.platforms.base import PlatformAPIError, TikHubClient
+
+
+class SequenceTransport:
+    def __init__(self, responses):
+        self.responses = list(responses)
+        self.requests = []
+
+    def __call__(self, request: httpx.Request) -> httpx.Response:
+        self.requests.append(request)
+        response = self.responses.pop(0)
+        response.request = request
+        return response
+
+
+def test_tikhub_client_retries_429_with_exponential_backoff(monkeypatch):
+    sleeps = []
+    monkeypatch.setattr("app.platforms.base.time.sleep", sleeps.append)
+    transport = SequenceTransport(
+        [
+            httpx.Response(429, json={"message": "Too Many Requests"}),
+            httpx.Response(429, json={"message": "Too Many Requests"}),
+            httpx.Response(200, json={"ok": True}),
+        ]
+    )
+    http_client = httpx.Client(transport=httpx.MockTransport(transport))
+    client = TikHubClient(base_url="https://api.test", api_key="secret-token", http_client=http_client)
+
+    result = client.get("/demo")
+
+    assert result == {"ok": True}
+    assert sleeps == [1, 2]
+    assert len(transport.requests) == 3
+    assert transport.requests[0].headers["Authorization"] == "Bearer secret-token"
+
+
+def test_tikhub_client_raises_structured_error_after_retries(monkeypatch):
+    monkeypatch.setattr("app.platforms.base.time.sleep", lambda _seconds: None)
+    transport = SequenceTransport([httpx.Response(429, json={"message": "Too Many Requests"}) for _ in range(4)])
+    http_client = httpx.Client(transport=httpx.MockTransport(transport))
+    client = TikHubClient(base_url="https://api.test", api_key="secret-token", http_client=http_client)
+
+    with pytest.raises(PlatformAPIError) as exc_info:
+        client.get("/demo")
+
+    assert exc_info.value.error_type == "rate_limited"
+    assert exc_info.value.status_code == 429
+    assert "secret-token" not in str(exc_info.value)

tests/unit/test_config.py

@@ -0,0 +1,76 @@
+from app.config import Settings
+
+
+def test_settings_use_t01_defaults(monkeypatch):
+    monkeypatch.delenv("APP_ENV", raising=False)
+    monkeypatch.delenv("DATABASE_URL", raising=False)
+    monkeypatch.delenv("TIKHUB_BASE_URL", raising=False)
+    monkeypatch.delenv("AI_PROVIDER", raising=False)
+
+    settings = Settings(_env_file=None)
+
+    assert settings.app_env == "development"
+    assert settings.database_url == "sqlite:///data/app.db"
+    assert settings.tikhub_base_url == "https://api.tikhub.io"
+    assert settings.ai_provider == "openai-compatible"
+
+
+def test_settings_read_environment_overrides(monkeypatch):
+    monkeypatch.setenv("APP_ENV", "test")
+    monkeypatch.setenv("DATABASE_URL", "sqlite:///:memory:")
+    monkeypatch.setenv("TIKHUB_API_KEY", "test-token")
+    monkeypatch.setenv("AI_API_KEY", "test-ai-key")
+
+    settings = Settings(_env_file=None)
+
+    assert settings.app_env == "test"
+    assert settings.database_url == "sqlite:///:memory:"
+    assert settings.tikhub_api_key == "test-token"
+    assert settings.ai_api_key == "test-ai-key"
+
+
+def test_settings_include_t02_defaults(monkeypatch):
+    monkeypatch.delenv("AI_BATCH_SIZE", raising=False)
+    monkeypatch.delenv("AI_CONCURRENCY", raising=False)
+    monkeypatch.delenv("AI_MAX_RETRIES", raising=False)
+    monkeypatch.delenv("AI_TIMEOUT_SECONDS", raising=False)
+    monkeypatch.delenv("HTTP_TIMEOUT_SECONDS", raising=False)
+    monkeypatch.delenv("HTTP_MAX_RETRIES", raising=False)
+    monkeypatch.delenv("CRAWL_PAGE_INTERVAL_SECONDS", raising=False)
+
+    settings = Settings(_env_file=None)
+
+    assert settings.ai_batch_size == 20
+    assert settings.ai_concurrency == 2
+    assert settings.ai_max_retries == 3
+    assert settings.ai_timeout_seconds == 30
+    assert settings.http_timeout_seconds == 20
+    assert settings.http_max_retries == 3
+    assert settings.crawl_page_interval_seconds == 1.5
+
+
+def test_settings_read_crawl_page_interval_from_environment(monkeypatch):
+    monkeypatch.setenv("CRAWL_PAGE_INTERVAL_SECONDS", "0.25")
+
+    settings = Settings(_env_file=None)
+
+    assert settings.crawl_page_interval_seconds == 0.25
+
+
+def test_ai_concurrency_has_hard_upper_bound(monkeypatch):
+    monkeypatch.setenv("AI_CONCURRENCY", "4")
+
+    settings = Settings(_env_file=None)
+
+    assert settings.ai_concurrency == 3
+
+
+def test_task_limit_ranges_are_available_on_settings():
+    settings = Settings(_env_file=None)
+
+    assert settings.hot_limit_min == 1
+    assert settings.hot_limit_max == 10
+    assert settings.item_limit_per_hot_min == 1
+    assert settings.item_limit_per_hot_max == 10
+    assert settings.comment_limit_per_item_min == 10
+    assert settings.comment_limit_per_item_max == 100

tests/unit/test_deployment_config.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_docker_compose_reads_real_env_file_not_example():
+    compose_content = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
+
+    assert "- .env\n" in compose_content
+    assert "- .env.example" not in compose_content
+
+
+def test_real_env_file_is_gitignored():
+    gitignore_lines = (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
+
+    assert ".env" in gitignore_lines

tests/unit/test_douyin_mapping.py

@@ -0,0 +1,120 @@
+from app.platforms.douyin import DouyinPlatform
+
+
+def test_douyin_maps_hotspots():
+    platform = DouyinPlatform(client=None)
+    payload = {
+        "data": {
+            "word_list": [
+                {"query_id": "q1", "title": "热点", "rank": 2, "hot_score": "888"}
+            ]
+        }
+    }
+
+    hotspots = platform.map_hotspots(payload, limit=5)
+
+    assert hotspots[0].source_hot_id == "q1"
+    assert hotspots[0].title == "热点"
+    assert hotspots[0].rank == 2
+    assert hotspots[0].heat_value == "888"
+
+
+def test_douyin_maps_hotspots_from_real_item_list_shape():
+    platform = DouyinPlatform(client=None)
+    payload = {
+        "data": {
+            "item_list": [
+                {"query_id": "q-real", "sentence": "真实热点", "rank": 1, "hot_score": 999}
+            ]
+        }
+    }
+
+    hotspots = platform.map_hotspots(payload, limit=5)
+
+    assert hotspots[0].source_hot_id == "q-real"
+    assert hotspots[0].title == "真实热点"
+    assert hotspots[0].rank == 1
+    assert hotspots[0].heat_value == "999"
+
+
+def test_douyin_maps_videos():
+    platform = DouyinPlatform(client=None)
+    payload = {
+        "data": [
+            {
+                "aweme_info": {
+                    "aweme_id": "aweme-1",
+                    "desc": "视频描述",
+                    "author": {"nickname": "作者"},
+                    "statistics": {"comment_count": 10},
+                }
+            }
+        ]
+    }
+
+    items = platform.map_items(payload, limit=5)
+
+    assert items[0].source_item_id == "aweme-1"
+    assert items[0].title == "视频描述"
+    assert items[0].item_type == "video"
+
+
+def test_douyin_maps_videos_from_real_business_data_shape():
+    platform = DouyinPlatform(client=None)
+    payload = {
+        "data": {
+            "business_data": [
+                {
+                    "data": {
+                        "aweme_info": {
+                            "aweme_id": "aweme-real",
+                            "desc": "真实视频描述",
+                            "share_url": "https://example.com/video",
+                        }
+                    }
+                }
+            ]
+        }
+    }
+
+    items = platform.map_items(payload, limit=5)
+
+    assert items[0].source_item_id == "aweme-real"
+    assert items[0].title == "真实视频描述"
+    assert items[0].url == "https://example.com/video"
+
+
+def test_douyin_maps_comment_id_and_missing_optional_fields():
+    platform = DouyinPlatform(client=None)
+    payload = {
+        "comments": [
+            {"comment_id": "preferred", "cid": "fallback", "text": "评论"},
+            {"cid": "fallback-only", "text": "评论2", "digg_count": 9},
+        ]
+    }
+
+    comments = platform.map_comments(payload, limit=10)
+
+    assert comments[0].source_comment_id == "preferred"
+    assert comments[0].content == "评论"
+    assert comments[0].like_count is None
+    assert comments[0].comment_time is None
+    assert comments[1].source_comment_id == "fallback-only"
+    assert comments[1].like_count == 9
+
+
+def test_douyin_maps_comments_when_user_is_none():
+    platform = DouyinPlatform(client=None)
+    payload = {"data": {"comments": [{"cid": "c-none-user", "text": "评论", "user": None}]}}
+
+    comments = platform.map_comments(payload, limit=10)
+
+    assert comments[0].source_comment_id == "c-none-user"
+    assert comments[0].author is None
+
+
+def test_douyin_maps_null_comments_as_empty_list():
+    platform = DouyinPlatform(client=None)
+    payload = {"data": {"comments": None}}
+
+    assert platform.map_comments(payload, limit=10) == []

tests/unit/test_export.py

@@ -0,0 +1,19 @@
+from app.services.export_service import labels_to_text, safe_csv_field, safe_filename
+
+
+def test_labels_to_text_joins_json_array_with_chinese_comma():
+    assert labels_to_text('["认可", "质量好"]') == "认可，质量好"
+    assert labels_to_text("bad-json") == ""
+
+
+def test_safe_csv_field_prevents_formula_injection_and_newlines():
+    assert safe_csv_field("=1+1") == "'=1+1"
+    assert safe_csv_field("hello\nworld") == "hello world"
+
+
+def test_safe_filename_replaces_illegal_chars_and_truncates_keyword():
+    filename = safe_filename("douyin", "task", "a/b:c*d?e<f>g|很长很长很长很长很长")
+
+    assert "/" not in filename
+    assert "__" not in filename
+    assert filename.endswith(".csv")

tests/unit/test_models.py

@@ -0,0 +1,88 @@
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app.db import Base, create_sqlite_engine
+from app.models import Comment, ContentItem, Hotspot, Report, Task, create_report_record
+
+
+def test_all_t03_tables_can_be_created_in_memory_sqlite():
+    engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
+    try:
+        Base.metadata.create_all(engine)
+
+        assert set(Base.metadata.tables) >= {
+            "tasks",
+            "hotspots",
+            "content_items",
+            "comments",
+            "reports",
+        }
+    finally:
+        engine.dispose()
+
+
+def test_sqlite_engine_uses_required_connection_options():
+    engine = create_sqlite_engine("sqlite:///data/app.db")
+    try:
+        assert engine.url.database == "data/app.db"
+        assert engine.dialect.connect_args["check_same_thread"] is False
+        assert engine.dialect.connect_args["timeout"] == 10
+
+    finally:
+        engine.dispose()
+
+
+def test_task_status_rejects_partial_status_values():
+    engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
+    try:
+        Base.metadata.create_all(engine)
+
+        with Session(engine) as session:
+            session.add(Task(platform="douyin", status="partial_success"))
+
+            with pytest.raises(IntegrityError):
+                session.commit()
+    finally:
+        engine.dispose()
+
+
+def test_analysis_status_insufficient_does_not_change_task_status():
+    task = Task(
+        platform="xiaohongshu",
+        status="success",
+        analysis_status="insufficient",
+        analysis_success_rate=0.5,
+    )
+
+    assert task.status == "success"
+    assert task.analysis_status == "insufficient"
+
+
+def test_models_define_relationships_and_indexes():
+    assert any(index.name == "ix_comments_task_content_item" for index in Comment.__table__.indexes)
+    assert any(index.name == "ix_content_items_task_hotspot" for index in ContentItem.__table__.indexes)
+    assert any(index.name == "ix_reports_task_report_type" for index in Report.__table__.indexes)
+
+    assert Hotspot.task.property.mapper.class_ is Task
+    assert ContentItem.hotspot.property.mapper.class_ is Hotspot
+    assert Comment.content_item.property.mapper.class_ is ContentItem
+    assert Report.task.property.mapper.class_ is Task
+
+
+def test_report_record_application_constraint_requires_matching_owner_id():
+    with pytest.raises(ValueError, match="hotspot_id"):
+        create_report_record(task_id="task-1", report_type="hotspot", title="热点报告")
+
+    with pytest.raises(ValueError, match="content_item_id"):
+        create_report_record(task_id="task-1", report_type="content_item", title="内容报告")
+
+    report = create_report_record(
+        task_id="task-1",
+        report_type="hotspot",
+        title="热点报告",
+        hotspot_id="hot-1",
+    )
+
+    assert report.hotspot_id == "hot-1"

tests/unit/test_report_stats.py

@@ -0,0 +1,76 @@
+from datetime import UTC, datetime
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from app.db import Base
+from app.models import Comment, ContentItem, Hotspot, Task
+from app.services.report_service import build_comment_metrics, generate_item_report
+
+
+def test_build_comment_metrics_counts_sentiments_and_top_labels():
+    comments = [
+        Comment(content="好", sentiment="positive", labels='["质量好", "价格好"]', like_count=5),
+        Comment(content="差", sentiment="negative", labels='["质量好"]', like_count=2),
+        Comment(content="一般", sentiment="neutral", labels="[]", like_count=1),
+        Comment(content="未知", sentiment="unknown", labels="[]", like_count=0),
+    ]
+
+    metrics = build_comment_metrics(comments)
+
+    assert metrics["sample_count"] == 4
+    assert metrics["sentiment"]["positive"]["count"] == 1
+    assert metrics["sentiment"]["negative"]["count"] == 1
+    assert metrics["sentiment"]["neutral"]["count"] == 1
+    assert metrics["sentiment"]["unknown"]["count"] == 1
+    assert metrics["top_labels"][0] == {"name": "质量好", "count": 2}
+
+
+def test_generate_item_report_persists_markdown_and_default_summary():
+    engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
+    Base.metadata.create_all(engine)
+    try:
+        with Session(engine) as session:
+            task = Task(platform="douyin", status="success")
+            session.add(task)
+            session.flush()
+            hotspot = Hotspot(task_id=task.id, platform="douyin", title="热点", raw_data="{}")
+            session.add(hotspot)
+            session.flush()
+            item = ContentItem(
+                task_id=task.id,
+                hotspot_id=hotspot.id,
+                platform="douyin",
+                source_item_id="v1",
+                item_type="video",
+                title="视频",
+                status="success",
+                raw_data="{}",
+            )
+            session.add(item)
+            session.flush()
+            session.add(
+                Comment(
+                    task_id=task.id,
+                    hotspot_id=hotspot.id,
+                    content_item_id=item.id,
+                    platform="douyin",
+                    source_comment_id="c1",
+                    content="好评",
+                    sentiment="positive",
+                    labels='["认可"]',
+                    like_count=10,
+                    comment_time=datetime(2026, 1, 1, tzinfo=UTC),
+                    raw_data="{}",
+                )
+            )
+            session.commit()
+
+            report = generate_item_report(session, item.id, summary_provider=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("ai failed")))
+
+            assert report.report_type == "item"
+            assert report.content_item_id == item.id
+            assert "总结生成失败，请查看上方统计数据。" in report.markdown_content
+            assert "样本评论数量" in report.markdown_content
+    finally:
+        engine.dispose()

tests/unit/test_task_executor.py

@@ -0,0 +1,12 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from app.services import task_service
+
+
+def test_task_executor_is_single_worker_thread_pool():
+    assert isinstance(task_service.task_executor, ThreadPoolExecutor)
+    assert task_service.task_executor._max_workers == 1
+
+
+def test_run_task_entrypoint_is_synchronous_function():
+    assert task_service.inspect.iscoroutinefunction(task_service.run_task) is False

tests/unit/test_template_filters.py

@@ -0,0 +1,13 @@
+from app.templating import from_json_filter, status_badge_config
+
+
+def test_status_badge_config_centralizes_task_status_copy():
+    assert status_badge_config("running") == ("运行中", "bg-warning text-dark")
+    assert status_badge_config("success") == ("已完成", "bg-success")
+    assert status_badge_config("failed") == ("失败", "bg-danger")
+    assert status_badge_config("unknown") == ("未知", "bg-secondary")
+
+
+def test_from_json_filter_returns_empty_list_for_invalid_json():
+    assert from_json_filter('["认可"]') == ["认可"]
+    assert from_json_filter("bad-json") == []

tests/unit/test_xiaohongshu_mapping.py

@@ -0,0 +1,65 @@
+from app.platforms.xiaohongshu import XiaohongshuPlatform
+
+
+def test_xiaohongshu_maps_hotspots_from_items_not_outer_title():
+    platform = XiaohongshuPlatform(client=None)
+    payload = {
+        "data": {
+            "data": {
+                "title": "搜索发现",
+                "items": [
+                    {"id": "hot-1", "title": "真实热点", "score": "999", "rank_change": 1}
+                ],
+            }
+        }
+    }
+
+    hotspots = platform.map_hotspots(payload, limit=5)
+
+    assert hotspots[0].source_hot_id == "hot-1"
+    assert hotspots[0].title == "真实热点"
+    assert hotspots[0].heat_value == "999"
+
+
+def test_xiaohongshu_prefers_notes_with_comments_and_falls_back_to_zero_comment_notes():
+    platform = XiaohongshuPlatform(client=None)
+    payload = {
+        "data": {
+            "data": {
+                "items": [
+                    {"note": {"id": "n0", "title": "无评论", "desc": "", "comments_count": 0}},
+                    {"note": {"id": "n1", "title": "有评论1", "desc": "d1", "comments_count": 3}},
+                    {"note": {"id": "n2", "title": "有评论2", "desc": "d2", "comments_count": 1}},
+                ]
+            }
+        }
+    }
+
+    items = platform.map_items(payload, limit=3)
+
+    assert [item.source_item_id for item in items] == ["n1", "n2", "n0"]
+    assert all(item.item_type == "note" for item in items)
+
+
+def test_xiaohongshu_maps_comment_id_content_and_missing_optional_fields():
+    platform = XiaohongshuPlatform(client=None)
+    payload = {
+        "data": {
+            "data": {
+                "comments": [
+                    {"comment_id": "preferred", "id": "fallback", "content": "正文"},
+                    {"id": "fallback-only", "text": "文本字段", "like_count": 7},
+                ]
+            }
+        }
+    }
+
+    comments = platform.map_comments(payload, limit=10)
+
+    assert comments[0].source_comment_id == "preferred"
+    assert comments[0].content == "正文"
+    assert comments[0].like_count is None
+    assert comments[0].comment_time is None
+    assert comments[1].source_comment_id == "fallback-only"
+    assert comments[1].content == "文本字段"
+    assert comments[1].like_count == 7