fix: 修复报告摘要生成并稳定默认端口

This commit is contained in:
meijiali
2026-07-03 12:20:59 +08:00
parent 5c34ce75b6
commit 0a3477cc44
10 changed files with 230 additions and 20 deletions
+14 -2
View File
@@ -88,11 +88,23 @@ class DouyinPlatform:
return self.map_items(payload, limit=limit)
def fetch_comments(self, source_item_id: str, *, limit: int) -> list[CommentData]:
comments: list[CommentData] = []
cursor = 0
page_size = min(20, limit)
while len(comments) < limit:
payload = self.client.get(
"/api/v1/douyin/app/v3/fetch_video_comments",
params={"aweme_id": source_item_id, "cursor": 0, "count": 20},
params={"aweme_id": source_item_id, "cursor": cursor, "count": page_size},
)
return self.map_comments(payload, limit=limit)
page_comments = self.map_comments(payload, limit=limit - len(comments))
comments.extend(page_comments)
data = payload.get("data", {})
next_cursor = data.get("cursor") or payload.get("cursor")
has_more = data.get("has_more", payload.get("has_more", 0))
if not page_comments or not has_more or next_cursor in (None, cursor):
break
cursor = next_cursor
return comments[:limit]
def _author_name(self, comment: dict[str, Any]) -> str | None:
user = comment.get("user") or {}
+42 -2
View File
@@ -42,6 +42,46 @@ def build_comment_prompt(comments: list[dict[str, str]]) -> str:
)
def build_report_summary_prompt(metrics: dict, typical: dict, *, word_limit: int) -> str:
sentiment = metrics.get("sentiment", {})
sentiment_lines = []
for name in ("positive", "neutral", "negative", "unknown"):
entry = sentiment.get(name, {})
sentiment_lines.append(f"- {name}: {entry.get('count', 0)} ({entry.get('pct', 0)}%)")
label_lines = [
f"- {label.get('name', '')}: {label.get('count', 0)}"
for label in metrics.get("top_labels", [])
if label.get("name")
]
if not label_lines:
label_lines = ["- 暂无"]
typical_lines = []
for sentiment_name, comments in typical.items():
for comment in comments:
typical_lines.append(f"- [{sentiment_name}] {str(comment.get('content', ''))[:150]}")
if not typical_lines:
typical_lines = ["- 暂无"]
item_count_line = ""
if "item_count" in metrics:
item_count_line = f"内容条目数量:{metrics.get('item_count', 0)}\n"
return (
f"只返回一段中文总结,不使用 Markdown,不超过 {word_limit} 字。\n"
"总结必须基于以下统计数据和典型评论,不要编造未提供的信息。\n"
f"{item_count_line}"
f"样本评论数量:{metrics.get('sample_count', 0)}\n"
"情绪分布:\n"
f"{chr(10).join(sentiment_lines)}\n"
"Top 标签:\n"
f"{chr(10).join(label_lines)}\n"
"典型评论:\n"
f"{chr(10).join(typical_lines)}"
)
def parse_ai_comment_response(response_text: str, *, expected_comment_ids: set[str]) -> list[AIAnalysisResult]:
try:
raw = json.loads(response_text)
@@ -118,7 +158,7 @@ class OpenAICompatibleAIClient:
self.model = model
self._http_client = http_client or httpx.Client(timeout=timeout_seconds)
def request(self, prompt: str) -> str:
def request(self, prompt: str, *, system_prompt: str = "请严格遵循用户指令输出。") -> str:
endpoint = f"{self.base_url}/chat/completions" if self.base_url.endswith("/v1") else f"{self.base_url}/v1/chat/completions"
response = self._http_client.post(
endpoint,
@@ -128,7 +168,7 @@ class OpenAICompatibleAIClient:
"messages": [
{
"role": "system",
"content": "只返回 JSON Array,不输出 Markdown 或解释性自然语言。",
"content": system_prompt,
},
{"role": "user", "content": prompt},
],
+5
View File
@@ -127,10 +127,15 @@ def generate_hotspot_report(
def _safe_summary(summary_provider, metrics: dict, typical: dict, *, limit: int) -> str:
if summary_provider is None:
return DEFAULT_SUMMARY
try:
summary = summary_provider(metrics, typical, word_limit=limit)
except TypeError:
try:
summary = summary_provider(metrics, typical)
except Exception:
return DEFAULT_SUMMARY
except Exception:
return DEFAULT_SUMMARY
return str(summary)[:limit]
+37 -7
View File
@@ -13,7 +13,7 @@ from app.platforms.base import PlatformAPIError, TikHubClient
from app.platforms.douyin import DouyinPlatform
from app.platforms.xiaohongshu import XiaohongshuPlatform
from app.schemas import CreateTaskRequest
from app.services.ai_service import OpenAICompatibleAIClient, analyze_comments_with_retry, calculate_analysis_status
from app.services.ai_service import OpenAICompatibleAIClient, analyze_comments_with_retry, build_report_summary_prompt, calculate_analysis_status
from app.services.report_service import generate_hotspot_report, generate_item_report
@@ -52,16 +52,25 @@ def build_platform(platform: str):
raise ValueError(f"Unsupported platform: {platform}")
def build_ai_requester():
def build_ai_client() -> OpenAICompatibleAIClient | None:
settings = get_settings()
if settings.ai_base_url and settings.ai_api_key and settings.ai_model:
client = OpenAICompatibleAIClient(
return OpenAICompatibleAIClient(
base_url=settings.ai_base_url,
api_key=settings.ai_api_key,
model=settings.ai_model,
timeout_seconds=settings.ai_timeout_seconds,
)
return client.request
return None
def build_ai_requester(client: OpenAICompatibleAIClient | None = None):
client = client if client is not None else build_ai_client()
if client is not None:
return lambda prompt: client.request(
prompt,
system_prompt="只返回 JSON Array,不输出 Markdown 或解释性自然语言。",
)
def fallback_requester(_prompt: str) -> str:
return "[]"
@@ -69,6 +78,25 @@ def build_ai_requester():
return fallback_requester
def build_report_summary_provider(client: OpenAICompatibleAIClient | None):
if client is None:
return None
def summarize_report(metrics: dict, typical: dict, *, word_limit: int = 200) -> str:
return client.request(
build_report_summary_prompt(metrics, typical, word_limit=word_limit),
system_prompt="只返回一段中文总结,不使用 Markdown。",
)
return summarize_report
def build_ai_dependencies():
client = build_ai_client()
requester = build_ai_requester(client)
return requester, build_report_summary_provider(client)
def create_task(session: Session, request: CreateTaskRequest, *, submit_background: bool = True) -> Task:
task = Task(
platform=request.platform,
@@ -108,6 +136,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
return
try:
platform = build_platform(task.platform)
ai_requester, report_summary_provider = build_ai_dependencies()
try:
hotspots = platform.fetch_hotspots(limit=task.hotspot_limit)
except PlatformAPIError as exc:
@@ -183,7 +213,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
ai_results = analyze_comments_with_retry(
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
requester=build_ai_requester(),
requester=ai_requester,
max_retries=get_settings().ai_max_retries,
)
results_by_comment_id = {result.comment_id: result for result in ai_results}
@@ -201,7 +231,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
comment.ai_analysis_status = result.ai_analysis_status
item.status = "success"
task.successful_items_count += 1
generate_item_report(session, item.id)
generate_item_report(session, item.id, summary_provider=report_summary_provider)
except Exception as exc:
item.status = "failed"
item.error_stage = "crawl_comments"
@@ -223,7 +253,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
total_count=total_comments,
)
for hotspot in task.hotspots:
generate_hotspot_report(session, hotspot.id)
generate_hotspot_report(session, hotspot.id, summary_provider=report_summary_provider)
task.status = "success"
else:
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
+2
View File
@@ -1,3 +1,5 @@
name: hot-comments-tool
services:
app:
build: .
@@ -5,6 +5,7 @@ from sqlalchemy.orm import Session
from app.models import Comment, ContentItem, Hotspot, Report, Task
from app.schemas import CreateTaskRequest
from app.services.report_service import DEFAULT_SUMMARY
from app.services.task_service import create_task, run_task
from tests.helpers import make_test_client
@@ -28,7 +29,11 @@ class FakeXhsPlatform:
return [CommentData(source_comment_id="c1", content="评论一", like_count=5, raw_data={"id": "c1"})]
ai_prompts: list[str] = []
def successful_ai_response(prompt: str) -> str:
ai_prompts.append(prompt)
comments = json.loads(prompt[prompt.index("[") :])
comment_id = comments[0]["comment_id"]
return json.dumps(
@@ -38,9 +43,14 @@ def successful_ai_response(prompt: str) -> str:
def test_xiaohongshu_task_flow_persists_hotspots_items_and_comments(monkeypatch):
ai_prompts.clear()
with make_test_client() as (_client, engine):
monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform())
monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: successful_ai_response)
summary_provider = lambda metrics, typical, *, word_limit=200: (
ai_prompts.append(f"只返回一段中文总结 word_limit={word_limit} sample={metrics['sample_count']}")
or "这是一段真实 AI 报告摘要。"
)
monkeypatch.setattr("app.services.task_service.build_ai_dependencies", lambda: (successful_ai_response, summary_provider))
with Session(engine) as session:
task = create_task(
@@ -66,14 +76,21 @@ def test_xiaohongshu_task_flow_persists_hotspots_items_and_comments(monkeypatch)
assert comment.sentiment == "positive"
assert comment.labels == '["认可"]'
assert comment.reason == "喜欢"
assert session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "item")) is not None
assert session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "hotspot")) is not None
item_report = session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "item"))
hotspot_report = session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "hotspot"))
assert item_report is not None
assert hotspot_report is not None
assert item_report.summary == "这是一段真实 AI 报告摘要。"
assert hotspot_report.summary == "这是一段真实 AI 报告摘要。"
assert DEFAULT_SUMMARY not in item_report.markdown_content
assert DEFAULT_SUMMARY not in hotspot_report.markdown_content
assert any("只返回一段中文总结" in prompt for prompt in ai_prompts)
def test_xiaohongshu_task_flow_marks_comments_failed_when_ai_parse_fails(monkeypatch):
with make_test_client() as (_client, engine):
monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform())
monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: (lambda _prompt: "not-json"))
monkeypatch.setattr("app.services.task_service.build_ai_dependencies", lambda: (lambda _prompt: "not-json", None))
monkeypatch.setattr("app.services.ai_service.time.sleep", lambda _seconds: None)
with Session(engine) as session:
@@ -107,7 +124,7 @@ def test_xiaohongshu_task_flow_keeps_item_success_when_ai_request_raises(monkeyp
def failing_requester(_prompt):
raise RuntimeError("ai unauthorized")
monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: failing_requester)
monkeypatch.setattr("app.services.task_service.build_ai_dependencies", lambda: (failing_requester, None))
with Session(engine) as session:
task = create_task(
+52
View File
@@ -5,10 +5,12 @@ from app.services.ai_service import (
AIAnalysisResult,
OpenAICompatibleAIClient,
analyze_comments_with_retry,
build_report_summary_prompt,
build_comment_prompt,
calculate_analysis_status,
parse_ai_comment_response,
)
from app.services.task_service import build_report_summary_provider
def test_build_comment_prompt_contains_ids_and_truncates_content():
@@ -20,6 +22,35 @@ def test_build_comment_prompt_contains_ids_and_truncates_content():
assert "" * 151 not in prompt
def test_build_report_summary_prompt_contains_stats_labels_and_truncated_typical_comments():
prompt = build_report_summary_prompt(
{
"sample_count": 2,
"sentiment": {
"positive": {"count": 1, "pct": 50.0},
"neutral": {"count": 0, "pct": 0.0},
"negative": {"count": 1, "pct": 50.0},
"unknown": {"count": 0, "pct": 0.0},
},
"top_labels": [{"name": "价格争议", "count": 2}],
},
{"positive": [{"content": "" * 200}], "negative": [{"content": "太贵"}]},
word_limit=200,
)
assert "只返回一段中文总结" in prompt
assert "样本评论数量:2" in prompt
assert "positive: 1 (50.0%)" in prompt
assert "价格争议: 2" in prompt
assert "" * 150 in prompt
assert "" * 151 not in prompt
assert "JSON Array" not in prompt
def test_report_summary_provider_is_disabled_without_real_ai_client():
assert build_report_summary_provider(client=None) is None
def test_parse_ai_comment_response_validates_array_sentiment_labels_and_ids():
result = parse_ai_comment_response(
'[{"comment_id":"c1","sentiment":"positive","labels":["质量好"],"reason":"认可"}]',
@@ -94,6 +125,27 @@ def test_openai_compatible_client_posts_chat_completion_and_returns_message_cont
assert captured["authorization"] == "Bearer test-ai-key"
assert '"model":"test-model"' in captured["body"]
assert "prompt text" in captured["body"]
assert "请严格遵循用户指令输出。" in captured["body"]
def test_openai_compatible_client_accepts_custom_system_prompt():
captured = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["body"] = request.read().decode("utf-8")
return httpx.Response(200, json={"choices": [{"message": {"content": "报告摘要"}}]})
http_client = httpx.Client(transport=httpx.MockTransport(handler))
client = OpenAICompatibleAIClient(
base_url="https://ai.example.com",
api_key="test-ai-key",
model="test-model",
http_client=http_client,
)
assert client.request("总结 prompt", system_prompt="只返回一段中文总结,不使用 Markdown。") == "报告摘要"
assert "只返回一段中文总结" in captured["body"]
assert "JSON Array" not in captured["body"]
def test_openai_compatible_client_accepts_base_url_that_already_includes_v1():
+6
View File
@@ -11,6 +11,12 @@ def test_docker_compose_reads_real_env_file_not_example():
assert "- .env.example" not in compose_content
def test_docker_compose_uses_fixed_project_name_for_stable_port_owner():
compose_content = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "name: hot-comments-tool" in compose_content
def test_real_env_file_is_gitignored():
gitignore_lines = (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
+36
View File
@@ -118,3 +118,39 @@ def test_douyin_maps_null_comments_as_empty_list():
payload = {"data": {"comments": None}}
assert platform.map_comments(payload, limit=10) == []
class RecordingDouyinClient:
def __init__(self):
self.requests = []
def get(self, path, *, params=None):
self.requests.append((path, params))
cursor = params["cursor"]
if cursor == 0:
return {
"data": {
"comments": [{"cid": f"c-{index}", "text": f"评论 {index}"} for index in range(20)],
"cursor": 20,
"has_more": 1,
}
}
return {
"data": {
"comments": [{"cid": f"c-{index}", "text": f"评论 {index}"} for index in range(20, 55)],
"cursor": 55,
"has_more": 0,
}
}
def test_douyin_fetch_comments_paginates_until_limit():
client = RecordingDouyinClient()
platform = DouyinPlatform(client=client)
comments = platform.fetch_comments("aweme-1", limit=50)
assert len(comments) == 50
assert comments[0].source_comment_id == "c-0"
assert comments[-1].source_comment_id == "c-49"
assert [request[1]["cursor"] for request in client.requests] == [0, 20]
+10
View File
@@ -1,6 +1,7 @@
from concurrent.futures import ThreadPoolExecutor
from app.services import task_service
from app.config import Settings
def test_task_executor_is_single_worker_thread_pool():
@@ -10,3 +11,12 @@ def test_task_executor_is_single_worker_thread_pool():
def test_run_task_entrypoint_is_synchronous_function():
assert task_service.inspect.iscoroutinefunction(task_service.run_task) is False
def test_ai_dependencies_do_not_generate_report_summary_provider_without_ai_config(monkeypatch):
monkeypatch.setattr("app.services.task_service.get_settings", lambda: Settings(_env_file=None))
requester, summary_provider = task_service.build_ai_dependencies()
assert requester("prompt") == "[]"
assert summary_provider is None