fix: 修复报告摘要生成并稳定默认端口

This commit is contained in:
meijiali
2026-07-03 12:20:59 +08:00
parent 5c34ce75b6
commit 0a3477cc44
10 changed files with 230 additions and 20 deletions
+17 -5
View File
@@ -88,11 +88,23 @@ class DouyinPlatform:
return self.map_items(payload, limit=limit) return self.map_items(payload, limit=limit)
def fetch_comments(self, source_item_id: str, *, limit: int) -> list[CommentData]: def fetch_comments(self, source_item_id: str, *, limit: int) -> list[CommentData]:
payload = self.client.get( comments: list[CommentData] = []
"/api/v1/douyin/app/v3/fetch_video_comments", cursor = 0
params={"aweme_id": source_item_id, "cursor": 0, "count": 20}, page_size = min(20, limit)
) while len(comments) < limit:
return self.map_comments(payload, limit=limit) payload = self.client.get(
"/api/v1/douyin/app/v3/fetch_video_comments",
params={"aweme_id": source_item_id, "cursor": cursor, "count": page_size},
)
page_comments = self.map_comments(payload, limit=limit - len(comments))
comments.extend(page_comments)
data = payload.get("data", {})
next_cursor = data.get("cursor") or payload.get("cursor")
has_more = data.get("has_more", payload.get("has_more", 0))
if not page_comments or not has_more or next_cursor in (None, cursor):
break
cursor = next_cursor
return comments[:limit]
def _author_name(self, comment: dict[str, Any]) -> str | None: def _author_name(self, comment: dict[str, Any]) -> str | None:
user = comment.get("user") or {} user = comment.get("user") or {}
+42 -2
View File
@@ -42,6 +42,46 @@ def build_comment_prompt(comments: list[dict[str, str]]) -> str:
) )
def build_report_summary_prompt(metrics: dict, typical: dict, *, word_limit: int) -> str:
sentiment = metrics.get("sentiment", {})
sentiment_lines = []
for name in ("positive", "neutral", "negative", "unknown"):
entry = sentiment.get(name, {})
sentiment_lines.append(f"- {name}: {entry.get('count', 0)} ({entry.get('pct', 0)}%)")
label_lines = [
f"- {label.get('name', '')}: {label.get('count', 0)}"
for label in metrics.get("top_labels", [])
if label.get("name")
]
if not label_lines:
label_lines = ["- 暂无"]
typical_lines = []
for sentiment_name, comments in typical.items():
for comment in comments:
typical_lines.append(f"- [{sentiment_name}] {str(comment.get('content', ''))[:150]}")
if not typical_lines:
typical_lines = ["- 暂无"]
item_count_line = ""
if "item_count" in metrics:
item_count_line = f"内容条目数量:{metrics.get('item_count', 0)}\n"
return (
f"只返回一段中文总结,不使用 Markdown,不超过 {word_limit} 字。\n"
"总结必须基于以下统计数据和典型评论,不要编造未提供的信息。\n"
f"{item_count_line}"
f"样本评论数量:{metrics.get('sample_count', 0)}\n"
"情绪分布:\n"
f"{chr(10).join(sentiment_lines)}\n"
"Top 标签:\n"
f"{chr(10).join(label_lines)}\n"
"典型评论:\n"
f"{chr(10).join(typical_lines)}"
)
def parse_ai_comment_response(response_text: str, *, expected_comment_ids: set[str]) -> list[AIAnalysisResult]: def parse_ai_comment_response(response_text: str, *, expected_comment_ids: set[str]) -> list[AIAnalysisResult]:
try: try:
raw = json.loads(response_text) raw = json.loads(response_text)
@@ -118,7 +158,7 @@ class OpenAICompatibleAIClient:
self.model = model self.model = model
self._http_client = http_client or httpx.Client(timeout=timeout_seconds) self._http_client = http_client or httpx.Client(timeout=timeout_seconds)
def request(self, prompt: str) -> str: def request(self, prompt: str, *, system_prompt: str = "请严格遵循用户指令输出。") -> str:
endpoint = f"{self.base_url}/chat/completions" if self.base_url.endswith("/v1") else f"{self.base_url}/v1/chat/completions" endpoint = f"{self.base_url}/chat/completions" if self.base_url.endswith("/v1") else f"{self.base_url}/v1/chat/completions"
response = self._http_client.post( response = self._http_client.post(
endpoint, endpoint,
@@ -128,7 +168,7 @@ class OpenAICompatibleAIClient:
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
"content": "只返回 JSON Array,不输出 Markdown 或解释性自然语言。", "content": system_prompt,
}, },
{"role": "user", "content": prompt}, {"role": "user", "content": prompt},
], ],
+6 -1
View File
@@ -128,7 +128,12 @@ def _safe_summary(summary_provider, metrics: dict, typical: dict, *, limit: int)
if summary_provider is None: if summary_provider is None:
return DEFAULT_SUMMARY return DEFAULT_SUMMARY
try: try:
summary = summary_provider(metrics, typical) summary = summary_provider(metrics, typical, word_limit=limit)
except TypeError:
try:
summary = summary_provider(metrics, typical)
except Exception:
return DEFAULT_SUMMARY
except Exception: except Exception:
return DEFAULT_SUMMARY return DEFAULT_SUMMARY
return str(summary)[:limit] return str(summary)[:limit]
+37 -7
View File
@@ -13,7 +13,7 @@ from app.platforms.base import PlatformAPIError, TikHubClient
from app.platforms.douyin import DouyinPlatform from app.platforms.douyin import DouyinPlatform
from app.platforms.xiaohongshu import XiaohongshuPlatform from app.platforms.xiaohongshu import XiaohongshuPlatform
from app.schemas import CreateTaskRequest from app.schemas import CreateTaskRequest
from app.services.ai_service import OpenAICompatibleAIClient, analyze_comments_with_retry, calculate_analysis_status from app.services.ai_service import OpenAICompatibleAIClient, analyze_comments_with_retry, build_report_summary_prompt, calculate_analysis_status
from app.services.report_service import generate_hotspot_report, generate_item_report from app.services.report_service import generate_hotspot_report, generate_item_report
@@ -52,16 +52,25 @@ def build_platform(platform: str):
raise ValueError(f"Unsupported platform: {platform}") raise ValueError(f"Unsupported platform: {platform}")
def build_ai_requester(): def build_ai_client() -> OpenAICompatibleAIClient | None:
settings = get_settings() settings = get_settings()
if settings.ai_base_url and settings.ai_api_key and settings.ai_model: if settings.ai_base_url and settings.ai_api_key and settings.ai_model:
client = OpenAICompatibleAIClient( return OpenAICompatibleAIClient(
base_url=settings.ai_base_url, base_url=settings.ai_base_url,
api_key=settings.ai_api_key, api_key=settings.ai_api_key,
model=settings.ai_model, model=settings.ai_model,
timeout_seconds=settings.ai_timeout_seconds, timeout_seconds=settings.ai_timeout_seconds,
) )
return client.request return None
def build_ai_requester(client: OpenAICompatibleAIClient | None = None):
client = client if client is not None else build_ai_client()
if client is not None:
return lambda prompt: client.request(
prompt,
system_prompt="只返回 JSON Array,不输出 Markdown 或解释性自然语言。",
)
def fallback_requester(_prompt: str) -> str: def fallback_requester(_prompt: str) -> str:
return "[]" return "[]"
@@ -69,6 +78,25 @@ def build_ai_requester():
return fallback_requester return fallback_requester
def build_report_summary_provider(client: OpenAICompatibleAIClient | None):
if client is None:
return None
def summarize_report(metrics: dict, typical: dict, *, word_limit: int = 200) -> str:
return client.request(
build_report_summary_prompt(metrics, typical, word_limit=word_limit),
system_prompt="只返回一段中文总结,不使用 Markdown。",
)
return summarize_report
def build_ai_dependencies():
client = build_ai_client()
requester = build_ai_requester(client)
return requester, build_report_summary_provider(client)
def create_task(session: Session, request: CreateTaskRequest, *, submit_background: bool = True) -> Task: def create_task(session: Session, request: CreateTaskRequest, *, submit_background: bool = True) -> Task:
task = Task( task = Task(
platform=request.platform, platform=request.platform,
@@ -108,6 +136,8 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
return return
try: try:
platform = build_platform(task.platform) platform = build_platform(task.platform)
ai_requester, report_summary_provider = build_ai_dependencies()
try: try:
hotspots = platform.fetch_hotspots(limit=task.hotspot_limit) hotspots = platform.fetch_hotspots(limit=task.hotspot_limit)
except PlatformAPIError as exc: except PlatformAPIError as exc:
@@ -183,7 +213,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id))) item_comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
ai_results = analyze_comments_with_retry( ai_results = analyze_comments_with_retry(
[{"comment_id": comment.id, "content": comment.content} for comment in item_comments], [{"comment_id": comment.id, "content": comment.content} for comment in item_comments],
requester=build_ai_requester(), requester=ai_requester,
max_retries=get_settings().ai_max_retries, max_retries=get_settings().ai_max_retries,
) )
results_by_comment_id = {result.comment_id: result for result in ai_results} results_by_comment_id = {result.comment_id: result for result in ai_results}
@@ -201,7 +231,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
comment.ai_analysis_status = result.ai_analysis_status comment.ai_analysis_status = result.ai_analysis_status
item.status = "success" item.status = "success"
task.successful_items_count += 1 task.successful_items_count += 1
generate_item_report(session, item.id) generate_item_report(session, item.id, summary_provider=report_summary_provider)
except Exception as exc: except Exception as exc:
item.status = "failed" item.status = "failed"
item.error_stage = "crawl_comments" item.error_stage = "crawl_comments"
@@ -223,7 +253,7 @@ def run_task(task_id: str, *, session_factory: Callable[[], Session] | sessionma
total_count=total_comments, total_count=total_comments,
) )
for hotspot in task.hotspots: for hotspot in task.hotspots:
generate_hotspot_report(session, hotspot.id) generate_hotspot_report(session, hotspot.id, summary_provider=report_summary_provider)
task.status = "success" task.status = "success"
else: else:
_mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功") _mark_task_failed(task, task.error_stage or "crawl_items", task.error_type or "no_successful_items", task.error_message or "没有任何内容条目成功")
+2
View File
@@ -1,3 +1,5 @@
name: hot-comments-tool
services: services:
app: app:
build: . build: .
@@ -5,6 +5,7 @@ from sqlalchemy.orm import Session
from app.models import Comment, ContentItem, Hotspot, Report, Task from app.models import Comment, ContentItem, Hotspot, Report, Task
from app.schemas import CreateTaskRequest from app.schemas import CreateTaskRequest
from app.services.report_service import DEFAULT_SUMMARY
from app.services.task_service import create_task, run_task from app.services.task_service import create_task, run_task
from tests.helpers import make_test_client from tests.helpers import make_test_client
@@ -28,7 +29,11 @@ class FakeXhsPlatform:
return [CommentData(source_comment_id="c1", content="评论一", like_count=5, raw_data={"id": "c1"})] return [CommentData(source_comment_id="c1", content="评论一", like_count=5, raw_data={"id": "c1"})]
ai_prompts: list[str] = []
def successful_ai_response(prompt: str) -> str: def successful_ai_response(prompt: str) -> str:
ai_prompts.append(prompt)
comments = json.loads(prompt[prompt.index("[") :]) comments = json.loads(prompt[prompt.index("[") :])
comment_id = comments[0]["comment_id"] comment_id = comments[0]["comment_id"]
return json.dumps( return json.dumps(
@@ -38,9 +43,14 @@ def successful_ai_response(prompt: str) -> str:
def test_xiaohongshu_task_flow_persists_hotspots_items_and_comments(monkeypatch): def test_xiaohongshu_task_flow_persists_hotspots_items_and_comments(monkeypatch):
ai_prompts.clear()
with make_test_client() as (_client, engine): with make_test_client() as (_client, engine):
monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform()) monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform())
monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: successful_ai_response) summary_provider = lambda metrics, typical, *, word_limit=200: (
ai_prompts.append(f"只返回一段中文总结 word_limit={word_limit} sample={metrics['sample_count']}")
or "这是一段真实 AI 报告摘要。"
)
monkeypatch.setattr("app.services.task_service.build_ai_dependencies", lambda: (successful_ai_response, summary_provider))
with Session(engine) as session: with Session(engine) as session:
task = create_task( task = create_task(
@@ -66,14 +76,21 @@ def test_xiaohongshu_task_flow_persists_hotspots_items_and_comments(monkeypatch)
assert comment.sentiment == "positive" assert comment.sentiment == "positive"
assert comment.labels == '["认可"]' assert comment.labels == '["认可"]'
assert comment.reason == "喜欢" assert comment.reason == "喜欢"
assert session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "item")) is not None item_report = session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "item"))
assert session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "hotspot")) is not None hotspot_report = session.scalar(select(Report).where(Report.task_id == task_id, Report.report_type == "hotspot"))
assert item_report is not None
assert hotspot_report is not None
assert item_report.summary == "这是一段真实 AI 报告摘要。"
assert hotspot_report.summary == "这是一段真实 AI 报告摘要。"
assert DEFAULT_SUMMARY not in item_report.markdown_content
assert DEFAULT_SUMMARY not in hotspot_report.markdown_content
assert any("只返回一段中文总结" in prompt for prompt in ai_prompts)
def test_xiaohongshu_task_flow_marks_comments_failed_when_ai_parse_fails(monkeypatch): def test_xiaohongshu_task_flow_marks_comments_failed_when_ai_parse_fails(monkeypatch):
with make_test_client() as (_client, engine): with make_test_client() as (_client, engine):
monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform()) monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FakeXhsPlatform())
monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: (lambda _prompt: "not-json")) monkeypatch.setattr("app.services.task_service.build_ai_dependencies", lambda: (lambda _prompt: "not-json", None))
monkeypatch.setattr("app.services.ai_service.time.sleep", lambda _seconds: None) monkeypatch.setattr("app.services.ai_service.time.sleep", lambda _seconds: None)
with Session(engine) as session: with Session(engine) as session:
@@ -107,7 +124,7 @@ def test_xiaohongshu_task_flow_keeps_item_success_when_ai_request_raises(monkeyp
def failing_requester(_prompt): def failing_requester(_prompt):
raise RuntimeError("ai unauthorized") raise RuntimeError("ai unauthorized")
monkeypatch.setattr("app.services.task_service.build_ai_requester", lambda: failing_requester) monkeypatch.setattr("app.services.task_service.build_ai_dependencies", lambda: (failing_requester, None))
with Session(engine) as session: with Session(engine) as session:
task = create_task( task = create_task(
+52
View File
@@ -5,10 +5,12 @@ from app.services.ai_service import (
AIAnalysisResult, AIAnalysisResult,
OpenAICompatibleAIClient, OpenAICompatibleAIClient,
analyze_comments_with_retry, analyze_comments_with_retry,
build_report_summary_prompt,
build_comment_prompt, build_comment_prompt,
calculate_analysis_status, calculate_analysis_status,
parse_ai_comment_response, parse_ai_comment_response,
) )
from app.services.task_service import build_report_summary_provider
def test_build_comment_prompt_contains_ids_and_truncates_content(): def test_build_comment_prompt_contains_ids_and_truncates_content():
@@ -20,6 +22,35 @@ def test_build_comment_prompt_contains_ids_and_truncates_content():
assert "" * 151 not in prompt assert "" * 151 not in prompt
def test_build_report_summary_prompt_contains_stats_labels_and_truncated_typical_comments():
prompt = build_report_summary_prompt(
{
"sample_count": 2,
"sentiment": {
"positive": {"count": 1, "pct": 50.0},
"neutral": {"count": 0, "pct": 0.0},
"negative": {"count": 1, "pct": 50.0},
"unknown": {"count": 0, "pct": 0.0},
},
"top_labels": [{"name": "价格争议", "count": 2}],
},
{"positive": [{"content": "" * 200}], "negative": [{"content": "太贵"}]},
word_limit=200,
)
assert "只返回一段中文总结" in prompt
assert "样本评论数量:2" in prompt
assert "positive: 1 (50.0%)" in prompt
assert "价格争议: 2" in prompt
assert "" * 150 in prompt
assert "" * 151 not in prompt
assert "JSON Array" not in prompt
def test_report_summary_provider_is_disabled_without_real_ai_client():
assert build_report_summary_provider(client=None) is None
def test_parse_ai_comment_response_validates_array_sentiment_labels_and_ids(): def test_parse_ai_comment_response_validates_array_sentiment_labels_and_ids():
result = parse_ai_comment_response( result = parse_ai_comment_response(
'[{"comment_id":"c1","sentiment":"positive","labels":["质量好"],"reason":"认可"}]', '[{"comment_id":"c1","sentiment":"positive","labels":["质量好"],"reason":"认可"}]',
@@ -94,6 +125,27 @@ def test_openai_compatible_client_posts_chat_completion_and_returns_message_cont
assert captured["authorization"] == "Bearer test-ai-key" assert captured["authorization"] == "Bearer test-ai-key"
assert '"model":"test-model"' in captured["body"] assert '"model":"test-model"' in captured["body"]
assert "prompt text" in captured["body"] assert "prompt text" in captured["body"]
assert "请严格遵循用户指令输出。" in captured["body"]
def test_openai_compatible_client_accepts_custom_system_prompt():
captured = {}
def handler(request: httpx.Request) -> httpx.Response:
captured["body"] = request.read().decode("utf-8")
return httpx.Response(200, json={"choices": [{"message": {"content": "报告摘要"}}]})
http_client = httpx.Client(transport=httpx.MockTransport(handler))
client = OpenAICompatibleAIClient(
base_url="https://ai.example.com",
api_key="test-ai-key",
model="test-model",
http_client=http_client,
)
assert client.request("总结 prompt", system_prompt="只返回一段中文总结,不使用 Markdown。") == "报告摘要"
assert "只返回一段中文总结" in captured["body"]
assert "JSON Array" not in captured["body"]
def test_openai_compatible_client_accepts_base_url_that_already_includes_v1(): def test_openai_compatible_client_accepts_base_url_that_already_includes_v1():
+6
View File
@@ -11,6 +11,12 @@ def test_docker_compose_reads_real_env_file_not_example():
assert "- .env.example" not in compose_content assert "- .env.example" not in compose_content
def test_docker_compose_uses_fixed_project_name_for_stable_port_owner():
compose_content = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "name: hot-comments-tool" in compose_content
def test_real_env_file_is_gitignored(): def test_real_env_file_is_gitignored():
gitignore_lines = (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines() gitignore_lines = (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines()
+36
View File
@@ -118,3 +118,39 @@ def test_douyin_maps_null_comments_as_empty_list():
payload = {"data": {"comments": None}} payload = {"data": {"comments": None}}
assert platform.map_comments(payload, limit=10) == [] assert platform.map_comments(payload, limit=10) == []
class RecordingDouyinClient:
def __init__(self):
self.requests = []
def get(self, path, *, params=None):
self.requests.append((path, params))
cursor = params["cursor"]
if cursor == 0:
return {
"data": {
"comments": [{"cid": f"c-{index}", "text": f"评论 {index}"} for index in range(20)],
"cursor": 20,
"has_more": 1,
}
}
return {
"data": {
"comments": [{"cid": f"c-{index}", "text": f"评论 {index}"} for index in range(20, 55)],
"cursor": 55,
"has_more": 0,
}
}
def test_douyin_fetch_comments_paginates_until_limit():
client = RecordingDouyinClient()
platform = DouyinPlatform(client=client)
comments = platform.fetch_comments("aweme-1", limit=50)
assert len(comments) == 50
assert comments[0].source_comment_id == "c-0"
assert comments[-1].source_comment_id == "c-49"
assert [request[1]["cursor"] for request in client.requests] == [0, 20]
+10
View File
@@ -1,6 +1,7 @@
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from app.services import task_service from app.services import task_service
from app.config import Settings
def test_task_executor_is_single_worker_thread_pool(): def test_task_executor_is_single_worker_thread_pool():
@@ -10,3 +11,12 @@ def test_task_executor_is_single_worker_thread_pool():
def test_run_task_entrypoint_is_synchronous_function(): def test_run_task_entrypoint_is_synchronous_function():
assert task_service.inspect.iscoroutinefunction(task_service.run_task) is False assert task_service.inspect.iscoroutinefunction(task_service.run_task) is False
def test_ai_dependencies_do_not_generate_report_summary_provider_without_ai_config(monkeypatch):
monkeypatch.setattr("app.services.task_service.get_settings", lambda: Settings(_env_file=None))
requester, summary_provider = task_service.build_ai_dependencies()
assert requester("prompt") == "[]"
assert summary_provider is None