fix: 完善失败重试与跳过边界

This commit is contained in:
meijiali
2026-07-03 15:19:50 +08:00
parent accfaeffaf
commit 89c93b5051
3 changed files with 119 additions and 1 deletions
+21
View File
@@ -470,6 +470,27 @@ test: 记录双平台默认规模真实验收
fix: 完善失败重试与跳过边界
```
完成记录:
```text
完成日期:2026-07-03
相关 commit:fix: 完善失败重试与跳过边界
验证命令:
- .venv/bin/python -m pytest tests/integration/test_failure_tolerance.py tests/unit/test_report_stats.py tests/integration/test_task_creation.py tests/unit/test_comment_pagination.py tests/unit/test_ai_schema.py -q
- .venv/bin/python -m pytest tests/unit tests/integration -q
- .venv/bin/python -m pytest tests/unit tests/integration --cov=app --cov-branch --cov-report=term-missing
- curl -f http://localhost:8000/health
验收结论:
- 连续 429 超限测试覆盖并断言退避序列 1s→2s→4s,错误类型为 rate_limited 且不泄露 API Key。
- 单个内容条目评论抓取失败时记录 failed item,后续内容继续处理,任务可成功。
- 所有内容条目失败时任务 status=failederror_stage/error_type/error_message 可见。
- AI 全批失败已有测试覆盖:评论标记 failed,任务 analysis_status=insufficient。
- 报告摘要失败时 item/hotspot 报告仍创建,并使用默认总结文案。
- 已有 running 任务时拒绝新建任务,返回 400。
- 未发现 httpx.AsyncClient 使用。
遗留问题:W04 发现的 SQLite WAL/SHM deleted 句柄问题转入 WO-06。
```
### WO-06 SQLite 数据库稳定性与恢复策略
优先级:P0
@@ -34,6 +34,24 @@ class OneFailedOneSuccessfulItemPlatform:
return [CommentData(source_comment_id="c1", content="继续成功")]
class AllItemsFailPlatform:
def fetch_hotspots(self, *, limit):
from app.platforms.base import HotspotData
return [HotspotData(source_hot_id="h1", title="热点一", rank=1)]
def search_items_by_hotspot(self, keyword, *, limit):
from app.platforms.base import ContentItemData
return [
ContentItemData(source_item_id="bad-one", item_type="video", title="失败内容一"),
ContentItemData(source_item_id="bad-two", item_type="video", title="失败内容二"),
]
def fetch_comments(self, source_item_id, *, limit):
raise PlatformAPIError("comments exhausted", error_type="rate_limited", status_code=429)
def test_hotspot_failure_marks_task_failed(monkeypatch):
with make_test_client() as (_client, engine):
monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: FailingHotspotPlatform())
@@ -102,3 +120,30 @@ def test_failed_content_item_is_recorded_and_following_item_continues(monkeypatc
assert failed_item.error_stage == "crawl_comments"
assert failed_item.error_type == "rate_limited"
assert successful_item.status == "success"
def test_task_fails_with_visible_reason_when_all_content_items_fail(monkeypatch):
with make_test_client() as (_client, engine):
monkeypatch.setattr("app.services.task_service.build_platform", lambda _platform: AllItemsFailPlatform())
with Session(engine) as session:
task = create_task(
session,
CreateTaskRequest(platform="douyin", hotspot_limit=1, item_limit_per_hotspot=2, comment_limit_per_item=10),
submit_background=False,
)
task_id = task.id
run_task(task_id, session_factory=lambda: session)
persisted = session.get(Task, task_id)
failed_items = session.query(ContentItem).filter_by(task_id=task_id, status="failed").all()
assert persisted.status == "failed"
assert persisted.total_items_count == 2
assert persisted.processed_items_count == 2
assert persisted.successful_items_count == 0
assert persisted.failed_items_count == 2
assert persisted.error_stage == "crawl_comments"
assert persisted.error_type == "rate_limited"
assert persisted.error_message == "comments exhausted"
assert len(failed_items) == 2
+53 -1
View File
@@ -5,7 +5,7 @@ from sqlalchemy.orm import Session
from app.db import Base
from app.models import Comment, ContentItem, Hotspot, Task
from app.services.report_service import build_comment_metrics, generate_item_report
from app.services.report_service import DEFAULT_SUMMARY, build_comment_metrics, generate_hotspot_report, generate_item_report
def test_build_comment_metrics_counts_sentiments_and_top_labels():
@@ -74,3 +74,55 @@ def test_generate_item_report_persists_markdown_and_default_summary():
assert "样本评论数量" in report.markdown_content
finally:
engine.dispose()
def test_generate_hotspot_report_persists_default_summary_when_ai_summary_fails():
engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False})
Base.metadata.create_all(engine)
try:
with Session(engine) as session:
task = Task(platform="xiaohongshu", status="success")
session.add(task)
session.flush()
hotspot = Hotspot(task_id=task.id, platform="xiaohongshu", title="热点", raw_data="{}")
session.add(hotspot)
session.flush()
item = ContentItem(
task_id=task.id,
hotspot_id=hotspot.id,
platform="xiaohongshu",
source_item_id="n1",
item_type="note",
title="笔记",
status="success",
raw_data="{}",
)
session.add(item)
session.flush()
session.add(
Comment(
task_id=task.id,
hotspot_id=hotspot.id,
content_item_id=item.id,
platform="xiaohongshu",
source_comment_id="c1",
content="好评",
sentiment="positive",
labels='["认可"]',
raw_data="{}",
)
)
session.commit()
report = generate_hotspot_report(
session,
hotspot.id,
summary_provider=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("summary failed")),
)
assert report.report_type == "hotspot"
assert report.hotspot_id == hotspot.id
assert report.summary == DEFAULT_SUMMARY
assert DEFAULT_SUMMARY in report.markdown_content
finally:
engine.dispose()