merge: 合并移除导出功能分支

# Conflicts:
#	tests/integration/test_routes.py
This commit is contained in:
meijiali
2026-07-03 19:28:59 +08:00
9 changed files with 31 additions and 289 deletions
-47
View File
@@ -1,11 +1,9 @@
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
import logging import logging
from urllib.parse import quote
from fastapi import Depends, FastAPI, HTTPException, Request, status from fastapi import Depends, FastAPI, HTTPException, Request, status
from fastapi.responses import HTMLResponse, JSONResponse from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.responses import Response
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
from sqlalchemy.exc import OperationalError, SQLAlchemyError from sqlalchemy.exc import OperationalError, SQLAlchemyError
from sqlalchemy import select from sqlalchemy import select
@@ -14,7 +12,6 @@ from sqlalchemy.orm import Session
from app.db import SessionLocal, backup_sqlite_files, get_db_session, init_db from app.db import SessionLocal, backup_sqlite_files, get_db_session, init_db
from app.models import Comment, ContentItem, Hotspot, Report from app.models import Comment, ContentItem, Hotspot, Report
from app.schemas import CreateTaskRequest, CreateTaskResponse, TaskResponse from app.schemas import CreateTaskRequest, CreateTaskResponse, TaskResponse
from app.services.export_service import export_hotspot_comments_csv, export_item_comments_csv, export_report_markdown
from app.services.task_service import ( from app.services.task_service import (
RUNNING_TASK_MESSAGE, RUNNING_TASK_MESSAGE,
create_task, create_task,
@@ -146,47 +143,3 @@ def item_detail_page(item_id: str, request: Request, session: Session = Depends(
"items/detail.html", "items/detail.html",
{"task": task, "hotspot": hotspot, "item": item, "report": report, "comments": comments}, {"task": task, "hotspot": hotspot, "item": item, "report": report, "comments": comments},
) )
@app.get("/api/export/items/{item_id}/comments.csv")
def export_item_comments(item_id: str, session: Session = Depends(get_db_session)) -> Response:
try:
content, filename = export_item_comments_csv(session, item_id)
except ValueError:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="导出数据不存在")
return Response(
content=content,
media_type="text/csv; charset=utf-8",
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
)
@app.get("/api/export/hotspots/{hotspot_id}/comments.csv")
def export_hotspot_comments(hotspot_id: str, session: Session = Depends(get_db_session)) -> Response:
try:
content, filename = export_hotspot_comments_csv(session, hotspot_id)
except ValueError:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="导出数据不存在")
return Response(
content=content,
media_type="text/csv; charset=utf-8",
headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"},
)
@app.get("/api/export/hotspots/{hotspot_id}.md")
def export_hotspot_markdown(hotspot_id: str, session: Session = Depends(get_db_session)) -> Response:
try:
content, filename = export_report_markdown(session, report_type="hotspot", hotspot_id=hotspot_id)
except ValueError:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="报告不存在")
return Response(content=content, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"})
@app.get("/api/export/items/{item_id}.md")
def export_item_markdown(item_id: str, session: Session = Depends(get_db_session)) -> Response:
try:
content, filename = export_report_markdown(session, report_type="item", item_id=item_id)
except ValueError:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="报告不存在")
return Response(content=content, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f"attachment; filename*=UTF-8''{quote(filename)}"})
-104
View File
@@ -1,104 +0,0 @@
import csv
import io
import json
import re
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.models import Comment, ContentItem, Hotspot, Report
def labels_to_text(labels_json: str | None) -> str:
try:
labels = json.loads(labels_json or "[]")
except json.JSONDecodeError:
return ""
return "".join(str(label) for label in labels) if isinstance(labels, list) else ""
def safe_csv_field(value) -> str:
text = "" if value is None else str(value)
text = text.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")
if text.startswith(("=", "+", "-", "@")):
return f"'{text}"
return text
def safe_filename(platform: str, task_id: str, hotspot_keyword: str, *, extension: str = "csv") -> str:
keyword = hotspot_keyword[:20] + ("..." if len(hotspot_keyword) > 20 else "")
name = f"{platform}_{task_id}_{keyword}"
name = re.sub(r'[/\\:*?"<>|]+', "_", name)
name = re.sub(r"_+", "_", name).strip("_")
return f"{name}.{extension}"
def export_item_comments_csv(session: Session, item_id: str) -> tuple[bytes, str]:
item = session.get(ContentItem, item_id)
if item is None:
raise ValueError("content item not found")
hotspot = session.get(Hotspot, item.hotspot_id)
comments = list(session.scalars(select(Comment).where(Comment.content_item_id == item.id)))
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["平台", "任务 ID", "热点 ID", "热点标题", "内容条目 ID", "内容条目标题", "评论 ID", "评论内容", "情绪倾向", "方向标签", "点赞数", "评论时间"])
for comment in comments:
writer.writerow(
[
item.platform,
item.task_id,
hotspot.id if hotspot else "",
hotspot.title if hotspot else "",
item.id,
item.title or "",
comment.source_comment_id or "",
safe_csv_field(comment.content),
comment.sentiment,
labels_to_text(comment.labels),
comment.like_count or 0,
comment.comment_time or "",
]
)
filename = safe_filename(item.platform, item.task_id, hotspot.title if hotspot else item.title or "comments")
return output.getvalue().encode("utf-8-sig"), filename
def export_hotspot_comments_csv(session: Session, hotspot_id: str) -> tuple[bytes, str]:
hotspot = session.get(Hotspot, hotspot_id)
if hotspot is None:
raise ValueError("hotspot not found")
comments = list(session.scalars(select(Comment).where(Comment.hotspot_id == hotspot.id)))
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(["平台", "任务 ID", "热点 ID", "热点标题", "内容条目 ID", "内容条目标题", "评论 ID", "评论内容", "情绪倾向", "方向标签", "点赞数", "评论时间"])
for comment in comments:
item = session.get(ContentItem, comment.content_item_id)
writer.writerow(
[
comment.platform,
comment.task_id,
hotspot.id,
hotspot.title,
item.id if item else "",
item.title if item else "",
comment.source_comment_id or "",
safe_csv_field(comment.content),
comment.sentiment,
labels_to_text(comment.labels),
comment.like_count or 0,
comment.comment_time or "",
]
)
return output.getvalue().encode("utf-8-sig"), safe_filename(hotspot.platform, hotspot.task_id, hotspot.title)
def export_report_markdown(session: Session, *, report_type: str, hotspot_id: str | None = None, item_id: str | None = None) -> tuple[str, str]:
query = select(Report).where(Report.report_type == report_type)
if hotspot_id:
query = query.where(Report.hotspot_id == hotspot_id)
if item_id:
query = query.where(Report.content_item_id == item_id)
report = session.scalar(query)
if report is None:
raise ValueError("report not found")
return report.markdown_content or report.markdown, safe_filename("report", report.task_id, report.title, extension="md")
-43
View File
@@ -111,46 +111,3 @@ function pollTaskListStatus() {
}; };
window.setTimeout(poll, intervalMs); window.setTimeout(poll, intervalMs);
} }
async function downloadExport(url, defaultFilename, event) {
if (event) event.preventDefault();
const button = event ? event.currentTarget : null;
const originalText = button ? button.textContent : "";
if (button) {
button.disabled = true;
button.textContent = "下载中...";
}
try {
const resp = await fetch(url, { cache: "no-store" });
if (!resp.ok) {
let message = "导出失败,请稍后重试。";
try {
const data = await resp.json();
if (data.detail) message = data.detail;
} catch (_error) {
message = resp.status === 503 ? "数据库暂时不可用,请稍后重试。" : message;
}
alert(message);
return;
}
const blob = await resp.blob();
const disposition = resp.headers.get("Content-Disposition") || "";
const match = disposition.match(/filename\*=UTF-8''([^;]+)/);
const filename = match ? decodeURIComponent(match[1]) : defaultFilename;
const blobUrl = URL.createObjectURL(blob);
const link = document.createElement("a");
link.href = blobUrl;
link.download = filename;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
URL.revokeObjectURL(blobUrl);
} catch (_error) {
alert("网络异常,导出未完成。");
} finally {
if (button) {
button.disabled = false;
button.textContent = originalText;
}
}
}
-14
View File
@@ -10,19 +10,6 @@
{% block content %} {% block content %}
<div class="d-flex justify-content-between align-items-center mb-3"> <div class="d-flex justify-content-between align-items-center mb-3">
<h1 class="h3 mb-0">{{ hotspot.title }} 汇总报告</h1> <h1 class="h3 mb-0">{{ hotspot.title }} 汇总报告</h1>
<div class="d-flex gap-2">
<button
class="btn btn-outline-primary"
onclick="downloadExport('/api/export/hotspots/{{ hotspot.id }}.md', 'hotspot-report.md', event)"
{% if not report %}disabled title="报告尚未生成"{% endif %}>
导出 Markdown 报告
</button>
<button
class="btn btn-outline-secondary"
onclick="downloadExport('/api/export/hotspots/{{ hotspot.id }}/comments.csv', 'hotspot-comments.csv', event)">
导出热点评论 CSV
</button>
</div>
</div> </div>
{% if report %} {% if report %}
{% include "partials/report_panel.html" %} {% include "partials/report_panel.html" %}
@@ -30,7 +17,6 @@
<div class="text-center text-muted py-5"> <div class="text-center text-muted py-5">
<div class="spinner-border text-warning mb-3" role="status"></div> <div class="spinner-border text-warning mb-3" role="status"></div>
<p>报告生成中,请稍候...</p> <p>报告生成中,请稍候...</p>
<p class="small mb-0">Markdown 报告将在分析完成后开放下载;评论 CSV 可先导出已抓取的数据。</p>
</div> </div>
{% endif %} {% endif %}
{% endblock %} {% endblock %}
-14
View File
@@ -11,19 +11,6 @@
{% block content %} {% block content %}
<div class="d-flex justify-content-between align-items-center mb-3"> <div class="d-flex justify-content-between align-items-center mb-3">
<h1 class="h3 mb-0">{{ item.title or item.source_item_id }}</h1> <h1 class="h3 mb-0">{{ item.title or item.source_item_id }}</h1>
<div class="d-flex gap-2">
<button
class="btn btn-outline-primary"
onclick="downloadExport('/api/export/items/{{ item.id }}.md', 'item-report.md', event)"
{% if not report %}disabled title="报告尚未生成"{% endif %}>
导出 Markdown 报告
</button>
<button
class="btn btn-outline-secondary"
onclick="downloadExport('/api/export/items/{{ item.id }}/comments.csv', 'item-comments.csv', event)">
导出评论 CSV
</button>
</div>
</div> </div>
{% if report %} {% if report %}
{% include "partials/report_panel.html" %} {% include "partials/report_panel.html" %}
@@ -31,7 +18,6 @@
<div class="text-center text-muted py-5"> <div class="text-center text-muted py-5">
<div class="spinner-border text-warning mb-3" role="status"></div> <div class="spinner-border text-warning mb-3" role="status"></div>
<p>报告生成中,请稍候...</p> <p>报告生成中,请稍候...</p>
<p class="small mb-0">Markdown 报告将在分析完成后开放下载;评论 CSV 可先导出已抓取的数据。</p>
</div> </div>
{% endif %} {% endif %}
<section class="card"><div class="card-header">评论明细</div><div class="table-responsive"> <section class="card"><div class="card-header">评论明细</div><div class="table-responsive">
+6 -9
View File
@@ -93,16 +93,13 @@ http://localhost:8000
如果报告尚未生成,页面会显示“报告生成中,请稍候...”,不会返回 500。 如果报告尚未生成,页面会显示“报告生成中,请稍候...”,不会返回 500。
## 5. 导出文件 ## 5. 文件导出
支持导出 当前版本已关闭 CSV / Markdown 文件下载入口和接口。页面仍可直接查看
- 热点报告 Markdown - 热点级汇总报告
- 内容报告 Markdown - 内容条目级报告
- 热点评论 CSV - 评论明细
- 内容评论 CSV
CSV 使用 UTF-8-BOM,适合 Windows Excel 打开。评论内容中的换行会被替换为空格,`= + - @` 开头的内容会自动加单引号,避免被 Excel 当作公式执行。
## 6. 小规模验收 ## 6. 小规模验收
@@ -119,7 +116,7 @@ CSV 使用 UTF-8-BOM,适合 Windows Excel 打开。评论内容中的换行会
- 至少有热点、内容、评论、报告。 - 至少有热点、内容、评论、报告。
- AI 成功率正常,或页面有明确“AI 样本不足”提示。 - AI 成功率正常,或页面有明确“AI 样本不足”提示。
- 能打开热点报告、内容详情页。 - 能打开热点报告、内容详情页。
- 能导出 Markdown 和 CSV - 页面不展示 CSV / Markdown 下载入口
## 7. 默认规模验收 ## 7. 默认规模验收
+21 -32
View File
@@ -422,8 +422,10 @@ def test_result_pages_render_seeded_data():
assert 'onclick="window.location.reload()"' in task_response.text assert 'onclick="window.location.reload()"' in task_response.text
assert hotspot_response.status_code == 200 assert hotspot_response.status_code == 200
assert "热点总结" in hotspot_response.text assert "热点总结" in hotspot_response.text
assert "downloadExport('/api/export/hotspots/hot-result.md'" in hotspot_response.text assert "downloadExport(" not in hotspot_response.text
assert "downloadExport('/api/export/hotspots/hot-result/comments.csv'" in hotspot_response.text assert "/api/export/" not in hotspot_response.text
assert "导出 Markdown" not in hotspot_response.text
assert "导出热点评论 CSV" not in hotspot_response.text
assert "评论样本" in hotspot_response.text assert "评论样本" in hotspot_response.text
assert "关联内容" in hotspot_response.text assert "关联内容" in hotspot_response.text
assert "正向" in hotspot_response.text assert "正向" in hotspot_response.text
@@ -437,8 +439,10 @@ def test_result_pages_render_seeded_data():
item_visible_text = BeautifulSoup(item_response.text, "html.parser").get_text(" ") item_visible_text = BeautifulSoup(item_response.text, "html.parser").get_text(" ")
assert "热点 1:热点标题" in item_visible_text assert "热点 1:热点标题" in item_visible_text
assert "#" not in item_visible_text assert "#" not in item_visible_text
assert "downloadExport('/api/export/items/item-result.md'" in item_response.text assert "downloadExport(" not in item_response.text
assert "downloadExport('/api/export/items/item-result/comments.csv'" in item_response.text assert "/api/export/" not in item_response.text
assert "导出 Markdown" not in item_response.text
assert "导出评论 CSV" not in item_response.text
assert "评论样本" in item_response.text assert "评论样本" in item_response.text
assert "Top 标签" in item_response.text assert "Top 标签" in item_response.text
assert "评论内容" in item_response.text assert "评论内容" in item_response.text
@@ -473,14 +477,16 @@ def test_report_pages_render_empty_state_when_report_missing():
assert hotspot_response.status_code == 200 assert hotspot_response.status_code == 200
assert "报告生成中,请稍候" in hotspot_response.text assert "报告生成中,请稍候" in hotspot_response.text
assert 'title="报告尚未生成"' in hotspot_response.text assert 'title="报告尚未生成"' not in hotspot_response.text
assert "Markdown 报告将在分析完成后开放下载" in hotspot_response.text assert "Markdown 报告将在分析完成后开放下载" not in hotspot_response.text
assert "downloadExport('/api/export/hotspots/hot-no-report/comments.csv'" in hotspot_response.text assert "downloadExport(" not in hotspot_response.text
assert "/api/export/" not in hotspot_response.text
assert item_response.status_code == 200 assert item_response.status_code == 200
assert "报告生成中,请稍候" in item_response.text assert "报告生成中,请稍候" in item_response.text
assert 'title="报告尚未生成"' in item_response.text assert 'title="报告尚未生成"' not in item_response.text
assert "Markdown 报告将在分析完成后开放下载" in item_response.text assert "Markdown 报告将在分析完成后开放下载" not in item_response.text
assert "downloadExport('/api/export/items/item-no-report/comments.csv'" in item_response.text assert "downloadExport(" not in item_response.text
assert "/api/export/" not in item_response.text
def test_report_pages_warn_when_ai_sample_is_insufficient(): def test_report_pages_warn_when_ai_sample_is_insufficient():
@@ -503,7 +509,7 @@ def test_report_pages_warn_when_ai_sample_is_insufficient():
assert "当前有效评论样本不足" in item_response.text assert "当前有效评论样本不足" in item_response.text
def test_export_routes_return_csv_and_markdown(): def test_export_routes_are_removed():
with make_test_client() as (client, engine): with make_test_client() as (client, engine):
seed_result_data(engine) seed_result_data(engine)
@@ -512,15 +518,10 @@ def test_export_routes_return_csv_and_markdown():
hotspot_md = client.get("/api/export/hotspots/hot-result.md") hotspot_md = client.get("/api/export/hotspots/hot-result.md")
item_md = client.get("/api/export/items/item-result.md") item_md = client.get("/api/export/items/item-result.md")
assert csv_response.status_code == 200 assert csv_response.status_code == 404
assert csv_response.content.startswith(b"\xef\xbb\xbf") assert hotspot_csv_response.status_code == 404
assert "评论内容".encode("utf-8") in csv_response.content assert hotspot_md.status_code == 404
assert hotspot_csv_response.status_code == 200 assert item_md.status_code == 404
assert "评论内容".encode("utf-8") in hotspot_csv_response.content
assert hotspot_md.status_code == 200
assert "热点总结" in hotspot_md.text
assert item_md.status_code == 200
assert "内容总结" in item_md.text
def test_api_tasks_returns_service_unavailable_when_database_has_io_error(monkeypatch): def test_api_tasks_returns_service_unavailable_when_database_has_io_error(monkeypatch):
@@ -616,18 +617,6 @@ def test_item_detail_page_shows_database_error_state_when_database_has_io_error(
assert "数据库暂时不可用" in response.text assert "数据库暂时不可用" in response.text
def test_export_route_returns_service_unavailable_when_database_has_io_error(monkeypatch):
def broken_export(_session, _item_id):
raise OperationalError("SELECT 1", {}, Exception("disk I/O error"))
monkeypatch.setattr("app.main.export_item_comments_csv", broken_export)
with make_test_client() as (client, _engine):
response = client.get("/api/export/items/item-io/comments.csv")
assert response.status_code == 503
assert response.json()["detail"] == "数据库暂时不可用,请稍后重试或联系维护者恢复数据。"
def test_missing_html_pages_render_friendly_not_found_page(): def test_missing_html_pages_render_friendly_not_found_page():
with make_test_client() as (client, _engine): with make_test_client() as (client, _engine):
+4 -2
View File
@@ -4,7 +4,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[2] ROOT = Path(__file__).resolve().parents[2]
def test_user_guide_covers_startup_acceptance_exports_and_troubleshooting(): def test_user_guide_covers_startup_acceptance_no_exports_and_troubleshooting():
content = (ROOT / "docs" / "UserGuide.md").read_text(encoding="utf-8") content = (ROOT / "docs" / "UserGuide.md").read_text(encoding="utf-8")
for required in [ for required in [
@@ -12,7 +12,9 @@ def test_user_guide_covers_startup_acceptance_exports_and_troubleshooting():
"http://localhost:8000", "http://localhost:8000",
"1 × 1 × 10", "1 × 1 × 10",
"5 × 5 × 50", "5 × 5 × 50",
"导出文件", "文件导出",
"已关闭 CSV / Markdown 文件下载入口和接口",
"页面不展示 CSV / Markdown 下载入口",
"8000 端口被占用", "8000 端口被占用",
"API Key 缺失", "API Key 缺失",
"任务长期 running", "任务长期 running",
-24
View File
@@ -1,24 +0,0 @@
from app.services.export_service import labels_to_text, safe_csv_field, safe_filename
def test_labels_to_text_joins_json_array_with_chinese_comma():
assert labels_to_text('["认可", "质量好"]') == "认可,质量好"
assert labels_to_text("bad-json") == ""
def test_safe_csv_field_prevents_formula_injection_and_newlines():
assert safe_csv_field("=1+1") == "'=1+1"
assert safe_csv_field("+1+1") == "'+1+1"
assert safe_csv_field("-1+1") == "'-1+1"
assert safe_csv_field("@SUM(1,1)") == "'@SUM(1,1)"
assert safe_csv_field("hello\nworld") == "hello world"
assert safe_csv_field("hello\r\nworld") == "hello world"
assert safe_csv_field("hello\rworld") == "hello world"
def test_safe_filename_replaces_illegal_chars_and_truncates_keyword():
filename = safe_filename("douyin", "task", "a/b:c*d?e<f>g|很长很长很长很长很长")
assert "/" not in filename
assert "__" not in filename
assert filename.endswith(".csv")