6 Commits
9 changed files with 216 additions and 59 deletions
+5 -1
View File
@@ -5,6 +5,7 @@ from datetime import UTC, datetime
from sqlalchemy import Engine, create_engine, event from sqlalchemy import Engine, create_engine, event
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
from sqlalchemy.pool import NullPool
from app.config import get_settings from app.config import get_settings
@@ -20,7 +21,10 @@ def create_sqlite_engine(database_url: str):
db_path.parent.mkdir(parents=True, exist_ok=True) db_path.parent.mkdir(parents=True, exist_ok=True)
connect_args = {"check_same_thread": False, "timeout": 10} connect_args = {"check_same_thread": False, "timeout": 10}
engine = create_engine(database_url, connect_args=connect_args) engine_kwargs = {"connect_args": connect_args}
if database_url.startswith("sqlite:///") and database_url != "sqlite:///:memory:":
engine_kwargs["poolclass"] = NullPool
engine = create_engine(database_url, **engine_kwargs)
engine.dialect.connect_args = connect_args engine.dialect.connect_args = connect_args
@event.listens_for(engine, "connect") @event.listens_for(engine, "connect")
+4 -5
View File
@@ -3,6 +3,7 @@ import json
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from collections.abc import Callable from collections.abc import Callable
from datetime import UTC, datetime from datetime import UTC, datetime
from zoneinfo import ZoneInfo
from sqlalchemy import func, select from sqlalchemy import func, select
from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.orm import Session, sessionmaker
@@ -22,6 +23,7 @@ RUNNING_TASK_MESSAGE = "当前有正在运行的任务,请稍后再试"
RESTART_ERROR_MESSAGE = "系统重启,任务被中断" RESTART_ERROR_MESSAGE = "系统重启,任务被中断"
STALE_PROGRESS_THRESHOLD_SECONDS = 10 * 60 STALE_PROGRESS_THRESHOLD_SECONDS = 10 * 60
STALE_PROGRESS_ERROR_TYPE = "stale_progress_timeout" STALE_PROGRESS_ERROR_TYPE = "stale_progress_timeout"
DISPLAY_TIMEZONE = ZoneInfo("Asia/Shanghai")
task_executor = ThreadPoolExecutor(max_workers=1) task_executor = ThreadPoolExecutor(max_workers=1)
@@ -183,10 +185,7 @@ def hydrate_task_progress(session: Session, task: Task) -> Task:
task.display_number = display_number task.display_number = display_number
task.display_id = str(display_number) task.display_id = str(display_number)
task.created_at_label = format_datetime_minute(task.created_at) task.created_at_label = format_datetime_minute(task.created_at)
task.scale_label = ( task.scale_label = f"{task.hotspot_limit}热点 × {task.item_limit_per_hotspot}内容 × {task.comment_limit_per_item}评论"
f"{PLATFORM_LABELS.get(task.platform, task.platform)} · "
f"{task.hotspot_limit}热点 × {task.item_limit_per_hotspot}内容 × {task.comment_limit_per_item}评论"
)
task.error_summary = build_task_error_summary(task) task.error_summary = build_task_error_summary(task)
task.hotspots_count = session.scalar(select(func.count(Hotspot.id)).where(Hotspot.task_id == task.id)) or 0 task.hotspots_count = session.scalar(select(func.count(Hotspot.id)).where(Hotspot.task_id == task.id)) or 0
task.comments_count = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0 task.comments_count = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
@@ -230,7 +229,7 @@ def calculate_task_display_number(session: Session, task: Task) -> int:
def format_datetime_minute(value: datetime | None) -> str: def format_datetime_minute(value: datetime | None) -> str:
value = ensure_utc_datetime(value) value = ensure_utc_datetime(value)
return value.strftime("%Y-%m-%d %H:%M") if value else "" return value.astimezone(DISPLAY_TIMEZONE).strftime("%Y-%m-%d %H:%M") if value else ""
def build_task_error_summary(task: Task) -> str | None: def build_task_error_summary(task: Task) -> str | None:
+116 -14
View File
@@ -1,16 +1,63 @@
body { body {
background: #f5f7fb; background:
color: #172033; linear-gradient(180deg, #f4f7fb 0%, #eef3f8 42%, #f8fafc 100%);
color: #182235;
min-height: 100vh;
} }
.app-navbar {
background: rgba(255, 255, 255, 0.94) !important;
border-bottom: 1px solid #dfe7f1;
box-shadow: 0 10px 30px rgba(19, 34, 56, 0.04);
}
.navbar-brand {
color: #102033;
}
.nav-link {
color: #526173;
}
.tool-card,
.card { .card {
background: rgba(255, 255, 255, 0.96);
border-radius: 8px; border-radius: 8px;
border: 1px solid #dfe5ef; border: 1px solid #dbe4ef;
box-shadow: 0 10px 28px rgba(31, 42, 68, 0.06); box-shadow: 0 16px 42px rgba(31, 42, 68, 0.07);
overflow: hidden;
} }
.navbar { .card-header,
background: #fff !important; .section-heading {
background: linear-gradient(180deg, #ffffff, #f8fafc);
border-bottom: 1px solid #e4ebf3;
padding: 18px 20px;
}
.section-heading {
align-items: center;
display: flex;
justify-content: space-between;
gap: 16px;
}
.section-heading h2,
.page-heading h1 {
font-size: 22px;
font-weight: 700;
letter-spacing: 0;
margin: 0;
}
.section-kicker,
.eyebrow {
color: #0e766e;
font-size: 12px;
font-weight: 700;
letter-spacing: 0;
margin: 0 0 4px;
text-transform: uppercase;
} }
.hero-band { .hero-band {
@@ -20,7 +67,9 @@ body {
url("https://images.unsplash.com/photo-1551288049-bebda4e38f71?auto=format&fit=crop&w=1600&q=80"); url("https://images.unsplash.com/photo-1551288049-bebda4e38f71?auto=format&fit=crop&w=1600&q=80");
background-position: center; background-position: center;
background-size: cover; background-size: cover;
border: 1px solid rgba(255, 255, 255, 0.18);
border-radius: 8px; border-radius: 8px;
box-shadow: 0 24px 60px rgba(15, 36, 62, 0.18);
color: #fff; color: #fff;
display: flex; display: flex;
justify-content: space-between; justify-content: space-between;
@@ -52,20 +101,67 @@ body {
gap: 12px; gap: 12px;
} }
.eyebrow { .page-heading {
color: #71d4c7; align-items: center;
font-size: 13px; display: flex;
font-weight: 700; justify-content: space-between;
letter-spacing: 0; gap: 16px;
text-transform: uppercase;
} }
.metric-box { .metric-box {
background: #f8fafc; background: linear-gradient(180deg, #f9fbfd, #f3f7fb);
border: 1px solid #e3e9f2; border: 1px solid #e3e9f2;
border-radius: 8px; border-radius: 8px;
height: 100%; height: 100%;
padding: 14px; padding: 16px;
}
.metric-box strong {
color: #132033;
display: block;
font-size: 18px;
margin: 4px 0;
}
.app-table thead th {
background: #f8fafc;
color: #536174;
font-size: 12px;
font-weight: 700;
text-transform: uppercase;
white-space: nowrap;
}
.app-table tbody tr {
border-color: #e8eef5;
}
.app-table tbody tr:hover {
background: #f7fbff;
}
.task-number {
align-items: center;
background: #e7f0ff;
border: 1px solid #cfe0ff;
border-radius: 999px;
color: #0b5ed7;
display: inline-flex;
font-weight: 700;
height: 32px;
justify-content: center;
min-width: 32px;
padding: 0 10px;
}
.task-overview {
border-top: 3px solid #0d6efd;
}
.hotspot-item {
border-color: #e3eaf2;
margin-bottom: 10px;
overflow: hidden;
} }
.status-panel { .status-panel {
@@ -98,4 +194,10 @@ body {
.hero-copy h1 { .hero-copy h1 {
font-size: 36px; font-size: 36px;
} }
.section-heading,
.page-heading {
align-items: flex-start;
flex-direction: column;
}
} }
+2 -2
View File
@@ -10,9 +10,9 @@
<script src="/static/app.js" defer></script> <script src="/static/app.js" defer></script>
</head> </head>
<body> <body>
<nav class="navbar navbar-expand-lg bg-body-tertiary border-bottom"> <nav class="navbar navbar-expand-lg app-navbar">
<div class="container"> <div class="container">
<a class="navbar-brand" href="/">热榜评论分析工具</a> <a class="navbar-brand fw-semibold" href="/">热榜评论雷达</a>
<div class="navbar-nav"> <div class="navbar-nav">
<a class="nav-link" href="/">任务列表</a> <a class="nav-link" href="/">任务列表</a>
</div> </div>
+16 -11
View File
@@ -8,14 +8,16 @@
<h1>热榜评论雷达</h1> <h1>热榜评论雷达</h1>
<p>从小红书和抖音热点出发,抓取真实评论,生成 AI 情绪、标签和可导出的分析报告。</p> <p>从小红书和抖音热点出发,抓取真实评论,生成 AI 情绪、标签和可导出的分析报告。</p>
</div> </div>
<div class="hero-actions">
<a class="btn btn-light" href="#create-task">创建新任务</a>
<a class="btn btn-outline-light" href="#task-history">查看 Demo 数据</a>
</div>
</section> </section>
<section class="card mb-4" id="create-task"> <section class="tool-card mb-4" id="create-task">
<div class="card-header">创建抓取任务</div> <div class="section-heading">
<div>
<p class="section-kicker">采集配置</p>
<h2>创建抓取任务</h2>
</div>
<span class="text-muted small">默认规模 5 × 5 × 50</span>
</div>
<div class="card-body"> <div class="card-body">
<div id="form-error" class="alert alert-danger d-none" role="alert"></div> <div id="form-error" class="alert alert-danger d-none" role="alert"></div>
<form id="task-form" onsubmit="submitTask(event)"> <form id="task-form" onsubmit="submitTask(event)">
@@ -50,13 +52,16 @@
</div> </div>
</section> </section>
<section class="card" id="task-history"> <section class="tool-card" id="task-history">
<div class="card-header d-flex justify-content-between align-items-center"> <div class="section-heading">
<span>最近任务与 Demo 数据</span> <div>
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button> <p class="section-kicker">任务历史</p>
<h2>最近任务</h2>
</div>
{% if has_running_tasks %}<span class="badge text-bg-warning">自动更新中</span>{% endif %}
</div> </div>
<div class="table-responsive"> <div class="table-responsive">
<table class="table table-hover align-middle mb-0"> <table class="table app-table align-middle mb-0">
<thead> <thead>
<tr> <tr>
<th>任务 ID</th> <th>任务 ID</th>
+2 -5
View File
@@ -4,10 +4,7 @@
{% set ai_percent = rate_percent(task.analysis_success_rate) %} {% set ai_percent = rate_percent(task.analysis_success_rate) %}
<tr> <tr>
<td> <td>
<strong>{{ task.display_id or loop.index }}</strong> <span class="task-number">{{ task.display_id or loop.index }}</span>
{% if task.is_demo %}
<br><span class="badge text-bg-info">Demo 数据</span>
{% endif %}
</td> </td>
<td>{{ task.platform | platform_label }}</td> <td>{{ task.platform | platform_label }}</td>
<td>{{ task.created_at_label }}</td> <td>{{ task.created_at_label }}</td>
@@ -36,7 +33,7 @@
<br><small class="text-danger">{{ task.error_summary }}</small> <br><small class="text-danger">{{ task.error_summary }}</small>
{% endif %} {% endif %}
</td> </td>
<td><a class="btn btn-sm btn-outline-primary" href="/tasks/{{ task.id }}">查看</a></td> <td><a class="btn btn-sm btn-primary" href="/tasks/{{ task.id }}">查看</a></td>
</tr> </tr>
{% else %} {% else %}
<tr> <tr>
+10 -8
View File
@@ -7,20 +7,22 @@
</ol></nav> </ol></nav>
{% endblock %} {% endblock %}
{% block content %} {% block content %}
<div class="d-flex justify-content-between align-items-center mb-3"> <div class="page-heading mb-3">
<h1 class="h3 mb-0">任务 {{ task.display_id }}</h1> <div>
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.reload()">手动刷新</button> <p class="section-kicker">任务详情</p>
<h1>任务 {{ task.display_id }}</h1>
</div> </div>
<section class="card mb-4" {% if task.status == "running" %}data-task-id="{{ task.id }}" data-auto-poll="true"{% endif %}><div class="card-body">
{% set label, cls = status_badge_config(task.status) %} {% set label, cls = status_badge_config(task.status) %}
<span class="badge {{ cls }}">{{ label }}</span>
</div>
<section class="tool-card task-overview mb-4" {% if task.status == "running" %}data-task-id="{{ task.id }}" data-auto-poll="true"{% endif %}><div class="card-body">
{% set percent = progress_percent(task.processed_items_count, task.total_items_count) %} {% set percent = progress_percent(task.processed_items_count, task.total_items_count) %}
{% set ai_percent = rate_percent(task.analysis_success_rate) %} {% set ai_percent = rate_percent(task.analysis_success_rate) %}
{% set target_comments = task.hotspot_limit * task.item_limit_per_hotspot * task.comment_limit_per_item %} {% set target_comments = task.hotspot_limit * task.item_limit_per_hotspot * task.comment_limit_per_item %}
<div class="d-flex flex-wrap justify-content-between gap-3 mb-3"> <div class="d-flex flex-wrap justify-content-between gap-3 mb-3">
<p class="mb-0">平台:{{ task.platform | platform_label }} <span class="badge {{ cls }}">{{ label }}</span>{% if task.is_demo %}<span class="badge text-bg-info ms-1">Demo 数据</span>{% endif %}</p> <p class="mb-0">平台:{{ task.platform | platform_label }}</p>
<p class="mb-0 text-muted">阶段{{ task.current_stage_label or "等待启动" }}</p> <p class="mb-0 text-muted">创建时间 {{ task.created_at_label }} · 阶段 {{ task.current_stage_label or "等待启动" }}</p>
</div> </div>
<div class="small text-muted mb-3">创建时间 {{ task.created_at_label }} · 完整 ID {{ task.id }}</div>
<div class="mb-3"> <div class="mb-3">
<div class="d-flex justify-content-between align-items-center gap-2"> <div class="d-flex justify-content-between align-items-center gap-2">
<span>已处理 {{ task.processed_items_count }} / 共 {{ task.total_items_count }} 条内容</span> <span>已处理 {{ task.processed_items_count }} / 共 {{ task.total_items_count }} 条内容</span>
@@ -80,7 +82,7 @@
{% endif %} {% endif %}
<div class="accordion" id="hotspot-list"> <div class="accordion" id="hotspot-list">
{% for hotspot in hotspots %} {% for hotspot in hotspots %}
<div class="accordion-item"> <div class="accordion-item hotspot-item">
<h2 class="accordion-header"> <h2 class="accordion-header">
<button class="accordion-button {% if not loop.first %}collapsed{% endif %}" type="button" data-bs-toggle="collapse" data-bs-target="#hotspot-{{ hotspot.id }}"> <button class="accordion-button {% if not loop.first %}collapsed{% endif %}" type="button" data-bs-toggle="collapse" data-bs-target="#hotspot-{{ hotspot.id }}">
热点 {{ hotspot.rank }}{{ hotspot.title }} 热点 {{ hotspot.rank }}{{ hotspot.title }}
+51 -12
View File
@@ -18,7 +18,10 @@ def test_index_page_renders_task_form_empty_state_and_default_scale():
assert 'name="item_limit_per_hotspot"' in response.text assert 'name="item_limit_per_hotspot"' in response.text
assert 'name="comment_limit_per_item"' in response.text assert 'name="comment_limit_per_item"' in response.text
assert "1250" in response.text assert "1250" in response.text
assert 'onclick="window.location.href=\'/\'"' in response.text assert "手动刷新" not in response.text
assert 'onclick="window.location.href=\'/\'"' not in response.text
assert 'href="#create-task"' not in response.text
assert 'href="#task-history"' not in response.text
assert "还没有任何任务" in response.text assert "还没有任何任务" in response.text
@@ -104,10 +107,13 @@ def test_index_page_uses_short_task_numbers_compact_fields_and_friendly_error_co
assert "#" not in visible_text assert "#" not in visible_text
assert first_id not in visible_text assert first_id not in visible_text
assert second_id not in visible_text assert second_id not in visible_text
assert "2026-07-03 10:30" in visible_text assert "2026-07-03 18:30" in visible_text
assert "2026-07-03 10:35" in visible_text assert "2026-07-03 18:35" in visible_text
assert "小红书 · 5热点 × 5内容 × 50评论" in visible_text assert "Demo 数据" not in visible_text
assert "抖音 · 1热点 × 1内容 × 10评论" in visible_text assert "5热点 × 5内容 × 50评论" in visible_text
assert "1热点 × 1内容 × 10评论" in visible_text
assert "小红书 · 5热点" not in visible_text
assert "抖音 · 1热点" not in visible_text
assert "系统重启,任务被中断" in visible_text assert "系统重启,任务被中断" in visible_text
assert "system / unexpected_restart" not in visible_text assert "system / unexpected_restart" not in visible_text
assert "unexpected_restart" not in visible_text assert "unexpected_restart" not in visible_text
@@ -167,12 +173,13 @@ def test_running_task_detail_page_auto_polls_current_task():
assert response.status_code == 200 assert response.status_code == 200
assert 'data-task-id="task-running"' in response.text assert 'data-task-id="task-running"' in response.text
assert "pollTaskDetailStatus" in response.text assert "pollTaskDetailStatus" in response.text
assert 'onclick="window.location.reload()"' in response.text assert "手动刷新" not in response.text
assert 'onclick="window.location.reload()"' not in response.text
assert "已处理 1 / 共 2 条内容" in response.text assert "已处理 1 / 共 2 条内容" in response.text
assert "AI 成功率 100%" in response.text assert "AI 成功率 100%" in response.text
def test_task_detail_uses_short_number_title_and_keeps_full_uuid_for_diagnostics(): def test_task_detail_uses_short_number_title_and_hides_full_uuid():
task_id = "33333333-3333-4333-8333-333333333333" task_id = "33333333-3333-4333-8333-333333333333"
with make_test_client() as (client, engine): with make_test_client() as (client, engine):
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -197,11 +204,42 @@ def test_task_detail_uses_short_number_title_and_keeps_full_uuid_for_diagnostics
visible_text = BeautifulSoup(response.text, "html.parser").get_text(" ") visible_text = BeautifulSoup(response.text, "html.parser").get_text(" ")
assert "<title>任务 1 - 热榜评论分析工具</title>" in response.text assert "<title>任务 1 - 热榜评论分析工具</title>" in response.text
assert "任务 1" in visible_text assert "任务 1" in visible_text
assert "完整 ID" in visible_text assert "完整 ID" not in visible_text
assert task_id in visible_text assert task_id not in visible_text
assert "任务 #33333333-3333-4333-8333-333333333333" not in visible_text assert "任务 #33333333-3333-4333-8333-333333333333" not in visible_text
assert "#" not in visible_text assert "#" not in visible_text
assert "创建时间 2026-07-03 10:30" in visible_text assert "创建时间 2026-07-03 18:30" in visible_text
assert "Demo 数据" not in visible_text
def test_demo_task_flag_is_kept_but_demo_copy_is_hidden_from_pages():
with make_test_client() as (client, engine):
from sqlalchemy.orm import Session
with Session(engine) as session:
session.add(
Task(
id="demo-task-hidden-copy",
platform="xiaohongshu",
status="success",
created_at=datetime(2026, 7, 3, 10, 30, tzinfo=UTC),
hotspot_limit=1,
item_limit_per_hotspot=1,
comment_limit_per_item=10,
)
)
session.commit()
index_response = client.get("/")
detail_response = client.get("/tasks/demo-task-hidden-copy")
api_response = client.get("/api/tasks/demo-task-hidden-copy")
assert index_response.status_code == 200
assert detail_response.status_code == 200
assert api_response.status_code == 200
assert api_response.json()["is_demo"] is True
assert "Demo 数据" not in BeautifulSoup(index_response.text, "html.parser").get_text(" ")
assert "Demo 数据" not in BeautifulSoup(detail_response.text, "html.parser").get_text(" ")
def test_running_task_detail_page_keeps_polling_after_hotspots_exist(): def test_running_task_detail_page_keeps_polling_after_hotspots_exist():
@@ -296,7 +334,7 @@ def test_running_task_detail_page_shows_stage_runtime_and_stale_progress_warning
response = client.get("/tasks/task-stale") response = client.get("/tasks/task-stale")
assert response.status_code == 200 assert response.status_code == 200
assert "阶段AI 分析中" in response.text assert "阶段 AI 分析中" in response.text
assert "运行时长" in response.text assert "运行时长" in response.text
assert "2小时5分钟" in response.text assert "2小时5分钟" in response.text
assert "最近进度" in response.text assert "最近进度" in response.text
@@ -419,7 +457,8 @@ def test_result_pages_render_seeded_data():
assert task_response.status_code == 200 assert task_response.status_code == 200
assert "热点标题" in task_response.text assert "热点标题" in task_response.text
assert "热点 1:热点标题" in BeautifulSoup(task_response.text, "html.parser").get_text(" ") assert "热点 1:热点标题" in BeautifulSoup(task_response.text, "html.parser").get_text(" ")
assert 'onclick="window.location.reload()"' in task_response.text assert 'onclick="window.location.reload()"' not in task_response.text
assert "手动刷新" not in task_response.text
assert hotspot_response.status_code == 200 assert hotspot_response.status_code == 200
assert "热点总结" in hotspot_response.text assert "热点总结" in hotspot_response.text
assert "downloadExport(" not in hotspot_response.text assert "downloadExport(" not in hotspot_response.text
+9
View File
@@ -1,4 +1,5 @@
import pytest import pytest
from sqlalchemy.pool import NullPool
from app.db import check_database_integrity, checkpoint_sqlite_wal, create_sqlite_engine, ensure_sqlite_schema_compat from app.db import check_database_integrity, checkpoint_sqlite_wal, create_sqlite_engine, ensure_sqlite_schema_compat
@@ -11,6 +12,14 @@ def test_check_database_integrity_returns_ok_for_valid_sqlite_database():
engine.dispose() engine.dispose()
def test_file_sqlite_engine_does_not_reuse_connections_after_operational_errors(tmp_path):
engine = create_sqlite_engine(f"sqlite:///{tmp_path / 'app.db'}")
try:
assert isinstance(engine.pool, NullPool)
finally:
engine.dispose()
def test_check_database_integrity_raises_when_sqlite_reports_problem(monkeypatch): def test_check_database_integrity_raises_when_sqlite_reports_problem(monkeypatch):
class FakeCursor: class FakeCursor:
def execute(self, _sql): def execute(self, _sql):