Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
918dd62e21 | ||
|
|
1d24cbf841 | ||
|
|
9d212b54be | ||
|
|
c406364d14 | ||
|
|
89dcf49186 | ||
|
|
78cc5df73d |
@@ -5,6 +5,7 @@ from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Engine, create_engine, event
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
@@ -20,7 +21,10 @@ def create_sqlite_engine(database_url: str):
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
connect_args = {"check_same_thread": False, "timeout": 10}
|
||||
engine = create_engine(database_url, connect_args=connect_args)
|
||||
engine_kwargs = {"connect_args": connect_args}
|
||||
if database_url.startswith("sqlite:///") and database_url != "sqlite:///:memory:":
|
||||
engine_kwargs["poolclass"] = NullPool
|
||||
engine = create_engine(database_url, **engine_kwargs)
|
||||
engine.dialect.connect_args = connect_args
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
|
||||
@@ -3,6 +3,7 @@ import json
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
@@ -22,6 +23,7 @@ RUNNING_TASK_MESSAGE = "当前有正在运行的任务,请稍后再试"
|
||||
RESTART_ERROR_MESSAGE = "系统重启,任务被中断"
|
||||
STALE_PROGRESS_THRESHOLD_SECONDS = 10 * 60
|
||||
STALE_PROGRESS_ERROR_TYPE = "stale_progress_timeout"
|
||||
DISPLAY_TIMEZONE = ZoneInfo("Asia/Shanghai")
|
||||
task_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
|
||||
@@ -183,10 +185,7 @@ def hydrate_task_progress(session: Session, task: Task) -> Task:
|
||||
task.display_number = display_number
|
||||
task.display_id = str(display_number)
|
||||
task.created_at_label = format_datetime_minute(task.created_at)
|
||||
task.scale_label = (
|
||||
f"{PLATFORM_LABELS.get(task.platform, task.platform)} · "
|
||||
f"{task.hotspot_limit}热点 × {task.item_limit_per_hotspot}内容 × {task.comment_limit_per_item}评论"
|
||||
)
|
||||
task.scale_label = f"{task.hotspot_limit}热点 × {task.item_limit_per_hotspot}内容 × {task.comment_limit_per_item}评论"
|
||||
task.error_summary = build_task_error_summary(task)
|
||||
task.hotspots_count = session.scalar(select(func.count(Hotspot.id)).where(Hotspot.task_id == task.id)) or 0
|
||||
task.comments_count = session.scalar(select(func.count(Comment.id)).where(Comment.task_id == task.id)) or 0
|
||||
@@ -230,7 +229,7 @@ def calculate_task_display_number(session: Session, task: Task) -> int:
|
||||
|
||||
def format_datetime_minute(value: datetime | None) -> str:
|
||||
value = ensure_utc_datetime(value)
|
||||
return value.strftime("%Y-%m-%d %H:%M") if value else ""
|
||||
return value.astimezone(DISPLAY_TIMEZONE).strftime("%Y-%m-%d %H:%M") if value else ""
|
||||
|
||||
|
||||
def build_task_error_summary(task: Task) -> str | None:
|
||||
|
||||
+116
-14
@@ -1,16 +1,63 @@
|
||||
body {
|
||||
background: #f5f7fb;
|
||||
color: #172033;
|
||||
background:
|
||||
linear-gradient(180deg, #f4f7fb 0%, #eef3f8 42%, #f8fafc 100%);
|
||||
color: #182235;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
.app-navbar {
|
||||
background: rgba(255, 255, 255, 0.94) !important;
|
||||
border-bottom: 1px solid #dfe7f1;
|
||||
box-shadow: 0 10px 30px rgba(19, 34, 56, 0.04);
|
||||
}
|
||||
|
||||
.navbar-brand {
|
||||
color: #102033;
|
||||
}
|
||||
|
||||
.nav-link {
|
||||
color: #526173;
|
||||
}
|
||||
|
||||
.tool-card,
|
||||
.card {
|
||||
background: rgba(255, 255, 255, 0.96);
|
||||
border-radius: 8px;
|
||||
border: 1px solid #dfe5ef;
|
||||
box-shadow: 0 10px 28px rgba(31, 42, 68, 0.06);
|
||||
border: 1px solid #dbe4ef;
|
||||
box-shadow: 0 16px 42px rgba(31, 42, 68, 0.07);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.navbar {
|
||||
background: #fff !important;
|
||||
.card-header,
|
||||
.section-heading {
|
||||
background: linear-gradient(180deg, #ffffff, #f8fafc);
|
||||
border-bottom: 1px solid #e4ebf3;
|
||||
padding: 18px 20px;
|
||||
}
|
||||
|
||||
.section-heading {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.section-heading h2,
|
||||
.page-heading h1 {
|
||||
font-size: 22px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.section-kicker,
|
||||
.eyebrow {
|
||||
color: #0e766e;
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
margin: 0 0 4px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.hero-band {
|
||||
@@ -20,7 +67,9 @@ body {
|
||||
url("https://images.unsplash.com/photo-1551288049-bebda4e38f71?auto=format&fit=crop&w=1600&q=80");
|
||||
background-position: center;
|
||||
background-size: cover;
|
||||
border: 1px solid rgba(255, 255, 255, 0.18);
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 24px 60px rgba(15, 36, 62, 0.18);
|
||||
color: #fff;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
@@ -52,20 +101,67 @@ body {
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
color: #71d4c7;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0;
|
||||
text-transform: uppercase;
|
||||
.page-heading {
|
||||
align-items: center;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.metric-box {
|
||||
background: #f8fafc;
|
||||
background: linear-gradient(180deg, #f9fbfd, #f3f7fb);
|
||||
border: 1px solid #e3e9f2;
|
||||
border-radius: 8px;
|
||||
height: 100%;
|
||||
padding: 14px;
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
.metric-box strong {
|
||||
color: #132033;
|
||||
display: block;
|
||||
font-size: 18px;
|
||||
margin: 4px 0;
|
||||
}
|
||||
|
||||
.app-table thead th {
|
||||
background: #f8fafc;
|
||||
color: #536174;
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
text-transform: uppercase;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.app-table tbody tr {
|
||||
border-color: #e8eef5;
|
||||
}
|
||||
|
||||
.app-table tbody tr:hover {
|
||||
background: #f7fbff;
|
||||
}
|
||||
|
||||
.task-number {
|
||||
align-items: center;
|
||||
background: #e7f0ff;
|
||||
border: 1px solid #cfe0ff;
|
||||
border-radius: 999px;
|
||||
color: #0b5ed7;
|
||||
display: inline-flex;
|
||||
font-weight: 700;
|
||||
height: 32px;
|
||||
justify-content: center;
|
||||
min-width: 32px;
|
||||
padding: 0 10px;
|
||||
}
|
||||
|
||||
.task-overview {
|
||||
border-top: 3px solid #0d6efd;
|
||||
}
|
||||
|
||||
.hotspot-item {
|
||||
border-color: #e3eaf2;
|
||||
margin-bottom: 10px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.status-panel {
|
||||
@@ -98,4 +194,10 @@ body {
|
||||
.hero-copy h1 {
|
||||
font-size: 36px;
|
||||
}
|
||||
|
||||
.section-heading,
|
||||
.page-heading {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,9 +10,9 @@
|
||||
<script src="/static/app.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="navbar navbar-expand-lg bg-body-tertiary border-bottom">
|
||||
<nav class="navbar navbar-expand-lg app-navbar">
|
||||
<div class="container">
|
||||
<a class="navbar-brand" href="/">热榜评论分析工具</a>
|
||||
<a class="navbar-brand fw-semibold" href="/">热榜评论雷达</a>
|
||||
<div class="navbar-nav">
|
||||
<a class="nav-link" href="/">任务列表</a>
|
||||
</div>
|
||||
|
||||
+16
-11
@@ -8,14 +8,16 @@
|
||||
<h1>热榜评论雷达</h1>
|
||||
<p>从小红书和抖音热点出发,抓取真实评论,生成 AI 情绪、标签和可导出的分析报告。</p>
|
||||
</div>
|
||||
<div class="hero-actions">
|
||||
<a class="btn btn-light" href="#create-task">创建新任务</a>
|
||||
<a class="btn btn-outline-light" href="#task-history">查看 Demo 数据</a>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card mb-4" id="create-task">
|
||||
<div class="card-header">创建抓取任务</div>
|
||||
<section class="tool-card mb-4" id="create-task">
|
||||
<div class="section-heading">
|
||||
<div>
|
||||
<p class="section-kicker">采集配置</p>
|
||||
<h2>创建抓取任务</h2>
|
||||
</div>
|
||||
<span class="text-muted small">默认规模 5 × 5 × 50</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div id="form-error" class="alert alert-danger d-none" role="alert"></div>
|
||||
<form id="task-form" onsubmit="submitTask(event)">
|
||||
@@ -50,13 +52,16 @@
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="card" id="task-history">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<span>最近任务与 Demo 数据</span>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.href='/'">手动刷新</button>
|
||||
<section class="tool-card" id="task-history">
|
||||
<div class="section-heading">
|
||||
<div>
|
||||
<p class="section-kicker">任务历史</p>
|
||||
<h2>最近任务</h2>
|
||||
</div>
|
||||
{% if has_running_tasks %}<span class="badge text-bg-warning">自动更新中</span>{% endif %}
|
||||
</div>
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0">
|
||||
<table class="table app-table align-middle mb-0">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>任务 ID</th>
|
||||
|
||||
@@ -4,10 +4,7 @@
|
||||
{% set ai_percent = rate_percent(task.analysis_success_rate) %}
|
||||
<tr>
|
||||
<td>
|
||||
<strong>{{ task.display_id or loop.index }}</strong>
|
||||
{% if task.is_demo %}
|
||||
<br><span class="badge text-bg-info">Demo 数据</span>
|
||||
{% endif %}
|
||||
<span class="task-number">{{ task.display_id or loop.index }}</span>
|
||||
</td>
|
||||
<td>{{ task.platform | platform_label }}</td>
|
||||
<td>{{ task.created_at_label }}</td>
|
||||
@@ -36,7 +33,7 @@
|
||||
<br><small class="text-danger">{{ task.error_summary }}</small>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td><a class="btn btn-sm btn-outline-primary" href="/tasks/{{ task.id }}">查看</a></td>
|
||||
<td><a class="btn btn-sm btn-primary" href="/tasks/{{ task.id }}">查看</a></td>
|
||||
</tr>
|
||||
{% else %}
|
||||
<tr>
|
||||
|
||||
@@ -7,20 +7,22 @@
|
||||
</ol></nav>
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h1 class="h3 mb-0">任务 {{ task.display_id }}</h1>
|
||||
<button class="btn btn-outline-secondary btn-sm" onclick="window.location.reload()">手动刷新</button>
|
||||
</div>
|
||||
<section class="card mb-4" {% if task.status == "running" %}data-task-id="{{ task.id }}" data-auto-poll="true"{% endif %}><div class="card-body">
|
||||
<div class="page-heading mb-3">
|
||||
<div>
|
||||
<p class="section-kicker">任务详情</p>
|
||||
<h1>任务 {{ task.display_id }}</h1>
|
||||
</div>
|
||||
{% set label, cls = status_badge_config(task.status) %}
|
||||
<span class="badge {{ cls }}">{{ label }}</span>
|
||||
</div>
|
||||
<section class="tool-card task-overview mb-4" {% if task.status == "running" %}data-task-id="{{ task.id }}" data-auto-poll="true"{% endif %}><div class="card-body">
|
||||
{% set percent = progress_percent(task.processed_items_count, task.total_items_count) %}
|
||||
{% set ai_percent = rate_percent(task.analysis_success_rate) %}
|
||||
{% set target_comments = task.hotspot_limit * task.item_limit_per_hotspot * task.comment_limit_per_item %}
|
||||
<div class="d-flex flex-wrap justify-content-between gap-3 mb-3">
|
||||
<p class="mb-0">平台:{{ task.platform | platform_label }} <span class="badge {{ cls }}">{{ label }}</span>{% if task.is_demo %}<span class="badge text-bg-info ms-1">Demo 数据</span>{% endif %}</p>
|
||||
<p class="mb-0 text-muted">阶段:{{ task.current_stage_label or "等待启动" }}</p>
|
||||
<p class="mb-0">平台:{{ task.platform | platform_label }}</p>
|
||||
<p class="mb-0 text-muted">创建时间 {{ task.created_at_label }} · 阶段 {{ task.current_stage_label or "等待启动" }}</p>
|
||||
</div>
|
||||
<div class="small text-muted mb-3">创建时间 {{ task.created_at_label }} · 完整 ID {{ task.id }}</div>
|
||||
<div class="mb-3">
|
||||
<div class="d-flex justify-content-between align-items-center gap-2">
|
||||
<span>已处理 {{ task.processed_items_count }} / 共 {{ task.total_items_count }} 条内容</span>
|
||||
@@ -80,7 +82,7 @@
|
||||
{% endif %}
|
||||
<div class="accordion" id="hotspot-list">
|
||||
{% for hotspot in hotspots %}
|
||||
<div class="accordion-item">
|
||||
<div class="accordion-item hotspot-item">
|
||||
<h2 class="accordion-header">
|
||||
<button class="accordion-button {% if not loop.first %}collapsed{% endif %}" type="button" data-bs-toggle="collapse" data-bs-target="#hotspot-{{ hotspot.id }}">
|
||||
热点 {{ hotspot.rank }}:{{ hotspot.title }}
|
||||
|
||||
@@ -18,7 +18,10 @@ def test_index_page_renders_task_form_empty_state_and_default_scale():
|
||||
assert 'name="item_limit_per_hotspot"' in response.text
|
||||
assert 'name="comment_limit_per_item"' in response.text
|
||||
assert "1250" in response.text
|
||||
assert 'onclick="window.location.href=\'/\'"' in response.text
|
||||
assert "手动刷新" not in response.text
|
||||
assert 'onclick="window.location.href=\'/\'"' not in response.text
|
||||
assert 'href="#create-task"' not in response.text
|
||||
assert 'href="#task-history"' not in response.text
|
||||
assert "还没有任何任务" in response.text
|
||||
|
||||
|
||||
@@ -104,10 +107,13 @@ def test_index_page_uses_short_task_numbers_compact_fields_and_friendly_error_co
|
||||
assert "#" not in visible_text
|
||||
assert first_id not in visible_text
|
||||
assert second_id not in visible_text
|
||||
assert "2026-07-03 10:30" in visible_text
|
||||
assert "2026-07-03 10:35" in visible_text
|
||||
assert "小红书 · 5热点 × 5内容 × 50评论" in visible_text
|
||||
assert "抖音 · 1热点 × 1内容 × 10评论" in visible_text
|
||||
assert "2026-07-03 18:30" in visible_text
|
||||
assert "2026-07-03 18:35" in visible_text
|
||||
assert "Demo 数据" not in visible_text
|
||||
assert "5热点 × 5内容 × 50评论" in visible_text
|
||||
assert "1热点 × 1内容 × 10评论" in visible_text
|
||||
assert "小红书 · 5热点" not in visible_text
|
||||
assert "抖音 · 1热点" not in visible_text
|
||||
assert "系统重启,任务被中断" in visible_text
|
||||
assert "system / unexpected_restart" not in visible_text
|
||||
assert "unexpected_restart" not in visible_text
|
||||
@@ -167,12 +173,13 @@ def test_running_task_detail_page_auto_polls_current_task():
|
||||
assert response.status_code == 200
|
||||
assert 'data-task-id="task-running"' in response.text
|
||||
assert "pollTaskDetailStatus" in response.text
|
||||
assert 'onclick="window.location.reload()"' in response.text
|
||||
assert "手动刷新" not in response.text
|
||||
assert 'onclick="window.location.reload()"' not in response.text
|
||||
assert "已处理 1 / 共 2 条内容" in response.text
|
||||
assert "AI 成功率 100%" in response.text
|
||||
|
||||
|
||||
def test_task_detail_uses_short_number_title_and_keeps_full_uuid_for_diagnostics():
|
||||
def test_task_detail_uses_short_number_title_and_hides_full_uuid():
|
||||
task_id = "33333333-3333-4333-8333-333333333333"
|
||||
with make_test_client() as (client, engine):
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -197,11 +204,42 @@ def test_task_detail_uses_short_number_title_and_keeps_full_uuid_for_diagnostics
|
||||
visible_text = BeautifulSoup(response.text, "html.parser").get_text(" ")
|
||||
assert "<title>任务 1 - 热榜评论分析工具</title>" in response.text
|
||||
assert "任务 1" in visible_text
|
||||
assert "完整 ID" in visible_text
|
||||
assert task_id in visible_text
|
||||
assert "完整 ID" not in visible_text
|
||||
assert task_id not in visible_text
|
||||
assert "任务 #33333333-3333-4333-8333-333333333333" not in visible_text
|
||||
assert "#" not in visible_text
|
||||
assert "创建时间 2026-07-03 10:30" in visible_text
|
||||
assert "创建时间 2026-07-03 18:30" in visible_text
|
||||
assert "Demo 数据" not in visible_text
|
||||
|
||||
|
||||
def test_demo_task_flag_is_kept_but_demo_copy_is_hidden_from_pages():
|
||||
with make_test_client() as (client, engine):
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
with Session(engine) as session:
|
||||
session.add(
|
||||
Task(
|
||||
id="demo-task-hidden-copy",
|
||||
platform="xiaohongshu",
|
||||
status="success",
|
||||
created_at=datetime(2026, 7, 3, 10, 30, tzinfo=UTC),
|
||||
hotspot_limit=1,
|
||||
item_limit_per_hotspot=1,
|
||||
comment_limit_per_item=10,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
index_response = client.get("/")
|
||||
detail_response = client.get("/tasks/demo-task-hidden-copy")
|
||||
api_response = client.get("/api/tasks/demo-task-hidden-copy")
|
||||
|
||||
assert index_response.status_code == 200
|
||||
assert detail_response.status_code == 200
|
||||
assert api_response.status_code == 200
|
||||
assert api_response.json()["is_demo"] is True
|
||||
assert "Demo 数据" not in BeautifulSoup(index_response.text, "html.parser").get_text(" ")
|
||||
assert "Demo 数据" not in BeautifulSoup(detail_response.text, "html.parser").get_text(" ")
|
||||
|
||||
|
||||
def test_running_task_detail_page_keeps_polling_after_hotspots_exist():
|
||||
@@ -296,7 +334,7 @@ def test_running_task_detail_page_shows_stage_runtime_and_stale_progress_warning
|
||||
response = client.get("/tasks/task-stale")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "阶段:AI 分析中" in response.text
|
||||
assert "阶段 AI 分析中" in response.text
|
||||
assert "运行时长" in response.text
|
||||
assert "2小时5分钟" in response.text
|
||||
assert "最近进度" in response.text
|
||||
@@ -419,7 +457,8 @@ def test_result_pages_render_seeded_data():
|
||||
assert task_response.status_code == 200
|
||||
assert "热点标题" in task_response.text
|
||||
assert "热点 1:热点标题" in BeautifulSoup(task_response.text, "html.parser").get_text(" ")
|
||||
assert 'onclick="window.location.reload()"' in task_response.text
|
||||
assert 'onclick="window.location.reload()"' not in task_response.text
|
||||
assert "手动刷新" not in task_response.text
|
||||
assert hotspot_response.status_code == 200
|
||||
assert "热点总结" in hotspot_response.text
|
||||
assert "downloadExport(" not in hotspot_response.text
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import pytest
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app.db import check_database_integrity, checkpoint_sqlite_wal, create_sqlite_engine, ensure_sqlite_schema_compat
|
||||
|
||||
@@ -11,6 +12,14 @@ def test_check_database_integrity_returns_ok_for_valid_sqlite_database():
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_file_sqlite_engine_does_not_reuse_connections_after_operational_errors(tmp_path):
|
||||
engine = create_sqlite_engine(f"sqlite:///{tmp_path / 'app.db'}")
|
||||
try:
|
||||
assert isinstance(engine.pool, NullPool)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_check_database_integrity_raises_when_sqlite_reports_problem(monkeypatch):
|
||||
class FakeCursor:
|
||||
def execute(self, _sql):
|
||||
|
||||
Reference in New Issue
Block a user