intelligrow/kol-insight
15
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(backend): 视频分析模块增加缓存优先策略和并发API调用

Browse Source 
- SessionPool 新增 get_distinct_configs 方法，支持获取不同配置用于并发调用
- video_analysis 重构为缓存优先策略：数据库有 A3/Cost 数据时直接使用
- 并发 API 调用预分配不同 cookie，避免 session 冲突
- API 数据写回数据库，实现下次查询缓存命中
- 新增 heated_cost 字段追踪
- 测试全面重写，覆盖缓存/API/混合/降级场景
This commit is contained in:
zfc
2026-01-29 18:21:50 +08:00
parent c53b5008df
commit 376f0be6b4
4 changed files with 983 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/session_pool.py
+67 -1
View File
@@ -11,8 +11,8 @@ T-027 修复:
import asyncio
import logging
import random
from typing import Dict, Optional, Any, List
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
import httpx
@@ -150,6 +150,46 @@ class SessionPool:
"""检查池是否为空"""
return len(self._configs) == 0
def get_distinct_configs(self, count: int) -> List[Dict[str, Any]]:
"""
获取 count 个不同的配置，用于并发调用。
- 池中配置 >= count：随机抽样 count 个不重复的
- 池中配置 < count：全部取出，循环复用补足
- 池为空：返回空列表
Args:
count: 需要的配置数量
Returns:
List[Dict]: 配置字典列表
"""
if not self._configs or count <= 0:
return []
def _to_dict(config: CookieConfig) -> Dict[str, Any]:
return {
"brand_id": config.brand_id,
"aadvid": config.aadvid,
"auth_token": config.auth_token,
"industry_id": config.industry_id,
"brand_name": config.brand_name,
}
if len(self._configs) >= count:
sampled = random.sample(self._configs, count)
return [_to_dict(c) for c in sampled]
# 池中配置不足，全部取出后循环复用
result = [_to_dict(c) for c in self._configs]
shuffled = list(self._configs)
random.shuffle(shuffled)
idx = 0
while len(result) < count:
result.append(_to_dict(shuffled[idx % len(shuffled)]))
idx += 1
return result
# 兼容旧接口
def get_random(self) -> Optional[str]:
"""兼容旧接口：随机获取一个 SessionID"""
@@ -218,6 +258,32 @@ async def get_session_with_retry(max_retries: int = 3) -> Optional[str]:
return None
async def get_distinct_configs(count: int, max_retries: int = 3) -> List[Dict[str, Any]]:
"""
获取 count 个不同的配置，必要时刷新池。
Args:
count: 需要的配置数量
max_retries: 最大重试次数
Returns:
List[Dict]: 配置字典列表
"""
for attempt in range(max_retries):
if session_pool.is_empty:
success = await session_pool.refresh()
if not success:
logger.warning(f"Session pool refresh failed, attempt {attempt + 1}")
continue
configs = session_pool.get_distinct_configs(count)
if configs:
return configs
logger.error("Failed to get distinct configs after all retries")
return []
async def get_config_for_brand(brand_id: str, max_retries: int = 3) -> Optional[Any]:
"""
兼容旧接口：获取品牌对应的配置。
backend/app/services/video_analysis.py
+262 -125
View File
@@ -4,25 +4,43 @@
实现视频分析数据获取和成本指标计算。
"""
import asyncio
import logging
from datetime import datetime
from typing import Dict, Optional, Any
from typing import Any, Dict, Optional
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from sqlalchemy import update
from app.models.kol_video import KolVideo
from app.services.session_pool import (
get_distinct_configs,
get_random_config,
session_pool,
)
from app.services.yuntu_api import (
SessionInvalidError,
call_yuntu_api,
parse_analysis_response,
)
from app.services.yuntu_api import (
get_video_analysis as fetch_yuntu_analysis,
parse_analysis_response,
YuntuAPIError,
)
logger = logging.getLogger(__name__)
def _needs_api_call(video: KolVideo) -> bool:
"""
判断是否需要调用 Yuntu API 获取 A3/Cost 数据。
如果数据库中已有 A3 或 Cost 数据，直接使用数据库数据，不调 API。
"""
has_a3 = (video.total_new_a3_cnt or 0) > 0
has_cost = (video.total_cost or 0) > 0
return not (has_a3 or has_cost)
def calculate_cost_metrics(
cost: float,
natural_play_cnt: int,
@@ -151,33 +169,58 @@ async def get_video_analysis_data(
brand_map = await get_brand_names([video.brand_id])
brand_name = brand_map.get(video.brand_id, video.brand_id)
# 3. 调用巨量云图API获取实时 A3 数据和 cost
# 3. 获取 A3 数据和 cost（缓存优先策略）
a3_increase_cnt = 0
ad_a3_increase_cnt = 0
natural_a3_increase_cnt = 0
api_cost = 0.0
ad_cost = 0.0
try:
publish_time = video.publish_time or datetime.now()
industry_id = video.industry_id or ""
api_response = await fetch_yuntu_analysis(
item_id=item_id,
publish_time=publish_time,
industry_id=industry_id,
)
analysis_data = parse_analysis_response(api_response)
a3_increase_cnt = analysis_data.get("a3_increase_cnt", 0)
ad_a3_increase_cnt = analysis_data.get("ad_a3_increase_cnt", 0)
natural_a3_increase_cnt = analysis_data.get("natural_a3_increase_cnt", 0)
api_cost = analysis_data.get("cost", 0)
except Exception as e:
logger.warning(f"API failed for {item_id}: {e}, using DB data")
if not _needs_api_call(video):
# 数据库已有数据，直接使用
logger.info(f"Using DB data for {item_id} (A3/Cost already cached)")
a3_increase_cnt = video.total_new_a3_cnt or 0
ad_a3_increase_cnt = video.heated_new_a3_cnt or 0
natural_a3_increase_cnt = video.natural_new_a3_cnt or 0
api_cost = video.total_cost or 0.0
ad_cost = video.heated_cost or 0.0
else:
# 需要调用 API 获取数据
try:
publish_time = video.publish_time or datetime.now()
industry_id = video.industry_id or ""
api_response = await fetch_yuntu_analysis(
item_id=item_id,
publish_time=publish_time,
industry_id=industry_id,
)
analysis_data = parse_analysis_response(api_response)
a3_increase_cnt = analysis_data.get("a3_increase_cnt", 0)
ad_a3_increase_cnt = analysis_data.get("ad_a3_increase_cnt", 0)
natural_a3_increase_cnt = analysis_data.get("natural_a3_increase_cnt", 0)
api_cost = analysis_data.get("cost", 0)
ad_cost = analysis_data.get("ad_cost", 0)
# 写回数据库
await update_video_a3_metrics(
session=session,
item_id=item_id,
total_new_a3_cnt=int(a3_increase_cnt),
heated_new_a3_cnt=int(ad_a3_increase_cnt),
natural_new_a3_cnt=int(natural_a3_increase_cnt),
total_cost=float(api_cost),
heated_cost=float(ad_cost),
)
logger.info(f"API data fetched and saved to DB for {item_id}")
except Exception as e:
logger.warning(f"API failed for {item_id}: {e}, using DB data")
a3_increase_cnt = video.total_new_a3_cnt or 0
ad_a3_increase_cnt = video.heated_new_a3_cnt or 0
natural_a3_increase_cnt = video.natural_new_a3_cnt or 0
api_cost = video.total_cost or 0.0
ad_cost = video.heated_cost or 0.0
# 4. 数据库字段
estimated_video_cost = video.estimated_video_cost or 0.0
@@ -187,8 +230,7 @@ async def get_video_analysis_data(
after_view_search_uv = video.after_view_search_uv or 0
# 5. 计算成本指标
# 预估加热费用 = max(total_cost - estimated_video_cost, 0)
heated_cost = max(api_cost - estimated_video_cost, 0) if api_cost > estimated_video_cost else 0
heated_cost = ad_cost
# 预估自然看后搜人数
estimated_natural_search_uv = None
@@ -271,9 +313,10 @@ async def update_video_a3_metrics(
heated_new_a3_cnt: int,
natural_new_a3_cnt: int,
total_cost: float,
heated_cost: float = 0.0,
) -> bool:
"""
更新数据库中的A3指标 (T-025)。
更新数据库中的A3指标和费用数据 (T-025)。
Args:
session: 数据库会话
@@ -281,7 +324,8 @@ async def update_video_a3_metrics(
total_new_a3_cnt: 总新增A3
heated_new_a3_cnt: 加热新增A3
natural_new_a3_cnt: 自然新增A3
total_cost: 总花费
total_cost: 预估总费用
heated_cost: 预估加热费用
Returns:
bool: 更新是否成功
@@ -295,6 +339,7 @@ async def update_video_a3_metrics(
heated_new_a3_cnt=heated_new_a3_cnt,
natural_new_a3_cnt=natural_new_a3_cnt,
total_cost=total_cost,
heated_cost=heated_cost,
)
)
result = await session.execute(stmt)
@@ -313,39 +358,6 @@ async def update_video_a3_metrics(
return False
async def get_and_update_video_analysis(
session: AsyncSession, item_id: str
) -> Dict[str, Any]:
"""
获取视频分析数据并更新数据库中的A3指标 (T-024 + T-025 组合)。
Args:
session: 数据库会话
item_id: 视频ID
Returns:
Dict: 完整的视频分析数据
"""
# 获取分析数据
result = await get_video_analysis_data(session, item_id)
# 提取A3指标
a3_metrics = result.get("a3_metrics", {})
cost_raw = result.get("cost_metrics_raw", {})
# 更新数据库
await update_video_a3_metrics(
session=session,
item_id=item_id,
total_new_a3_cnt=a3_metrics.get("a3_increase_cnt", 0),
heated_new_a3_cnt=a3_metrics.get("ad_a3_increase_cnt", 0),
natural_new_a3_cnt=a3_metrics.get("natural_a3_increase_cnt", 0),
total_cost=cost_raw.get("cost", 0),
)
return result
async def search_videos_by_star_id(
session: AsyncSession, star_id: str
) -> list[KolVideo]:
@@ -373,11 +385,60 @@ async def search_videos_by_nickname(
return list(result.scalars().all())
def _build_video_list_item(
video: KolVideo,
a3_increase_cnt: int,
ad_a3_increase_cnt: int,
natural_a3_increase_cnt: int,
api_cost: float,
brand_name: str,
) -> Dict[str, Any]:
"""构建视频列表项的结果字典。"""
estimated_video_cost = video.estimated_video_cost or 0.0
natural_play_cnt = video.natural_play_cnt or 0
total_play_cnt = video.total_play_cnt or 0
after_view_search_uv = video.after_view_search_uv or 0
estimated_natural_search_uv = None
if total_play_cnt > 0 and after_view_search_uv > 0:
estimated_natural_search_uv = (natural_play_cnt / total_play_cnt) * after_view_search_uv
estimated_natural_cpm = round((estimated_video_cost / natural_play_cnt) * 1000, 2) if natural_play_cnt > 0 else None
estimated_cp_a3 = round(api_cost / a3_increase_cnt, 2) if a3_increase_cnt > 0 else None
estimated_natural_cp_a3 = round(estimated_video_cost / natural_a3_increase_cnt, 2) if natural_a3_increase_cnt > 0 else None
estimated_cp_search = round(api_cost / after_view_search_uv, 2) if after_view_search_uv > 0 else None
estimated_natural_cp_search = round(estimated_video_cost / estimated_natural_search_uv, 2) if estimated_natural_search_uv and estimated_natural_search_uv > 0 else None
return {
"item_id": video.item_id,
"star_nickname": video.star_nickname or "",
"title": video.title or "",
"video_url": video.video_url or "",
"create_date": video.publish_time.isoformat() if video.publish_time else None,
"hot_type": video.viral_type or "",
"industry_id": video.industry_id or "",
"brand_id": video.brand_id or "",
"brand_name": brand_name,
"total_new_a3_cnt": a3_increase_cnt,
"heated_new_a3_cnt": ad_a3_increase_cnt,
"natural_new_a3_cnt": natural_a3_increase_cnt,
"estimated_natural_cpm": estimated_natural_cpm,
"estimated_cp_a3": estimated_cp_a3,
"estimated_natural_cp_a3": estimated_natural_cp_a3,
"estimated_cp_search": estimated_cp_search,
"estimated_natural_cp_search": estimated_natural_cp_search,
}
async def get_video_list_with_a3(
session: AsyncSession, videos: list[KolVideo]
) -> list[Dict[str, Any]]:
"""
获取视频列表的摘要数据（实时调用云图API获取A3数据）。
获取视频列表的摘要数据。
缓存优先策略：
- 数据库有 A3/Cost 数据 → 直接使用
- 数据库无数据 → 并发调用云图 API（预分配不同 cookie）→ 写回数据库
"""
from app.services.brand_api import get_brand_names
@@ -385,73 +446,149 @@ async def get_video_list_with_a3(
brand_ids = [video.brand_id for video in videos if video.brand_id]
brand_map = await get_brand_names(brand_ids) if brand_ids else {}
result = []
for video in videos:
# 实时调用云图 API 获取 A3 数据和 cost
a3_increase_cnt = 0
ad_a3_increase_cnt = 0
natural_a3_increase_cnt = 0
api_cost = 0.0
# 分组：已有数据 vs 需要 API 调用
cached_videos: list[tuple[int, KolVideo]] = [] # (原始索引, video)
api_videos: list[tuple[int, KolVideo]] = [] # (原始索引, video)
try:
publish_time = video.publish_time or datetime.now()
industry_id = video.industry_id or ""
for idx, video in enumerate(videos):
if _needs_api_call(video):
api_videos.append((idx, video))
else:
cached_videos.append((idx, video))
api_response = await fetch_yuntu_analysis(
item_id=video.item_id,
publish_time=publish_time,
industry_id=industry_id,
logger.info(
f"Video list: {len(cached_videos)} cached, {len(api_videos)} need API"
)
# 结果数组（按原始索引填充）
results: list[Optional[Dict[str, Any]]] = [None] * len(videos)
# 组 A：直接用数据库数据
for idx, video in cached_videos:
brand_name = brand_map.get(video.brand_id, video.brand_id or "") if video.brand_id else ""
results[idx] = _build_video_list_item(
video=video,
a3_increase_cnt=video.total_new_a3_cnt or 0,
ad_a3_increase_cnt=video.heated_new_a3_cnt or 0,
natural_a3_increase_cnt=video.natural_new_a3_cnt or 0,
api_cost=video.total_cost or 0.0,
brand_name=brand_name,
)
# 组 B：并发调用 API（预分配不同 cookie）
if api_videos:
configs = await get_distinct_configs(len(api_videos))
semaphore = asyncio.Semaphore(5)
# 收集需要写回 DB 的数据（避免并发 session 操作）
pending_updates: list[Dict[str, Any]] = []
async def _fetch_single(
idx: int, video: KolVideo, config: Dict[str, Any]
) -> None:
a3_increase_cnt = 0
ad_a3_increase_cnt = 0
natural_a3_increase_cnt = 0
api_cost = 0.0
ad_cost_val = 0.0
api_success = False
async with semaphore:
try:
publish_time = video.publish_time or datetime.now()
industry_id = video.industry_id or ""
api_response = await call_yuntu_api(
item_id=video.item_id,
publish_time=publish_time,
industry_id=industry_id,
aadvid=config["aadvid"],
auth_token=config["auth_token"],
)
api_data = parse_analysis_response(api_response)
a3_increase_cnt = api_data.get("a3_increase_cnt", 0)
ad_a3_increase_cnt = api_data.get("ad_a3_increase_cnt", 0)
natural_a3_increase_cnt = api_data.get("natural_a3_increase_cnt", 0)
api_cost = api_data.get("cost", 0)
ad_cost_val = api_data.get("ad_cost", 0)
api_success = True
except SessionInvalidError:
# Session 失效，从池中移除，重新获取随机 config 重试
session_pool.remove_by_auth_token(config["auth_token"])
logger.warning(f"Session invalid for {video.item_id}, retrying")
retry_config = await get_random_config()
if retry_config:
try:
publish_time = video.publish_time or datetime.now()
industry_id = video.industry_id or ""
api_response = await call_yuntu_api(
item_id=video.item_id,
publish_time=publish_time,
industry_id=industry_id,
aadvid=retry_config["aadvid"],
auth_token=retry_config["auth_token"],
)
api_data = parse_analysis_response(api_response)
a3_increase_cnt = api_data.get("a3_increase_cnt", 0)
ad_a3_increase_cnt = api_data.get("ad_a3_increase_cnt", 0)
natural_a3_increase_cnt = api_data.get("natural_a3_increase_cnt", 0)
api_cost = api_data.get("cost", 0)
ad_cost_val = api_data.get("ad_cost", 0)
api_success = True
except Exception as e2:
logger.warning(f"Retry failed for {video.item_id}: {e2}")
a3_increase_cnt = video.total_new_a3_cnt or 0
ad_a3_increase_cnt = video.heated_new_a3_cnt or 0
natural_a3_increase_cnt = video.natural_new_a3_cnt or 0
api_cost = video.total_cost or 0.0
except Exception as e:
logger.warning(f"API failed for {video.item_id}: {e}")
a3_increase_cnt = video.total_new_a3_cnt or 0
ad_a3_increase_cnt = video.heated_new_a3_cnt or 0
natural_a3_increase_cnt = video.natural_new_a3_cnt or 0
api_cost = video.total_cost or 0.0
# 收集待写回 DB 的数据（不在并发中操作 session）
if api_success:
pending_updates.append({
"item_id": video.item_id,
"total_new_a3_cnt": int(a3_increase_cnt),
"heated_new_a3_cnt": int(ad_a3_increase_cnt),
"natural_new_a3_cnt": int(natural_a3_increase_cnt),
"total_cost": float(api_cost),
"heated_cost": float(ad_cost_val),
})
brand_name = brand_map.get(video.brand_id, video.brand_id or "") if video.brand_id else ""
results[idx] = _build_video_list_item(
video=video,
a3_increase_cnt=a3_increase_cnt,
ad_a3_increase_cnt=ad_a3_increase_cnt,
natural_a3_increase_cnt=natural_a3_increase_cnt,
api_cost=api_cost,
brand_name=brand_name,
)
api_data = parse_analysis_response(api_response)
a3_increase_cnt = api_data.get("a3_increase_cnt", 0)
ad_a3_increase_cnt = api_data.get("ad_a3_increase_cnt", 0)
natural_a3_increase_cnt = api_data.get("natural_a3_increase_cnt", 0)
api_cost = api_data.get("cost", 0)
except Exception as e:
logger.warning(f"API failed for {video.item_id}: {e}")
a3_increase_cnt = video.total_new_a3_cnt or 0
ad_a3_increase_cnt = video.heated_new_a3_cnt or 0
natural_a3_increase_cnt = video.natural_new_a3_cnt or 0
api_cost = video.total_cost or 0.0
# 为每个视频分配一个独立的 config，并发执行
tasks = []
for i, (idx, video) in enumerate(api_videos):
config = configs[i] if i < len(configs) else configs[i % len(configs)] if configs else {}
tasks.append(_fetch_single(idx, video, config))
# 数据库字段
estimated_video_cost = video.estimated_video_cost or 0.0
natural_play_cnt = video.natural_play_cnt or 0
total_play_cnt = video.total_play_cnt or 0
after_view_search_uv = video.after_view_search_uv or 0
await asyncio.gather(*tasks)
# 计算成本指标
estimated_natural_search_uv = None
if total_play_cnt > 0 and after_view_search_uv > 0:
estimated_natural_search_uv = (natural_play_cnt / total_play_cnt) * after_view_search_uv
# 顺序写回 DB（避免并发 session 操作导致状态损坏）
for upd in pending_updates:
await update_video_a3_metrics(
session=session,
item_id=upd["item_id"],
total_new_a3_cnt=upd["total_new_a3_cnt"],
heated_new_a3_cnt=upd["heated_new_a3_cnt"],
natural_new_a3_cnt=upd["natural_new_a3_cnt"],
total_cost=upd["total_cost"],
heated_cost=upd["heated_cost"],
)
estimated_natural_cpm = round((estimated_video_cost / natural_play_cnt) * 1000, 2) if natural_play_cnt > 0 else None
estimated_cp_a3 = round(api_cost / a3_increase_cnt, 2) if a3_increase_cnt > 0 else None
estimated_natural_cp_a3 = round(estimated_video_cost / natural_a3_increase_cnt, 2) if natural_a3_increase_cnt > 0 else None
estimated_cp_search = round(api_cost / after_view_search_uv, 2) if after_view_search_uv > 0 else None
estimated_natural_cp_search = round(estimated_video_cost / estimated_natural_search_uv, 2) if estimated_natural_search_uv and estimated_natural_search_uv > 0 else None
brand_name = brand_map.get(video.brand_id, video.brand_id) if video.brand_id else ""
result.append({
"item_id": video.item_id,
"star_nickname": video.star_nickname or "",
"title": video.title or "",
"video_url": video.video_url or "",
"create_date": video.publish_time.isoformat() if video.publish_time else None,
"hot_type": video.viral_type or "",
"industry_id": video.industry_id or "",
"brand_id": video.brand_id or "",
"brand_name": brand_name,
"total_new_a3_cnt": a3_increase_cnt,
"heated_new_a3_cnt": ad_a3_increase_cnt,
"natural_new_a3_cnt": natural_a3_increase_cnt,
"estimated_natural_cpm": estimated_natural_cpm,
"estimated_cp_a3": estimated_cp_a3,
"estimated_natural_cp_a3": estimated_natural_cp_a3,
"estimated_cp_search": estimated_cp_search,
"estimated_natural_cp_search": estimated_natural_cp_search,
})
return result
# 过滤 None（不应发生，防御性编程）
return [r for r in results if r is not None]