feat: 提交热榜评论分析工具 MVP 基线
This commit is contained in:
meijiali
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass
|
||||
class HotspotData:
|
||||
source_hot_id: str | None
|
||||
title: str
|
||||
rank: int | None = None
|
||||
heat_value: str | None = None
|
||||
raw_data: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContentItemData:
|
||||
source_item_id: str
|
||||
item_type: str
|
||||
title: str | None = None
|
||||
summary: str | None = None
|
||||
url: str | None = None
|
||||
raw_data: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommentData:
|
||||
source_comment_id: str | None
|
||||
content: str
|
||||
author: str | None = None
|
||||
like_count: int | None = None
|
||||
comment_time: datetime | None = None
|
||||
raw_data: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class PlatformAPIError(Exception):
|
||||
def __init__(self, message: str, *, error_type: str, status_code: int | None = None):
|
||||
super().__init__(message)
|
||||
self.error_type = error_type
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class TikHubClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
timeout_seconds: int = 20,
|
||||
max_retries: int = 3,
|
||||
http_client: httpx.Client | None = None,
|
||||
) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.max_retries = max_retries
|
||||
self._http_client = http_client or httpx.Client(timeout=timeout_seconds)
|
||||
|
||||
def get(self, path: str, *, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return self._request("GET", path, params=params)
|
||||
|
||||
def post(self, path: str, *, json: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return self._request("POST", path, json=json)
|
||||
|
||||
def _request(self, method: str, path: str, **kwargs) -> dict[str, Any]:
|
||||
url = f"{self.base_url}{path}"
|
||||
headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
|
||||
last_status: int | None = None
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
response = self._http_client.request(method, url, headers=headers, **kwargs)
|
||||
except httpx.RequestError as exc:
|
||||
if attempt >= self.max_retries:
|
||||
raise PlatformAPIError("External API request failed", error_type="network_error") from exc
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
|
||||
last_status = response.status_code
|
||||
if response.status_code == 429:
|
||||
if attempt >= self.max_retries:
|
||||
raise PlatformAPIError(
|
||||
"External API rate limited",
|
||||
error_type="rate_limited",
|
||||
status_code=response.status_code,
|
||||
)
|
||||
time.sleep(2**attempt)
|
||||
continue
|
||||
if response.is_error:
|
||||
raise PlatformAPIError(
|
||||
f"External API returned HTTP {response.status_code}",
|
||||
error_type="api_error",
|
||||
status_code=response.status_code,
|
||||
)
|
||||
return response.json()
|
||||
|
||||
raise PlatformAPIError("External API request failed", error_type="api_error", status_code=last_status)
|
||||
|
||||
|
||||
def parse_timestamp(value: Any) -> datetime | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
number = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if number > 10_000_000_000:
|
||||
number = number // 1000
|
||||
return datetime.fromtimestamp(number, tz=UTC)
|
||||
|
||||
|
||||
def first_present(data: dict[str, Any], *keys: str) -> Any:
|
||||
for key in keys:
|
||||
if key in data and data[key] not in (None, ""):
|
||||
return data[key]
|
||||
return None
|
||||
@@ -0,0 +1,99 @@
|
||||
from typing import Any
|
||||
|
||||
from app.platforms.base import CommentData, ContentItemData, HotspotData, TikHubClient, first_present, parse_timestamp
|
||||
|
||||
|
||||
class DouyinPlatform:
|
||||
def __init__(self, client: TikHubClient | None) -> None:
|
||||
self.client = client
|
||||
|
||||
def map_hotspots(self, payload: dict[str, Any], *, limit: int) -> list[HotspotData]:
|
||||
data = payload.get("data", {})
|
||||
items = data.get("word_list") or data.get("list") or data.get("item_list") or []
|
||||
result = []
|
||||
for index, item in enumerate(items[:limit], start=1):
|
||||
result.append(
|
||||
HotspotData(
|
||||
source_hot_id=str(item.get("query_id")) if item.get("query_id") is not None else None,
|
||||
title=str(first_present(item, "title", "sentence", "word") or ""),
|
||||
rank=item.get("rank") or index,
|
||||
heat_value=str(item.get("hot_score")) if item.get("hot_score") is not None else None,
|
||||
raw_data=item,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def map_items(self, payload: dict[str, Any], *, limit: int) -> list[ContentItemData]:
|
||||
data = payload.get("data", [])
|
||||
raw_items = data.get("business_data") or data.get("data") or data.get("items") or [] if isinstance(data, dict) else data
|
||||
result = []
|
||||
for item in raw_items[:limit]:
|
||||
nested = item.get("data", item) if isinstance(item, dict) else {}
|
||||
aweme = nested.get("aweme_info", nested) if isinstance(nested, dict) else {}
|
||||
aweme_id = aweme.get("aweme_id")
|
||||
if not aweme_id:
|
||||
continue
|
||||
result.append(
|
||||
ContentItemData(
|
||||
source_item_id=str(aweme_id),
|
||||
item_type="video",
|
||||
title=aweme.get("desc"),
|
||||
summary=aweme.get("desc"),
|
||||
url=aweme.get("share_url"),
|
||||
raw_data=aweme,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def map_comments(self, payload: dict[str, Any], *, limit: int) -> list[CommentData]:
|
||||
comments = payload.get("comments") or payload.get("data", {}).get("comments") or []
|
||||
result = []
|
||||
for comment in comments[:limit]:
|
||||
content = comment.get("text")
|
||||
if not content:
|
||||
continue
|
||||
result.append(
|
||||
CommentData(
|
||||
source_comment_id=first_present(comment, "comment_id", "cid"),
|
||||
content=str(content),
|
||||
author=self._author_name(comment),
|
||||
like_count=comment.get("digg_count"),
|
||||
comment_time=parse_timestamp(first_present(comment, "create_time", "create_time_str")),
|
||||
raw_data=comment,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def fetch_hotspots(self, *, limit: int) -> list[HotspotData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/douyin/creator/fetch_creator_hot_spot_billboard",
|
||||
params={"billboard_tag": 0, "hot_search_type": 1},
|
||||
)
|
||||
return self.map_hotspots(payload, limit=limit)
|
||||
|
||||
def search_items_by_hotspot(self, keyword: str, *, limit: int) -> list[ContentItemData]:
|
||||
payload = self.client.post(
|
||||
"/api/v1/douyin/search/fetch_video_search_v2",
|
||||
json={
|
||||
"keyword": keyword,
|
||||
"cursor": 0,
|
||||
"sort_type": "0",
|
||||
"publish_time": "0",
|
||||
"filter_duration": "0",
|
||||
"content_type": "1",
|
||||
"search_id": "",
|
||||
"backtrace": "",
|
||||
},
|
||||
)
|
||||
return self.map_items(payload, limit=limit)
|
||||
|
||||
def fetch_comments(self, source_item_id: str, *, limit: int) -> list[CommentData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/douyin/app/v3/fetch_video_comments",
|
||||
params={"aweme_id": source_item_id, "cursor": 0, "count": 20},
|
||||
)
|
||||
return self.map_comments(payload, limit=limit)
|
||||
|
||||
def _author_name(self, comment: dict[str, Any]) -> str | None:
|
||||
user = comment.get("user") or {}
|
||||
return user.get("nickname") or user.get("name") if isinstance(user, dict) else None
|
||||
@@ -0,0 +1,84 @@
|
||||
from typing import Any
|
||||
|
||||
from app.platforms.base import CommentData, ContentItemData, HotspotData, TikHubClient, first_present, parse_timestamp
|
||||
|
||||
|
||||
class XiaohongshuPlatform:
|
||||
def __init__(self, client: TikHubClient | None) -> None:
|
||||
self.client = client
|
||||
|
||||
def map_hotspots(self, payload: dict[str, Any], *, limit: int) -> list[HotspotData]:
|
||||
items = payload.get("data", {}).get("data", {}).get("items", [])
|
||||
hotspots = []
|
||||
for index, item in enumerate(items[:limit], start=1):
|
||||
hotspots.append(
|
||||
HotspotData(
|
||||
source_hot_id=str(item.get("id")) if item.get("id") is not None else None,
|
||||
title=str(item.get("title") or ""),
|
||||
rank=index,
|
||||
heat_value=str(item.get("score")) if item.get("score") is not None else None,
|
||||
raw_data=item,
|
||||
)
|
||||
)
|
||||
return hotspots
|
||||
|
||||
def map_items(self, payload: dict[str, Any], *, limit: int) -> list[ContentItemData]:
|
||||
raw_items = payload.get("data", {}).get("data", {}).get("items", [])
|
||||
notes = [item.get("note", item) for item in raw_items]
|
||||
notes.sort(key=lambda note: 0 if int(note.get("comments_count") or 0) > 0 else 1)
|
||||
result = []
|
||||
for note in notes[:limit]:
|
||||
note_id = note.get("id")
|
||||
if not note_id:
|
||||
continue
|
||||
result.append(
|
||||
ContentItemData(
|
||||
source_item_id=str(note_id),
|
||||
item_type="note",
|
||||
title=note.get("title"),
|
||||
summary=note.get("desc"),
|
||||
url=note.get("url"),
|
||||
raw_data=note,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def map_comments(self, payload: dict[str, Any], *, limit: int) -> list[CommentData]:
|
||||
comments = payload.get("data", {}).get("data", {}).get("comments", [])
|
||||
result = []
|
||||
for comment in comments[:limit]:
|
||||
content = first_present(comment, "content", "text")
|
||||
if not content:
|
||||
continue
|
||||
result.append(
|
||||
CommentData(
|
||||
source_comment_id=first_present(comment, "comment_id", "id"),
|
||||
content=str(content),
|
||||
author=self._author_name(comment),
|
||||
like_count=comment.get("like_count"),
|
||||
comment_time=parse_timestamp(first_present(comment, "create_time", "create_time_str")),
|
||||
raw_data=comment,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def fetch_hotspots(self, *, limit: int) -> list[HotspotData]:
|
||||
return self.map_hotspots(self.client.get("/api/v1/xiaohongshu/web_v2/fetch_hot_list"), limit=limit)
|
||||
|
||||
def search_items_by_hotspot(self, keyword: str, *, limit: int) -> list[ContentItemData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/xiaohongshu/app_v2/search_notes",
|
||||
params={"keyword": keyword, "page": 1, "sort": "general", "note_type": 0},
|
||||
)
|
||||
return self.map_items(payload, limit=limit)
|
||||
|
||||
def fetch_comments(self, source_item_id: str, *, limit: int) -> list[CommentData]:
|
||||
payload = self.client.get(
|
||||
"/api/v1/xiaohongshu/app_v2/get_note_comments",
|
||||
params={"note_id": source_item_id, "cursor": "", "index": 0, "pageArea": "UNFOLDED", "sort_strategy": "latest_v2"},
|
||||
)
|
||||
return self.map_comments(payload, limit=limit)
|
||||
|
||||
def _author_name(self, comment: dict[str, Any]) -> str | None:
|
||||
user = comment.get("user_info") or comment.get("user") or {}
|
||||
return user.get("nickname") or user.get("name") if isinstance(user, dict) else None
|
||||
Reference in New Issue
Block a user