wangshaoqing/xhs_video_crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix XHS video URL extraction

Browse Source
This commit is contained in:
wangshaoqing 2026-05-27 15:11:42 +08:00
parent 3a2a6a69e0
commit 16f262ada1
3 changed files with 164 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,5 +1,7 @@
.venv/
__pycache__/
*.pyc
.DS_Store
.xhs-chrome-profile/
video/
video_bad_*/

81
XHS.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import argparse
import html
import json
import re
import socket
@ -13,10 +14,10 @@ from typing import Any
DEFAULT_EXPLORE_URL = "https://www.xiaohongshu.com/explore"
DEFAULT_BROWSER_PORT = 9224
DEFAULT_OUTPUT_DIR = Path("video")
LISTEN_TARGET = "feed"
LISTEN_TARGET = "/api/sns/web/v1/feed"
MAX_FILENAME_BYTES = 240
INVALID_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|\r\n\t]')
VIDEO_URL_KEYS = {"master_url", "backup_url", "backup_urls", "url"}
VIDEO_URL_KEYS = {"master_url", "backup_url", "backup_urls"}
TITLE_KEYS = ("display_title", "title", "desc", "description")
ID_KEYS = ("id", "note_id", "noteId", "video_id", "file_id")
AUTHOR_KEYS = ("nickname", "name", "user_name", "userName")
@ -143,11 +144,41 @@ def extract_video_candidates(payload: Any) -> list[VideoCandidate]:
return candidates
def decode_html_video_url(value: str) -> str:
decoded = html.unescape(value)
return decoded.replace("\\u002F", "/").replace("\\/", "/")
def extract_video_candidates_from_html(source: str, video_id: str = "current-page") -> list[VideoCandidate]:
patterns = [
r'\\"master_url\\"\s*:\s*\\"(.*?)\\"',
r'"master_url"\s*:\s*"(.*?)"',
]
candidates: list[VideoCandidate] = []
seen_urls: set[str] = set()
for pattern in patterns:
for match in re.findall(pattern, source):
video_url = decode_html_video_url(match)
if video_url in seen_urls or not looks_like_video_url(video_url):
continue
seen_urls.add(video_url)
candidates.append(
VideoCandidate(
video_id=video_id,
title="current-page-video",
video_url=video_url,
author_name="unknown",
source_key="html_master_url",
)
)
return candidates
def choose_video_candidate(candidates: list[VideoCandidate]) -> VideoCandidate:
if not candidates:
raise ValueError("没有可用的视频候选地址。")
source_priority = {"master_url": 0, "backup_url": 1, "backup_urls": 2, "url": 3}
source_priority = {"master_url": 0, "html_master_url": 0, "backup_url": 1, "backup_urls": 2, "url": 3}
return sorted(candidates, key=lambda item: source_priority.get(item.source_key, 99))[0]
@ -253,10 +284,31 @@ def download_video(
) -> None:
response = requests_module.get(video_url, headers=headers, timeout=60)
response.raise_for_status()
validate_video_response(response, video_url)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(response.content)
def validate_video_response(response: Any, video_url: str) -> None:
content = getattr(response, "content", b"")
content_type = str(getattr(response, "headers", {}).get("content-type", "")).lower()
if content_type.startswith("image/"):
raise ValueError(f"非视频响应: {content_type} {video_url}")
if content.startswith(b"RIFF") and b"WEBP" in content[:16]:
raise ValueError(f"非视频响应: image/webp {video_url}")
if content.lstrip().startswith((b"<html", b"<!doctype", b"{")):
raise ValueError(f"非视频响应: {content_type or 'unknown'} {video_url}")
has_video_type = content_type.startswith("video/")
has_mp4_signature = len(content) >= 12 and content[4:8] == b"ftyp"
has_webm_signature = content.startswith(b"\x1a\x45\xdf\xa3")
if has_video_type or has_mp4_signature or has_webm_signature:
return
raise ValueError(f"非视频响应: {content_type or 'unknown'} {video_url}")
def wait_for_feed_packet(page: Any, timeout: int) -> Any | None:
try:
packet = page.listen.wait(timeout=timeout)
@ -294,6 +346,7 @@ def collect_videos(
output_dir: Path,
browser_port: int | None,
start_url: str = DEFAULT_EXPLORE_URL,
use_current_page: bool = False,
) -> int:
requests_module, chromium_page_cls, chromium_options_cls = import_runtime_dependencies()
if browser_port is not None:
@ -301,6 +354,10 @@ def collect_videos(
page = create_page(chromium_page_cls, chromium_options_cls, browser_port)
page.listen.start(LISTEN_TARGET)
if use_current_page:
print(f"[INFO] 使用当前页面: {getattr(page, 'url', '')}")
page.refresh()
else:
print("[INFO] 正在打开小红书发现页。若出现登录或验证码，请先在浏览器窗口里完成。")
page.get(start_url)
time.sleep(3)
@ -314,10 +371,17 @@ def collect_videos(
while downloaded < max_videos and consecutive_empty < max_consecutive_empty:
packet = wait_for_feed_packet(page, timeout=timeout)
if packet is None:
candidates = group_video_candidates(
extract_video_candidates_from_html(
page.run_js("return document.documentElement.outerHTML"),
video_id=extract_note_id_from_url(getattr(page, "url", "")),
)
)
if not candidates:
consecutive_empty += 1
scroll_feed(page)
continue
else:
try:
payload = extract_feed_payload(packet.response)
candidates = group_video_candidates(extract_video_candidates(payload))
@ -365,6 +429,13 @@ def collect_videos(
return downloaded
def extract_note_id_from_url(url: str) -> str:
match = re.search(r"/explore/([^/?#]+)", url)
if match:
return match.group(1)
return "current-page"
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="附着到已登录小红书 Chrome，监听 feed 响应并下载视频")
parser.add_argument("--max-videos", type=int, default=10, help="最多下载视频数量，默认 10")
@ -372,6 +443,7 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--browser-port", type=int, default=DEFAULT_BROWSER_PORT, help="Chrome 调试端口，默认 9224")
parser.add_argument("--timeout", type=int, default=20, help="等待单次 feed 响应的秒数，默认 20")
parser.add_argument("--start-url", default=DEFAULT_EXPLORE_URL, help="打开或刷新使用的小红书页面")
parser.add_argument("--use-current-page", action="store_true", help="使用浏览器当前页面，不强制打开发现页")
return parser
@ -388,6 +460,7 @@ def main(argv: list[str] | None = None) -> int:
output_dir=Path(args.output_dir),
browser_port=args.browser_port,
start_url=args.start_url,
use_current_page=args.use_current_page,
)
print(f"[INFO] 本次共下载 {downloaded} 个视频。")
return 0

72
test_xhs.py
View File

@ -9,6 +9,25 @@ class FakeResponse:
self.raw_body = raw_body
class FakeDownloadResponse:
def __init__(self, content: bytes, content_type: str = "video/mp4", status_code: int = 200):
self.content = content
self.headers = {"content-type": content_type}
self.status_code = status_code
def raise_for_status(self) -> None:
if self.status_code >= 400:
raise RuntimeError(f"HTTP {self.status_code}")
class FakeRequests:
def __init__(self, response: FakeDownloadResponse):
self.response = response
def get(self, video_url, headers, timeout):
return self.response
class XhsModuleTests(unittest.TestCase):
def test_module_can_import_without_optional_runtime_dependencies(self) -> None:
module = importlib.import_module("XHS")
@ -94,6 +113,32 @@ class XhsModuleTests(unittest.TestCase):
self.assertEqual(candidates[0].author_name, "摄影师")
self.assertEqual(candidates[0].source_key, "master_url")
def test_extract_video_candidates_ignores_plain_image_url_fields(self) -> None:
module = importlib.import_module("XHS")
payload = {
"id": "note-image",
"display_title": "图片笔记",
"user": {"nickname": "作者"},
"image_list": [
{"url": "https://sns-img.xhscdn.com/example.webp"},
{"url": "https://sns-img.xhscdn.com/example.jpg"},
],
}
self.assertEqual(module.extract_video_candidates(payload), [])
def test_extract_video_candidates_from_escaped_html_state(self) -> None:
module = importlib.import_module("XHS")
html = (
'<script>{"display_title":"视频标题","nickname":"作者",'
r'\"master_url\":\"http:\/\/sns-video-qc.xhscdn.com\/stream\/a.mp4?sign=1&t=2\"'
'}</script>'
)
candidates = module.extract_video_candidates_from_html(html, video_id="note123")
self.assertEqual(len(candidates), 1)
self.assertEqual(candidates[0].video_id, "note123")
self.assertEqual(candidates[0].video_url, "http://sns-video-qc.xhscdn.com/stream/a.mp4?sign=1&t=2")
self.assertEqual(candidates[0].source_key, "html_master_url")
def test_build_output_path_uses_author_title_and_video_id(self) -> None:
module = importlib.import_module("XHS")
candidate = module.VideoCandidate(
@ -147,6 +192,7 @@ class XhsModuleTests(unittest.TestCase):
self.assertEqual(args.browser_port, 9224)
self.assertEqual(args.timeout, 20)
self.assertEqual(args.start_url, module.DEFAULT_EXPLORE_URL)
self.assertFalse(args.use_current_page)
def test_main_invokes_collect_videos_with_cli_values(self) -> None:
module = importlib.import_module("XHS")
@ -172,6 +218,32 @@ class XhsModuleTests(unittest.TestCase):
self.assertEqual(kwargs["output_dir"].as_posix(), "downloads")
self.assertEqual(kwargs["browser_port"], 9334)
self.assertEqual(kwargs["timeout"], 7)
self.assertFalse(kwargs["use_current_page"])
def test_download_video_rejects_webp_response_before_writing_file(self) -> None:
module = importlib.import_module("XHS")
response = FakeDownloadResponse(b"RIFF....WEBP", content_type="image/webp")
with self.assertRaisesRegex(ValueError, "非视频响应"):
module.download_video(
requests_module=FakeRequests(response),
headers={},
video_url="https://sns-img.xhscdn.com/example.webp",
output_path=mock.MagicMock(),
)
def test_download_video_accepts_mp4_signature(self) -> None:
module = importlib.import_module("XHS")
output_path = mock.MagicMock()
output_path.parent.mkdir = mock.MagicMock()
output_path.write_bytes = mock.MagicMock()
response = FakeDownloadResponse(b"\x00\x00\x00\x18ftypmp42payload", content_type="application/octet-stream")
module.download_video(
requests_module=FakeRequests(response),
headers={},
video_url="https://sns-video.xhscdn.com/example.mp4",
output_path=output_path,
)
output_path.write_bytes.assert_called_once_with(b"\x00\x00\x00\x18ftypmp42payload")
if __name__ == "__main__":