wangshaoqing/douyin-crawler-poc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add recommendation URL recognition

Browse Source
This commit is contained in:
wangshaoqing 2026-05-06 17:15:19 +08:00
parent 86839a873f
commit f7374d2088
2 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Douyin.py
View File

@ -29,6 +29,7 @@ DEFAULT_BROWSER_PORT = 9223
LISTEN_TARGET = "web/aweme/post/" LISTEN_TARGET = "web/aweme/post/"
SINGLE_VIDEO_LISTEN_TARGET = "web/aweme/detail/" SINGLE_VIDEO_LISTEN_TARGET = "web/aweme/detail/"
INVALID_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|\r\n\t]') INVALID_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|\r\n\t]')
RECOMMENDATION_URL_PATTERN = re.compile(r"^https?://www\.douyin\.com/?(?:\?.*)?$")
CREATOR_URL_PATTERN = re.compile(r"^https?://www\.douyin\.com/user/[^/?#]+(?:\?.*)?$") CREATOR_URL_PATTERN = re.compile(r"^https?://www\.douyin\.com/user/[^/?#]+(?:\?.*)?$")
VIDEO_URL_PATTERN = re.compile(r"^https?://www\.douyin\.com/video/(?P<aweme_id>\d+)(?:[/?#].*)?$") VIDEO_URL_PATTERN = re.compile(r"^https?://www\.douyin\.com/video/(?P<aweme_id>\d+)(?:[/?#].*)?$")
AWEME_ID_PATTERN = re.compile(r"^\d{5,}$") AWEME_ID_PATTERN = re.compile(r"^\d{5,}$")
@ -47,6 +48,10 @@ def sanitize_filename(value: str, fallback: str = "untitled") -> str:
return cleaned or fallback return cleaned or fallback
def is_recommendation_url(value: str) -> bool:
return bool(RECOMMENDATION_URL_PATTERN.match(value.strip()))
def is_creator_url(value: str) -> bool: def is_creator_url(value: str) -> bool:
return bool(CREATOR_URL_PATTERN.match(value.strip())) return bool(CREATOR_URL_PATTERN.match(value.strip()))

8
test_douyin.py
View File

@ -115,6 +115,14 @@ class DouyinModuleTests(unittest.TestCase):
with self.assertRaisesRegex(RuntimeError, "login_douyin.py"): with self.assertRaisesRegex(RuntimeError, "login_douyin.py"):
module.ensure_browser_debug_port_ready(9223) module.ensure_browser_debug_port_ready(9223)
def test_is_recommendation_url_accepts_douyin_homepage(self) -> None:
module = importlib.import_module("Douyin")
self.assertTrue(module.is_recommendation_url("https://www.douyin.com/"))
self.assertTrue(module.is_recommendation_url("https://www.douyin.com"))
self.assertTrue(module.is_recommendation_url("https://www.douyin.com/?from=web"))
self.assertFalse(module.is_recommendation_url("https://www.douyin.com/user/xxx"))
self.assertFalse(module.is_recommendation_url("https://www.douyin.com/video/123"))
def test_is_creator_url_accepts_supported_douyin_creator_url(self) -> None: def test_is_creator_url_accepts_supported_douyin_creator_url(self) -> None:
module = importlib.import_module("Douyin") module = importlib.import_module("Douyin")
self.assertTrue( self.assertTrue(