"""Pure HTML/regex extraction — no browser, no I/O.
Single entry point: TikTokExtractor.(html)
Patterns compiled once at class definition.
video_metrics() tries JSON state blob first, falls back to regex.
"""
from __future__ import annotations
import json
import re
from datetime import UTC, datetime
from typing import Any, ClassVar, cast
from marketing_intelligence.core.models import VideoRef
class TikTokExtractor:
# ------------------------------------------------------------------ patterns
_VIDEO_URL: ClassVar[re.Pattern[str]] = re.compile(
r"""https://www\.tiktok\.com/@([^/"'\s]+)/video/(\d+)"""
)
_VIDEO_COUNT: ClassVar[re.Pattern[str]] = re.compile(
r"([\d.,]+[KMB]?)\s*(?:videos|Videos)"
)
_CREATE_TIME: ClassVar[re.Pattern[str]] = re.compile(r'"createTime":"?(\d+)"?')
_PLAY_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"playCount":(\d+)')
_DIGG_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"diggCount":(\d+)')
_COMMENT_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"commentCount":(\d+)')
_SHARE_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"shareCount":(\d+)')
_COLLECT_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"collectCount":"?(\d+)"?')
_HASHTAGS: ClassVar[re.Pattern[str]] = re.compile(
r'aria-label="Watch more videos of the (\w+) category"'
)
_DESC: ClassVar[re.Pattern[str]] = re.compile(r'"desc":"([^"]+)"')
_AUTHOR_VIDEO: ClassVar[re.Pattern[str]] = re.compile(
r"tiktok\.com/@([^/]+)/video/(\d+)"
)
_SOUND_ID: ClassVar[re.Pattern[str]] = re.compile(r"-(\d+)$")
_STATE_SCRIPT: ClassVar[re.Pattern[str]] = re.compile(
r'',
re.DOTALL,
)
# ------------------------------------------------------------------ state blob
@classmethod
def state_blob_from_html(cls, html: str) -> dict[str, Any] | None:
"""Extract __UNIVERSAL_DATA_FOR_REHYDRATION__ JSON embedded in page HTML."""
m = cls._STATE_SCRIPT.search(html)
if not m:
return None
try:
return cast(dict[str, Any], json.loads(m.group(1)))
except ValueError: # json.JSONDecodeError is a subclass of ValueError
return None
@classmethod
def video_metrics_from_state(cls, blob: dict[str, Any]) -> dict[str, Any] | None:
"""Parse video metrics from __UNIVERSAL_DATA_FOR_REHYDRATION__ blob.
Returns the same shape as _video_metrics_regex so callers are agnostic.
Returns None if the expected keys are missing (non-video page blob).
"""
try:
scope = blob.get("__DEFAULT_SCOPE__", {})
item = (
scope.get("webapp.video-detail", {})
.get("itemInfo", {})
.get("itemStruct", {})
)
if not item:
return None
stats = item.get("stats", {}) or {}
create_time = item.get("createTime")
created_at = (
datetime.fromtimestamp(int(create_time), tz=UTC).isoformat()
if create_time
else None
)
hashtags = [
c.get("title") for c in (item.get("challenges") or []) if c.get("title")
]
return {
"created_at": created_at,
"views": stats.get("playCount"),
"likes": stats.get("diggCount"),
"comments": stats.get("commentCount"),
"shares": stats.get("shareCount"),
"favorites": stats.get("collectCount"),
"hashtags": hashtags,
"caption_json": item.get("desc"),
}
except Exception:
return None
# ------------------------------------------------------------------ extractors
@classmethod
def tag_videos(cls, html: str) -> tuple[list[VideoRef], str | None]:
"""Parse tag page HTML → (videos, video_count_text)."""
seen: set[str] = set()
videos: list[VideoRef] = []
for m in cls._VIDEO_URL.finditer(html):
vid_id = m.group(2)
if vid_id not in seen:
seen.add(vid_id)
videos.append(
VideoRef(video_id=vid_id, video_url=m.group(0), author=m.group(1))
)
count_m = cls._VIDEO_COUNT.search(html)
return videos, count_m.group(0).strip() if count_m else None
@classmethod
def sound_video_urls(cls, html: str) -> list[str]:
"""Parse sound page HTML → deduplicated video URLs."""
seen: set[str] = set()
urls: list[str] = []
for m in cls._VIDEO_URL.finditer(html):
vid_id = m.group(2)
if vid_id not in seen:
seen.add(vid_id)
urls.append(m.group(0))
if not urls:
for vid_id in re.findall(
r'https://www\.tiktok\.com/@[^/"\'\\s]+/video/(\d+)', html
):
if vid_id not in seen:
seen.add(vid_id)
urls.append(f"https://www.tiktok.com/video/{vid_id}")
return urls
@classmethod
def video_count(cls, html: str) -> str | None:
"""Extract '942.9K videos' text from any TikTok listing page."""
m = cls._VIDEO_COUNT.search(html)
return m.group(0).strip() if m else None
@classmethod
def video_metrics(cls, html: str) -> dict[str, Any]:
"""Extract engagement counters — JSON state blob first, regex fallback."""
blob = cls.state_blob_from_html(html)
if blob:
result = cls.video_metrics_from_state(blob)
if result:
return result
return cls._video_metrics_regex(html)
@classmethod
def author_and_video_id(cls, url: str) -> tuple[str | None, str | None]:
"""Parse author and video_id from a TikTok video URL."""
m = cls._AUTHOR_VIDEO.search(url)
return (m.group(1), m.group(2)) if m else (None, None)
@classmethod
def sound_id_from_url(cls, sound_url: str) -> str | None:
"""Extract sound_id from a /music/- URL."""
m = cls._SOUND_ID.search(sound_url)
return m.group(1) if m else None
# ------------------------------------------------------------------ artist profile / sound
@classmethod
def artist_profile(cls, html: str) -> dict[str, Any] | None:
"""Extract artist profile stats from a TikTok user page.
Returns {followers, following, likes, video_count} or None."""
blob = cls.state_blob_from_html(html)
if blob:
result = cls._artist_profile_from_state(blob)
if result:
return result
return cls._artist_profile_regex(html)
@classmethod
def _artist_profile_from_state(cls, blob: dict[str, Any]) -> dict[str, Any] | None:
try:
scope = blob.get("__DEFAULT_SCOPE__", {})
user_info = scope.get("webapp.user-detail", {}).get("userInfo", {})
stats = user_info.get("stats", {})
if not stats:
return None
return {
"followers": stats.get("followerCount"),
"following": stats.get("followingCount"),
"likes": stats.get("heartCount") or stats.get("heart"),
"video_count": stats.get("videoCount"),
"nickname": user_info.get("user", {}).get("nickname"),
"unique_id": user_info.get("user", {}).get("uniqueId"),
}
except Exception:
return None
_FOLLOWER_COUNT: ClassVar[re.Pattern[str]] = re.compile(
r"([\d.]+[KMB]?)\s*Followers"
)
_LIKES_COUNT: ClassVar[re.Pattern[str]] = re.compile(r"([\d.]+[KMB]?)\s*Likes")
_FOLLOWING_COUNT: ClassVar[re.Pattern[str]] = re.compile(
r"([\d.]+[KMB]?)\s*Following"
)
@classmethod
def _parse_count(cls, text: str) -> int | None:
text = text.strip().replace(",", "")
multipliers = {"K": 1_000, "M": 1_000_000, "B": 1_000_000_000}
for suffix, mult in multipliers.items():
if text.endswith(suffix):
return int(float(text[:-1]) * mult)
try:
return int(float(text))
except ValueError:
return None
@classmethod
def _artist_profile_regex(cls, html: str) -> dict[str, Any] | None:
followers_m = cls._FOLLOWER_COUNT.search(html)
likes_m = cls._LIKES_COUNT.search(html)
following_m = cls._FOLLOWING_COUNT.search(html)
if not followers_m and not likes_m:
return None
return {
"followers": cls._parse_count(followers_m.group(1))
if followers_m
else None,
"following": cls._parse_count(following_m.group(1))
if following_m
else None,
"likes": cls._parse_count(likes_m.group(1)) if likes_m else None,
"video_count": None,
}
# ------------------------------------------------------------------ private
@classmethod
def _video_metrics_regex(cls, html: str) -> dict[str, Any]:
"""Regex fallback for video metrics when JSON state is unavailable."""
def _find(pattern: re.Pattern[str]) -> str | None:
m = pattern.search(html)
return m.group(1) if m else None
create_time = _find(cls._CREATE_TIME)
created_at = (
datetime.fromtimestamp(int(create_time), tz=UTC).isoformat()
if create_time
else None
)
play_count = _find(cls._PLAY_COUNT)
digg_count = _find(cls._DIGG_COUNT)
comment_count = _find(cls._COMMENT_COUNT)
share_count = _find(cls._SHARE_COUNT)
collect_count = _find(cls._COLLECT_COUNT)
desc_m = cls._DESC.search(html)
return {
"created_at": created_at,
"views": int(play_count) if play_count else None,
"likes": int(digg_count) if digg_count else None,
"comments": int(comment_count) if comment_count else None,
"shares": int(share_count) if share_count else None,
"favorites": int(collect_count) if collect_count else None,
"hashtags": cls._HASHTAGS.findall(html),
"caption_json": desc_m.group(1) if desc_m else None,
}