"""Pure HTML/regex extraction — no browser, no I/O. Single entry point: TikTokExtractor.(html) Patterns compiled once at class definition. video_metrics() tries JSON state blob first, falls back to regex. """ from __future__ import annotations import json import re from datetime import UTC, datetime from typing import Any, ClassVar, cast from marketing_intelligence.core.models import VideoRef class TikTokExtractor: # ------------------------------------------------------------------ patterns _VIDEO_URL: ClassVar[re.Pattern[str]] = re.compile( r"""https://www\.tiktok\.com/@([^/"'\s]+)/video/(\d+)""" ) _VIDEO_COUNT: ClassVar[re.Pattern[str]] = re.compile( r"([\d.,]+[KMB]?)\s*(?:videos|Videos)" ) _CREATE_TIME: ClassVar[re.Pattern[str]] = re.compile(r'"createTime":"?(\d+)"?') _PLAY_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"playCount":(\d+)') _DIGG_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"diggCount":(\d+)') _COMMENT_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"commentCount":(\d+)') _SHARE_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"shareCount":(\d+)') _COLLECT_COUNT: ClassVar[re.Pattern[str]] = re.compile(r'"collectCount":"?(\d+)"?') _HASHTAGS: ClassVar[re.Pattern[str]] = re.compile( r'aria-label="Watch more videos of the (\w+) category"' ) _DESC: ClassVar[re.Pattern[str]] = re.compile(r'"desc":"([^"]+)"') _AUTHOR_VIDEO: ClassVar[re.Pattern[str]] = re.compile( r"tiktok\.com/@([^/]+)/video/(\d+)" ) _SOUND_ID: ClassVar[re.Pattern[str]] = re.compile(r"-(\d+)$") _STATE_SCRIPT: ClassVar[re.Pattern[str]] = re.compile( r']+id="__UNIVERSAL_DATA_FOR_REHYDRATION__"[^>]*>(.*?)', re.DOTALL, ) # ------------------------------------------------------------------ state blob @classmethod def state_blob_from_html(cls, html: str) -> dict[str, Any] | None: """Extract __UNIVERSAL_DATA_FOR_REHYDRATION__ JSON embedded in page HTML.""" m = cls._STATE_SCRIPT.search(html) if not m: return None try: return cast(dict[str, Any], json.loads(m.group(1))) except ValueError: # json.JSONDecodeError is a subclass of ValueError return None @classmethod def video_metrics_from_state(cls, blob: dict[str, Any]) -> dict[str, Any] | None: """Parse video metrics from __UNIVERSAL_DATA_FOR_REHYDRATION__ blob. Returns the same shape as _video_metrics_regex so callers are agnostic. Returns None if the expected keys are missing (non-video page blob). """ try: scope = blob.get("__DEFAULT_SCOPE__", {}) item = ( scope.get("webapp.video-detail", {}) .get("itemInfo", {}) .get("itemStruct", {}) ) if not item: return None stats = item.get("stats", {}) or {} create_time = item.get("createTime") created_at = ( datetime.fromtimestamp(int(create_time), tz=UTC).isoformat() if create_time else None ) hashtags = [ c.get("title") for c in (item.get("challenges") or []) if c.get("title") ] return { "created_at": created_at, "views": stats.get("playCount"), "likes": stats.get("diggCount"), "comments": stats.get("commentCount"), "shares": stats.get("shareCount"), "favorites": stats.get("collectCount"), "hashtags": hashtags, "caption_json": item.get("desc"), } except Exception: return None # ------------------------------------------------------------------ extractors @classmethod def tag_videos(cls, html: str) -> tuple[list[VideoRef], str | None]: """Parse tag page HTML → (videos, video_count_text).""" seen: set[str] = set() videos: list[VideoRef] = [] for m in cls._VIDEO_URL.finditer(html): vid_id = m.group(2) if vid_id not in seen: seen.add(vid_id) videos.append( VideoRef(video_id=vid_id, video_url=m.group(0), author=m.group(1)) ) count_m = cls._VIDEO_COUNT.search(html) return videos, count_m.group(0).strip() if count_m else None @classmethod def sound_video_urls(cls, html: str) -> list[str]: """Parse sound page HTML → deduplicated video URLs.""" seen: set[str] = set() urls: list[str] = [] for m in cls._VIDEO_URL.finditer(html): vid_id = m.group(2) if vid_id not in seen: seen.add(vid_id) urls.append(m.group(0)) if not urls: for vid_id in re.findall( r'https://www\.tiktok\.com/@[^/"\'\\s]+/video/(\d+)', html ): if vid_id not in seen: seen.add(vid_id) urls.append(f"https://www.tiktok.com/video/{vid_id}") return urls @classmethod def video_count(cls, html: str) -> str | None: """Extract '942.9K videos' text from any TikTok listing page.""" m = cls._VIDEO_COUNT.search(html) return m.group(0).strip() if m else None @classmethod def video_metrics(cls, html: str) -> dict[str, Any]: """Extract engagement counters — JSON state blob first, regex fallback.""" blob = cls.state_blob_from_html(html) if blob: result = cls.video_metrics_from_state(blob) if result: return result return cls._video_metrics_regex(html) @classmethod def author_and_video_id(cls, url: str) -> tuple[str | None, str | None]: """Parse author and video_id from a TikTok video URL.""" m = cls._AUTHOR_VIDEO.search(url) return (m.group(1), m.group(2)) if m else (None, None) @classmethod def sound_id_from_url(cls, sound_url: str) -> str | None: """Extract sound_id from a /music/- URL.""" m = cls._SOUND_ID.search(sound_url) return m.group(1) if m else None # ------------------------------------------------------------------ artist profile / sound @classmethod def artist_profile(cls, html: str) -> dict[str, Any] | None: """Extract artist profile stats from a TikTok user page. Returns {followers, following, likes, video_count} or None.""" blob = cls.state_blob_from_html(html) if blob: result = cls._artist_profile_from_state(blob) if result: return result return cls._artist_profile_regex(html) @classmethod def _artist_profile_from_state(cls, blob: dict[str, Any]) -> dict[str, Any] | None: try: scope = blob.get("__DEFAULT_SCOPE__", {}) user_info = scope.get("webapp.user-detail", {}).get("userInfo", {}) stats = user_info.get("stats", {}) if not stats: return None return { "followers": stats.get("followerCount"), "following": stats.get("followingCount"), "likes": stats.get("heartCount") or stats.get("heart"), "video_count": stats.get("videoCount"), "nickname": user_info.get("user", {}).get("nickname"), "unique_id": user_info.get("user", {}).get("uniqueId"), } except Exception: return None _FOLLOWER_COUNT: ClassVar[re.Pattern[str]] = re.compile( r"([\d.]+[KMB]?)\s*Followers" ) _LIKES_COUNT: ClassVar[re.Pattern[str]] = re.compile(r"([\d.]+[KMB]?)\s*Likes") _FOLLOWING_COUNT: ClassVar[re.Pattern[str]] = re.compile( r"([\d.]+[KMB]?)\s*Following" ) @classmethod def _parse_count(cls, text: str) -> int | None: text = text.strip().replace(",", "") multipliers = {"K": 1_000, "M": 1_000_000, "B": 1_000_000_000} for suffix, mult in multipliers.items(): if text.endswith(suffix): return int(float(text[:-1]) * mult) try: return int(float(text)) except ValueError: return None @classmethod def _artist_profile_regex(cls, html: str) -> dict[str, Any] | None: followers_m = cls._FOLLOWER_COUNT.search(html) likes_m = cls._LIKES_COUNT.search(html) following_m = cls._FOLLOWING_COUNT.search(html) if not followers_m and not likes_m: return None return { "followers": cls._parse_count(followers_m.group(1)) if followers_m else None, "following": cls._parse_count(following_m.group(1)) if following_m else None, "likes": cls._parse_count(likes_m.group(1)) if likes_m else None, "video_count": None, } # ------------------------------------------------------------------ private @classmethod def _video_metrics_regex(cls, html: str) -> dict[str, Any]: """Regex fallback for video metrics when JSON state is unavailable.""" def _find(pattern: re.Pattern[str]) -> str | None: m = pattern.search(html) return m.group(1) if m else None create_time = _find(cls._CREATE_TIME) created_at = ( datetime.fromtimestamp(int(create_time), tz=UTC).isoformat() if create_time else None ) play_count = _find(cls._PLAY_COUNT) digg_count = _find(cls._DIGG_COUNT) comment_count = _find(cls._COMMENT_COUNT) share_count = _find(cls._SHARE_COUNT) collect_count = _find(cls._COLLECT_COUNT) desc_m = cls._DESC.search(html) return { "created_at": created_at, "views": int(play_count) if play_count else None, "likes": int(digg_count) if digg_count else None, "comments": int(comment_count) if comment_count else None, "shares": int(share_count) if share_count else None, "favorites": int(collect_count) if collect_count else None, "hashtags": cls._HASHTAGS.findall(html), "caption_json": desc_m.group(1) if desc_m else None, }