"""Tests for marketing_intelligence.scraper.extractor.TikTokExtractor.
All tests use real-shaped but anonymised HTML fixtures.
No browser, no network — pure unit tests.
"""
from __future__ import annotations
import json
from typing import Any
import pytest
from marketing_intelligence.scraper.extractor import TikTokExtractor
# ─────────────────────────── helpers / fixtures ────────────────────────────
def _make_state_html(item: dict, extra_scope: dict | None = None) -> str:
scope = {"webapp.video-detail": {"itemInfo": {"itemStruct": item}}}
if extra_scope:
scope |= extra_scope
blob = {"__DEFAULT_SCOPE__": scope}
return (
f'"
)
def _make_tag_html(video_ids: list[str], authors: list[str] | None = None) -> str:
authors = authors or [f"user{i}" for i in range(len(video_ids))]
links = "".join(
f'video'
for a, v in zip(authors, video_ids, strict=False)
)
return f"{links}
12.3K videos"
# ─────────────────────────── state_blob_from_html ──────────────────────────
class TestStateBlobFromHtml:
def test_returns_none_when_no_script_tag(self):
assert TikTokExtractor.state_blob_from_html("") is None
def test_returns_dict_when_script_present(self) -> None:
blob: dict[str, Any] = {"__DEFAULT_SCOPE__": {}}
html = (
f'"
)
result = TikTokExtractor.state_blob_from_html(html)
assert result == blob
def test_returns_none_on_malformed_json(self):
html = ''
assert TikTokExtractor.state_blob_from_html(html) is None
# ──────────────────────── video_metrics_from_state ─────────────────────────
class TestVideoMetricsFromState:
def _make_blob(
self, stats: dict, challenges: list | None = None, create_time: int = 0
) -> dict:
item = {
"stats": stats,
"createTime": create_time,
"challenges": challenges or [],
}
return {
"__DEFAULT_SCOPE__": {
"webapp.video-detail": {"itemInfo": {"itemStruct": item}}
}
}
def test_extracts_all_stats(self):
blob = self._make_blob(
stats={
"playCount": 5000,
"diggCount": 200,
"commentCount": 30,
"shareCount": 10,
"collectCount": 5,
},
create_time=1700000000,
)
result = TikTokExtractor.video_metrics_from_state(blob)
assert result is not None
assert result["views"] == 5000
assert result["likes"] == 200
assert result["comments"] == 30
assert result["shares"] == 10
assert result["favorites"] == 5
def test_extracts_hashtags_from_challenges(self):
blob = self._make_blob(
stats={"playCount": 100},
challenges=[{"title": "foryou"}, {"title": "viral"}, {}],
)
result = TikTokExtractor.video_metrics_from_state(blob)
assert result is not None
assert "foryou" in result["hashtags"]
assert "viral" in result["hashtags"]
assert len(result["hashtags"]) == 2 # empty title skipped
def test_returns_none_when_item_struct_missing(self) -> None:
blob: dict[str, Any] = {"__DEFAULT_SCOPE__": {"webapp.video-detail": {}}}
assert TikTokExtractor.video_metrics_from_state(blob) is None
def test_created_at_is_iso_string(self):
blob = self._make_blob(stats={"playCount": 0}, create_time=1700000000)
result = TikTokExtractor.video_metrics_from_state(blob)
assert result is not None
assert result["created_at"] is not None
assert "T" in result["created_at"]
# ──────────────────────────────── tag_videos ───────────────────────────────
class TestTagVideos:
def test_parses_video_urls(self):
html = _make_tag_html(["111", "222", "333"])
videos, _ = TikTokExtractor.tag_videos(html)
video_ids = [v.video_id for v in videos]
assert set(video_ids) == {"111", "222", "333"}
def test_deduplicates_videos(self):
html = _make_tag_html(["111", "111", "222"])
videos, _ = TikTokExtractor.tag_videos(html)
assert len(videos) == 2
def test_extracts_author(self):
html = _make_tag_html(["7001"], ["artistname"])
videos, _ = TikTokExtractor.tag_videos(html)
assert videos[0].author == "artistname"
def test_returns_video_count_text(self):
html = _make_tag_html(["111"])
_, count_text = TikTokExtractor.tag_videos(html)
assert count_text == "12.3K videos"
def test_empty_html_returns_empty_list(self):
videos, count = TikTokExtractor.tag_videos("")
assert videos == []
assert count is None
# ──────────────────────── sound_id_from_url ────────────────────────────────
class TestSoundIdFromUrl:
def test_extracts_id_from_music_url(self):
url = "https://www.tiktok.com/music/some-track-name-7123456789"
assert TikTokExtractor.sound_id_from_url(url) == "7123456789"
def test_returns_none_for_non_matching_url(self):
assert (
TikTokExtractor.sound_id_from_url("https://www.tiktok.com/tag/foryou")
is None
)
def test_extracts_last_numeric_segment(self):
url = "https://www.tiktok.com/music/feet-dont-fail-me-now-7000000001"
assert TikTokExtractor.sound_id_from_url(url) == "7000000001"
# ──────────────────────── author_and_video_id ──────────────────────────────
class TestAuthorAndVideoId:
def test_parses_standard_url(self):
url = "https://www.tiktok.com/@joycrookesmusic/video/7649525945887509782"
author, vid = TikTokExtractor.author_and_video_id(url)
assert author == "joycrookesmusic"
assert vid == "7649525945887509782"
def test_returns_none_tuple_for_non_video_url(self):
author, vid = TikTokExtractor.author_and_video_id(
"https://www.tiktok.com/tag/foryou"
)
assert author is None
assert vid is None
# ──────────────────────── video_count ──────────────────────────────────────
class TestVideoCount:
@pytest.mark.parametrize(
"html,expected",
[
("942.9K Videos", "942.9K Videos"),
("1.2M videos", "1.2M videos"),
("500 Videos", "500 Videos"),
("no count here
", None),
],
)
def test_extracts_count_text(self, html: str, expected: str | None) -> None:
assert TikTokExtractor.video_count(html) == expected
# ──────────────────────── _parse_count ─────────────────────────────────────
class TestParseCount:
@pytest.mark.parametrize(
"text,expected",
[
("1K", 1_000),
("2.5K", 2_500),
("1M", 1_000_000),
("1.5B", 1_500_000_000),
("500", 500),
("1,234", 1234),
],
)
def test_parses_count(self, text: str, expected: int) -> None:
assert TikTokExtractor._parse_count(text) == expected
def test_returns_none_on_invalid(self):
assert TikTokExtractor._parse_count("invalid") is None
# ──────────────────────── video_metrics fallback ───────────────────────────
class TestVideoMetricsRegexFallback:
def test_falls_back_when_no_state_blob(self):
html = (
'"createTime":"1700000000"'
'"playCount":9999'
'"diggCount":123'
'"commentCount":45'
'"shareCount":6'
'"collectCount":2'
)
result = TikTokExtractor.video_metrics(html)
assert result["views"] == 9999
assert result["likes"] == 123
assert result["comments"] == 45
def test_prefers_state_blob_over_regex(self):
item = {
"stats": {
"playCount": 777,
"diggCount": 88,
"commentCount": 9,
"shareCount": 2,
"collectCount": 1,
},
"createTime": 1700000000,
"challenges": [],
}
# State blob has 777 views; regex noise has different value
html = _make_state_html(item) + '"playCount":9999'
result = TikTokExtractor.video_metrics(html)
assert result["views"] == 777