"""Tiktok parsing module."""
import fake_useragent
import requests
from bs4 import BeautifulSoup
from lambdacommon.common_config import logger
import config
from src import errors
try:
ua = fake_useragent.UserAgent()
except Exception:
# If any error occurs (e.g., network, FakeUserAgentError), use a simple dummy UA.
class DummyUA:
"""Dummy User-Agent generator."""
@property
def random(self):
"""Return a dummy User-Agent string."""
return 'Mozilla/5.0 (compatible; test-agent)'
ua = DummyUA()
def _abbreviated_value_to_int(value: str) -> int:
"""Convert values like 4M, 35.6K to int."""
try:
if 'M' in value:
followers = int(float(value.replace(',', '.').rstrip('M')) * 1000000)
elif 'K' in value:
followers = int(float(value.replace(',', '.').rstrip('K')) * 1000)
else:
followers = int(value)
return followers
except: # noqa
logger.error('Failed to parse and convert the followers number: %s', value)
return None
def get_tiktok_followers(tiktok_url: str) -> int:
"""Get Tiktok followers.
Followers HTML block example:
0
Following
39.1M
Followers
641.7M
Likes
Alternative:
33M
Alternative 2:
Take the second from
Args:
tiktok_url (str): URL of the account to parse.
Returns:
int: the number of followers.
Raises:
HTTPError: Tiktok returned an error (usually 404).
ParsingError: Failed to parse the Tiktok artist page.
"""
followers = None
proxies = {
'http': config.SMARTPROXY_URL,
'https': config.SMARTPROXY_URL
}
headers = {
'User-Agent': ua.random
}
response = requests.get(
tiktok_url, params={'lang': 'en'}, proxies=proxies, headers=headers)
response.raise_for_status()
soup = BeautifulSoup(response.text)
# Check for CAPTCHA
if soup.find('div', {'id': 'verify-ele'}):
raise errors.TikTokCaptchaException()
followers_span = soup.find('strong', {'title': 'Followers'})
if not followers_span:
followers_span = soup.find('strong', {'data-e2e': 'followers-count'})
if not followers_span:
# fallback: try taking the second div.number child element
count_infos_h2 = soup.find('h2', {'class': 'count-infos'})
if count_infos_h2:
followers_span = count_infos_h2.select_one('div.number:nth-child(2) > strong')
if not followers_span:
logger.error('Failed to find the Followers tag.')
return followers
followers = _abbreviated_value_to_int(followers_span.text)
return followers