import asyncio import random import re import time from collections.abc import Awaitable, Callable from typing import Any from marketing_intelligence.browser.base import BrowserAdapter, PageSnapshot from marketing_intelligence.core.config import settings def _sync_fetch( url: str, scroll_distance: int, proxy: dict | None, human: bool, delay_factor: float, ) -> str: from DrissionPage import ChromiumOptions, ChromiumPage opts = ChromiumOptions() if settings.browser_headless: opts.headless() if proxy and proxy.get("server"): opts.set_proxy(proxy["server"]) page = ChromiumPage(opts) try: page.get(url) time.sleep( max(0.3, 5.0 * delay_factor + (random.uniform(-1, 1) if human else 0)) ) if human: steps = random.randint(2, 4) per_step = scroll_distance // steps for _ in range(steps): page.scroll.down(per_step + random.randint(-150, 150)) time.sleep(random.uniform(0.2, 0.7) * delay_factor) else: page.scroll.down(scroll_distance) time.sleep(max(0.3, 2.0 * delay_factor)) return str(page.html) finally: page.quit() class DrissionPageAdapter(BrowserAdapter): """ DrissionPage (Chrome via CDP). Sync under the hood — runs in a thread. Expects IP-whitelisted proxy — no credential auth support in Chrome. fetch_interactive not supported — use PlaywrightAdapter for scrape_video. """ async def fetch_with_scroll( self, url: str, scroll_distance: int = 3000 ) -> PageSnapshot: html = await asyncio.to_thread( _sync_fetch, url, scroll_distance, self.proxy, self.behavior.human, self.behavior.delay_factor, ) title_m = re.search(r"([^<]+)", html) return PageSnapshot( url=url, html=html, title=title_m.group(1) if title_m else None ) async def fetch_interactive( self, url: str, callback: Callable[[Any, str, str, int], Awaitable[Any]], ) -> Any: raise NotImplementedError( "DrissionPageAdapter does not support fetch_interactive. " "Use engine='camoufox' or engine='chromium' for scrape_video." )