"""Monitoring run without LLM — rule-based retry, browser rotation, checkpoint/resume.""" import asyncio import random from typing import Any import structlog _BROWSER_ROTATION = [ {"stealth": "camoufox", "profile": "desktop"}, {"stealth": "chromium", "profile": "desktop"}, {"stealth": "webkit", "profile": "mobile_ios"}, ] _MAX_RETRIES = 3 _SOFT_SLEEP = (5, 15) _POST_SLEEP = (2, 6) log = structlog.get_logger("watch.direct") async def _create_session(stealth: str, profile: str) -> str: from marketing_intelligence.mcp.tools.browser import create_browser_session result = await create_browser_session( stealth=stealth, profile=profile, human_behavior=True ) return str(result.session_id) async def _close_session(session_id: str) -> None: from marketing_intelligence.mcp.tools.browser import close_browser_session await close_browser_session(session_id) async def _slow_down(session_id: str) -> None: from marketing_intelligence.mcp.tools.browser import apply_behavior_tactics await apply_behavior_tactics(session_id, delay_factor=1.5, scroll_speed="slow") def _block_signal(result: Any) -> str: signals = getattr(result, "signals", None) if signals is None: return "none" return getattr(signals, "block_signal", None) or "none" async def _scrape_campaign( posts: list[Any], prev_metrics: dict[str, Any], cckey: str, run_id: str, campaign_id: str, ) -> tuple[list[dict[str, Any]], set[str]]: from marketing_intelligence.mcp.tools.scraping_tools.base import scrape_video from marketing_intelligence.storage.checkpoint import ( clear_checkpoint, load_checkpoint, save_checkpoint, ) ck_key = f"{campaign_id}_direct" checkpoint = load_checkpoint(ck_key) completed: set[str] = set(checkpoint.get("completed", [])) failed: set[str] = set(checkpoint.get("failed", [])) pending = [ p for p in posts if p.post_id not in completed and p.post_id not in failed ] clog = log.bind(campaign_id=campaign_id, total=len(posts), pending=len(pending)) clog.info("monitor_direct.campaign_start") if not pending: clog.info("monitor_direct.already_complete") return [], failed browser_idx = 0 session_id = await _create_session(**_BROWSER_ROTATION[browser_idx]) clog.info( "monitor_direct.session_created", stealth=_BROWSER_ROTATION[browser_idx]["stealth"], ) items: list[dict[str, Any]] = [] try: for post in pending: plog = clog.bind(post_id=post.post_id) retries = 0 scraped = False while retries < _MAX_RETRIES: result = await scrape_video(post.url, session_id=session_id) block = _block_signal(result) if result.success and (block == "none" or result.views is not None): item: dict[str, Any] = { "post_id": post.post_id, "video_id": result.video_id or post.post_id, "views": result.views, "likes": result.likes, "comments": result.comments, "shares": result.shares, "favorites": result.favorites, "sound_id": result.sound_id, } prev = prev_metrics.get(post.post_id) if prev: item["_prev"] = prev items.append(item) completed.add(post.post_id) plog.info( "monitor_direct.scraped", views=result.views, likes=result.likes ) scraped = True break elif block in ("soft", "captcha"): sleep_s = random.uniform(*_SOFT_SLEEP) plog.warning( "monitor_direct.soft_block", block=block, sleep_s=round(sleep_s, 1), retry=retries + 1, ) await _slow_down(session_id) await asyncio.sleep(sleep_s) retries += 1 else: plog.warning( "monitor_direct.hard_block", block=block, browser=_BROWSER_ROTATION[browser_idx]["stealth"], ) await _close_session(session_id) browser_idx = (browser_idx + 1) % len(_BROWSER_ROTATION) session_id = await _create_session(**_BROWSER_ROTATION[browser_idx]) plog.info( "monitor_direct.browser_rotated", stealth=_BROWSER_ROTATION[browser_idx]["stealth"], ) retries += 1 if scraped: await asyncio.sleep(random.uniform(*_POST_SLEEP)) else: plog.error("monitor_direct.post_failed", retries=retries) failed.add(post.post_id) save_checkpoint( ck_key, { "run_id": run_id, "completed": list(completed), "failed": list(failed), }, ) finally: await _close_session(session_id) if not failed: clear_checkpoint(ck_key) return items, failed async def run_monitor_direct(run_id: str) -> None: from marketing_intelligence.mcp.tools.storage import persist_post_metrics from marketing_intelligence.storage.campaign_config import ( list_active_campaign_ids, list_posts, ) from marketing_intelligence.storage.factory import get_backend campaign_ids = list_active_campaign_ids() if not campaign_ids: log.info("monitor_direct.empty") return log.info("monitor_direct.start", campaigns=len(campaign_ids), run_id=run_id) for campaign_id in campaign_ids: posts = list_posts(campaign_id) if not posts: log.info("monitor_direct.no_posts", campaign_id=campaign_id) continue cckey = posts[0].campaign_config_key prev_metrics = await get_backend().read_latest_metrics(cckey) items, failed = await _scrape_campaign( posts, prev_metrics, cckey, run_id, campaign_id ) if items: await persist_post_metrics( items=items, campaign_config_key=cckey, run_id=run_id, ) log.info( "monitor_direct.persisted", campaign_id=campaign_id, count=len(items), failed=len(failed), ) log.info("monitor_direct.done", campaigns=len(campaign_ids))