import logging from types import TracebackType from typing import Any, Self, overload import httpx import pydantic from httpx._types import QueryParamTypes, URLTypes # noqa from pydantic import TypeAdapter from dmp.adapters.fivetran.custom_reports import ( FACEBOOK_CUSTOM_REPORTS, GOOGLE_CUSTOM_REPORTS, TIKTOK_CUSTOM_REPORTS, ) from dmp.adapters.fivetran.enums import ( AdsAccountsSyncMode, AdsTimeframeMonths, ) from dmp.adapters.fivetran.exceptions import ( FivetranClientError, FivetranClientTimeoutError, ) from dmp.adapters.fivetran.models import ( AdsConnection, AdsConnectionWithConnectCard, ConnectCard, ConnectCardResponse, Response, ShopifyConnection, SimpleResponse, StandardConfig, StandardConfigUpdate, ) logger = logging.getLogger(__name__) DAILY_SYNC_TIME = "16:00" # Destination time is in PST (UTC-08) class FivetranClient: base_url = "https://api.fivetran.com/v1" DEFAULT_REQUEST_TIMEOUT = 20.0 DELETE_CONNECTION_REQUEST_TIMEOUT = 10.0 SYNC_OPERATION_TIMEOUT = 300.0 def __init__( self, api_key: str, api_secret: str, ads_timeframe_months: AdsTimeframeMonths, reporting_sync_frequency: int, ) -> None: self.api_key = api_key self.api_secret = api_secret self.client = httpx.Client(timeout=self.DEFAULT_REQUEST_TIMEOUT) self.ads_timeframe_months = ads_timeframe_months self.reporting_sync_frequency = reporting_sync_frequency def __enter__(self) -> Self: return self def __exit__( self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None, ) -> Any: self.client.close() def build_url(self, path: str) -> str: if not path.startswith("/"): path = f"/{path}" return self.base_url + path @overload def request[T]( self, method: str, path: str, *, params: QueryParamTypes | None = None, json: Any = None, timeout: float = DEFAULT_REQUEST_TIMEOUT, type: type[T], ) -> T: ... @overload def request( self, method: str, path: str, *, params: QueryParamTypes | None = None, json: Any = None, timeout: float = DEFAULT_REQUEST_TIMEOUT, ) -> httpx.Response: ... def request[T]( self, method: str, path: str, *, params: QueryParamTypes | None = None, json: Any = None, timeout: float = DEFAULT_REQUEST_TIMEOUT, type: type[T] | None = None, ) -> httpx.Response | T: try: return self._request( method, path, params=params, json=json, timeout=timeout, type=type, ) except FivetranClientError as exc: logger.error(exc.log_msg, exc_info=exc, extra=exc.log_extra) raise exc def _request[T]( self, method: str, path: str, *, params: QueryParamTypes | None = None, json: Any = None, timeout: float = DEFAULT_REQUEST_TIMEOUT, type: type[T] | None = None, ) -> httpx.Response | T: try: response = self.client.request( method, self.build_url(path), params=params, json=json, auth=(self.api_key, self.api_secret), timeout=timeout, ) except httpx.ReadTimeout as exc: raise FivetranClientTimeoutError( f"Fivetran REST API request timeout: {method} {exc.request.url.path}", request=exc.request, ) from exc except httpx.RequestError as exc: raise FivetranClientError( f"Failed to request Fivetran REST API: {method} {exc.request.url.path}", request=exc.request, ) from exc try: response.raise_for_status() except httpx.HTTPStatusError as exc: raise FivetranClientError( ( "Invalid Fivetran REST API " f"response status {exc.response.status_code}" ), request=exc.request, response=exc.response, ) from exc if type is not None: try: return TypeAdapter(type).validate_json(response.content) except pydantic.ValidationError as exc: raise FivetranClientError( "Failed to deserialize Fivetran REST API " f"response into {type}: {exc.errors()}" ) from exc return response def create_shopify_connection( self, *, group_id: str, schema: str, shop_domain: str, ) -> ShopifyConnection: payload = { "group_id": group_id, "service": "shopify", "run_setup_tests": False, "paused": False, "pause_after_trial": False, "daily_sync_time": DAILY_SYNC_TIME, "sync_frequency": "1440", "config": { "schema": schema, "shop": shop_domain, }, } response_obj = self.request( "POST", "/connections", type=Response[ShopifyConnection], json=payload ) return response_obj.data def create_facebook_ads_connection( self, *, group_id: str, schema: str, redirect_uri: str, accounts: list[str] | None = None, ) -> AdsConnectionWithConnectCard: config = { "schema": schema, "sync_metadata": True, "timeframe_months": self.ads_timeframe_months, "sync_mode": AdsAccountsSyncMode.SPECIFIC_ACCOUNTS, "custom_tables": FACEBOOK_CUSTOM_REPORTS, } if accounts: config["accounts"] = accounts payload = { "group_id": group_id, "service": "facebook_ads", "run_setup_tests": False, "paused": False, "pause_after_trial": False, "sync_frequency": self.reporting_sync_frequency, "connect_card_config": { "redirect_uri": redirect_uri, "hide_setup_guide": True, }, "config": config, } if self.reporting_sync_frequency == 1440: payload["daily_sync_time"] = DAILY_SYNC_TIME response_obj = self.request( "POST", "/connections", type=Response[AdsConnectionWithConnectCard], json=payload, ) return response_obj.data def create_tiktok_ads_connection( self, *, group_id: str, schema: str, redirect_uri: str, accounts: list[str] | None = None, ) -> AdsConnectionWithConnectCard: config = { "schema": schema, "sync_metadata": True, "timeframe_months": self.ads_timeframe_months, "sync_mode": AdsAccountsSyncMode.SPECIFIC_ACCOUNTS, "custom_reports": TIKTOK_CUSTOM_REPORTS, } if accounts: config["accounts"] = accounts payload = { "group_id": group_id, "service": "tiktok_ads", "run_setup_tests": False, "paused": False, "pause_after_trial": False, "sync_frequency": self.reporting_sync_frequency, "connect_card_config": { "redirect_uri": redirect_uri, "hide_setup_guide": True, }, "config": config, } if self.reporting_sync_frequency == 1440: payload["daily_sync_time"] = DAILY_SYNC_TIME response_obj = self.request( "POST", "/connections", type=Response[AdsConnectionWithConnectCard], json=payload, ) return response_obj.data def create_google_ads_connection( self, *, group_id: str, schema: str, redirect_uri: str, accounts: list[str] | None = None, ) -> AdsConnectionWithConnectCard: config: dict[str, Any] = { "schema": schema, "timeframe_months": self.ads_timeframe_months, "sync_mode": AdsAccountsSyncMode.SPECIFIC_ACCOUNTS, "reports": GOOGLE_CUSTOM_REPORTS, } if accounts: config["accounts"] = accounts payload = { "group_id": group_id, "service": "google_ads", "run_setup_tests": False, "paused": False, "pause_after_trial": False, "sync_frequency": self.reporting_sync_frequency, "connect_card_config": { "redirect_uri": redirect_uri, "hide_setup_guide": True, }, "config": config, } if self.reporting_sync_frequency == 1440: payload["daily_sync_time"] = DAILY_SYNC_TIME response_obj = self.request( "POST", "/connections", type=Response[AdsConnectionWithConnectCard], json=payload, ) return response_obj.data def get_ads_connection(self, connection_id: str) -> AdsConnection: response_obj = self.request( "GET", f"/connections/{connection_id}", type=Response[AdsConnection] ) return response_obj.data def get_shopify_connection(self, connection_id: str) -> ShopifyConnection: response_obj = self.request( "GET", f"/connections/{connection_id}", type=Response[ShopifyConnection] ) return response_obj.data def get_connect_card(self, connection_id: str, redirect_uri: str) -> ConnectCard: payload = { "connect_card_config": { "redirect_uri": redirect_uri, "hide_setup_guide": True, } } response_obj = self.request( "POST", f"/connections/{connection_id}/connect-card", type=Response[ConnectCardResponse], json=payload, ) return response_obj.data.connect_card def delete_connection(self, connection_id: str) -> None: try: self.request( "DELETE", f"/connections/{connection_id}", type=SimpleResponse, timeout=self.DELETE_CONNECTION_REQUEST_TIMEOUT, ) except FivetranClientTimeoutError: logger.warning( "Fivetran connection deletion timeout.", extra={"connection_id": connection_id}, ) # As the operation did not time out right away, # we should hope that the connection will be deleted by Fivetran anyway. return None return None def get_connection_schema_config(self, connection_id: str) -> StandardConfig: resp_obj = self.request( "GET", f"/connections/{connection_id}/schemas", type=Response[StandardConfig], ) return resp_obj.data def patch_connection_schema_config( self, connection_id: str, schema_config: StandardConfigUpdate ) -> StandardConfig: resp_obj = self.request( "PATCH", f"/connections/{connection_id}/schemas", type=Response[StandardConfig], json=schema_config.model_dump(), timeout=self.SYNC_OPERATION_TIMEOUT, ) return resp_obj.data def sync_connection_data(self, connection_id: str) -> str | None: payload = { "force": False, } response_obj = self.request( "POST", f"/connections/{connection_id}/sync", type=SimpleResponse, json=payload, timeout=self.SYNC_OPERATION_TIMEOUT, ) return response_obj.message def patch_ads_connection( self, connection_id: str, payload: dict[str, Any] ) -> AdsConnection: response_obj = self.request( "PATCH", f"/connections/{connection_id}", type=Response[AdsConnection], json=payload, timeout=self.SYNC_OPERATION_TIMEOUT, ) return response_obj.data