import logging from typing import Iterable from service.tasks.precondition.consequences.deletion import ( AllianceDeletedConsequence, AllianceSegmentDeletedConsequence, AllianceSourceDeletedConsequence, AudienceCollectionDeletedConsequence, AudienceDeletedConsequence, AudienceUnsharedConsequence, CollectionDeletedConsequence, FilteringSetDeletedConsequence, SupersetDeletedConsequence, SupersetSegmentDeletedConsequence, WorkspaceDeletedConsequence, WorkspaceSegmentDeletedConsequence, WorkspaceSourceDeletedConsequence, ) from service.tasks.precondition.consequences.removal import ( AllianceMemberRemovedConsequence, ) from service.tasks.precondition.exceptions import UndeletableObject from service.tasks.precondition.utils import exclude_keys from service.utils.aws_connectors import run_query from .basic import Check, CheckFor, add_check logger = logging.getLogger(__name__) class CheckWorkspacesFor(CheckFor): def __init__(self, check_class, management_schema, **kwargs): super().__init__( f"SELECT id FROM {management_schema}.workspace", management_schema=management_schema, **kwargs, ) self.check_class = check_class def launch_check(self, engine, workspace_schema, *args): checks, consequences = super().launch_check(engine) add_check( engine, checks, self.check_class(workspace_schema=workspace_schema, **self.kwargs), ) return checks, consequences class CheckAlliancesFor(CheckFor): def __init__(self, check_class, management_schema, **kwargs): super().__init__( f"SELECT id FROM {management_schema}.alliance", management_schema=management_schema, **kwargs, ) self.check_class = check_class def launch_check(self, engine, alliance_schema, *args): checks, consequences = super().launch_check(engine) add_check( engine, checks, self.check_class(alliance_schema=alliance_schema, **self.kwargs), ) return checks, consequences class CollectionCheck(Check): """Represents general class that can be accessed through collection_i""" def __init__( self, schema, consequence_class, id_field_name="parent_id", parent_id_filter=None, collection_type_filter=None, **kwargs, ): super().__init__(**kwargs) self.parent_id_filter = parent_id_filter self.id_field_name = id_field_name self.collection_type_filter = collection_type_filter self.consequence_class = consequence_class self.schema = schema def __repr__(self): return f"{self.__class__.__name__}(collection_ids={self.kwargs['parent_id_filter']})" def _get_parent_id_filter(self): if self.parent_id_filter is not None: if isinstance(self.parent_id_filter, list): ids = ", ".join([str(int(x)) for x in self.parent_id_filter]) else: ids = str(int(self.parent_id_filter)) return f"AND {self.id_field_name} IN ({ids})" else: return "" def _get_collection_type_filter(self): if self.collection_type_filter is not None: return f"AND collection_type = '{self.collection_type_filter}'" else: return "" def get_collection_data(self): query = f""" SELECT DISTINCT id collection_id FROM {self.schema}.collection WHERE 1 = 1 {self._get_parent_id_filter()} {self._get_collection_type_filter()} ORDER BY 1 DESC; """ df = run_query(query_sql=query, return_type="df") if df is None or df.empty: return [] else: return list(df["collection_id"]) def run(self, engine): checks, consequences = super().run(engine) data = self.get_collection_data() if data: consequences.append( self.consequence_class( schema=self.schema, collection_ids=data, **self.kwargs ) ) return checks, consequences class SetCollectionDeleteCheck(CollectionCheck): """Temporary sets created on filtering, but now also can be sources for audiences.""" def __init__( self, schema, collection_ids, id_field_name="parent_id", consequence_class=FilteringSetDeletedConsequence, collection_type_filter="set", **kwargs, ): super().__init__( schema=schema, consequence_class=consequence_class, id_field_name=id_field_name, parent_id_filter=collection_ids, collection_type_filter=collection_type_filter, **kwargs, ) def run(self, engine): checks, consequences = super().run(engine) for res in consequences: add_check( engine, checks, AudienceCollectionDeleteCheck( self.schema, collection_ids=res.params["collection_ids"], **self.kwargs, ), ) return checks, consequences class SegmentCollectionDeleteCheck(CollectionCheck): def __init__( self, schema, collection_ids, id_field_name="parent_id", consequence_class=CollectionDeletedConsequence, collection_type_filter="segment", **kwargs, ): super().__init__( schema=schema, consequence_class=consequence_class, id_field_name=id_field_name, parent_id_filter=collection_ids, collection_type_filter=collection_type_filter, **kwargs, ) def run(self, engine): checks, consequences = super().run(engine) for res in consequences: add_check( engine, checks, AudienceCollectionDeleteCheck( self.schema, collection_ids=res.params["collection_ids"], **self.kwargs, ), ) add_check( engine, checks, SetCollectionDeleteCheck( self.schema, collection_ids=res.params["collection_ids"], **self.kwargs, ), ) return checks, consequences class SourceCollectionDeleteCheck(CollectionCheck): def __init__( self, schema, collection_ids, consequence_class=CollectionDeletedConsequence, check_class=SegmentCollectionDeleteCheck, **kwargs, ): super().__init__( schema=schema, consequence_class=consequence_class, id_field_name="id", parent_id_filter=collection_ids, collection_type_filter="source", **kwargs, ) self.check_class = check_class self.schema = schema def run(self, engine): check_list, consequences = super().run(engine) for res in consequences: add_check( engine, check_list, AudienceCollectionDeleteCheck( self.schema, collection_ids=res.params["collection_ids"], **self.kwargs, ), ) add_check( engine, check_list, SetCollectionDeleteCheck( self.schema, collection_ids=res.params["collection_ids"], **self.kwargs, ), ) add_check( engine, check_list, self.check_class( self.schema, collection_ids=res.params["collection_ids"], done_by_parent=True, **self.kwargs, ), ) return check_list, consequences class WorkspaceSegmentDeleteCheck(SegmentCollectionDeleteCheck): def __init__( self, workspace_schema, collection_ids, consequence_class=WorkspaceSegmentDeletedConsequence, **kwargs, ): super().__init__( workspace_schema, collection_ids, consequence_class=consequence_class, **kwargs, ) class WorkspaceSegmentByIdDeleteCheck(SegmentCollectionDeleteCheck): def __init__( self, workspace_schema, collection_ids, consequence_class=WorkspaceSegmentDeletedConsequence, **kwargs, ): super().__init__( workspace_schema, collection_ids, id_field_name="id", consequence_class=consequence_class, **kwargs, ) class AllianceSegmentDeleteCheck(SegmentCollectionDeleteCheck): def __init__( self, alliance_schema, collection_ids, consequence_class=AllianceSegmentDeletedConsequence, **kwargs, ): super().__init__( alliance_schema, collection_ids, consequence_class=consequence_class, **kwargs, ) class WorkspaceCollectionDeleteCheck(SourceCollectionDeleteCheck): def __init__( self, workspace_schema, collection_ids, consequence_class=WorkspaceSourceDeletedConsequence, check_class=WorkspaceSegmentDeleteCheck, **kwargs, ): super().__init__( workspace_schema, collection_ids, consequence_class=consequence_class, check_class=check_class, **kwargs, ) def run(self, engine): checks, consequences = super().run(engine) # Launch Alliance checks for cons in consequences: for cid in cons.params["collection_ids"]: add_check( engine, checks, WorkspaceSupersetCollectionDeleteCheck( self.schema, collection_id=cid, **self.kwargs ), ) add_check( engine, checks, CheckAlliancesFor( AllianceSourceImportedFromCheck, workspace_schema=cons.params["schema"], collection_ids=cons.params["collection_ids"], **self.kwargs, ), ) return checks, consequences class AllianceCollectionDeleteCheck(SourceCollectionDeleteCheck): def __init__( self, alliance_schema, collection_ids, consequence_class=AllianceSourceDeletedConsequence, check_class=AllianceSegmentDeleteCheck, **kwargs, ): super().__init__( alliance_schema, collection_ids, consequence_class=consequence_class, check_class=check_class, **kwargs, ) def run(self, engine): check_list, consequences = super().run(engine) for res in consequences: for cid in ( res.params["collection_ids"] if isinstance(res.params["collection_ids"], Iterable) else [res.params["collection_ids"]] ): add_check( engine, check_list, AllianceSupersetCollectionDeleteCheck( self.schema, collection_id=cid, **self.kwargs ), ) return check_list, consequences class SupersetSegmentDeleteCheck(SegmentCollectionDeleteCheck): def __init__( self, schema, collection_ids, consequence_class=SupersetSegmentDeletedConsequence, **kwargs, ): super().__init__( schema, collection_ids, consequence_class=consequence_class, **kwargs ) class SupersetCollectionDeleteCheck(CheckFor): def __init__(self, schema, collection_id, **kwargs): super().__init__( f"SELECT c.id FROM {schema}.collection c " f"JOIN {schema}.set_collection sc ON c.id = sc.set_id " f"WHERE c.collection_type = 'segment' and c.parent_id is NULL " f"AND sc.collection_id = %(collection_id)s " f"ORDER BY 1", collection_id=collection_id, **kwargs, ) self.schema = schema class WorkspaceSupersetCollectionDeleteCheck(SupersetCollectionDeleteCheck): def launch_check(self, engine, cid, *args): checks, consequences = super().launch_check(engine) try: consequences.append( SupersetDeletedConsequence( schema=self.schema, collection_ids=[cid], **self.kwargs ) ) add_check( engine, checks, SupersetSegmentDeleteCheck( self.schema, collection_ids=cid, **self.kwargs ), ) add_check( engine, checks, AudienceCollectionDeleteCheck( self.schema, collection_ids=cid, **self.kwargs ), ) add_check( engine, checks, SetCollectionDeleteCheck( self.schema, collection_ids=cid, **self.kwargs ), ) except Exception as e: logger.exception( "WorkspaceSupersetCollectionDeleteCheck.launch_check couldn't handle collection.source", exc_info=e, ) return checks, consequences class AllianceSupersetCollectionDeleteCheck(SupersetCollectionDeleteCheck): def launch_check(self, engine, cid, *args): checks, consequences = super().launch_check(engine) try: add_check( engine, checks, SupersetSegmentDeleteCheck( self.schema, collection_ids=cid, done_by_parent=True, **self.kwargs ), ) add_check( engine, checks, AudienceCollectionDeleteCheck( self.schema, collection_ids=cid, **self.kwargs ), ) add_check( engine, checks, SetCollectionDeleteCheck( self.schema, collection_ids=cid, **self.kwargs ), ) except Exception as e: logger.exception( "AllianceSupersetCollectionDeleteCheck.launch_check couldn't handle collection.source", exc_info=e, ) return checks, consequences class AudienceCollectionDeleteCheck(CollectionCheck): """Audience collection is created in the source schema as a placeholder for all the profiles that go into actual exported audience """ def __init__(self, schema, collection_ids, id_field_name="parent_id", **kwargs): super().__init__( schema, consequence_class=AudienceCollectionDeletedConsequence, collection_type_filter="audience", parent_id_filter=collection_ids, id_field_name=id_field_name, **kwargs, ) def run(self, engine): check_list, consequences = super().run(engine) for res in consequences: for collection_id in res.params["collection_ids"]: add_check( engine, check_list, FacebookAudienceDeleteCheck( self.schema, audience_id=collection_id, **self.kwargs ), ) return check_list, consequences class FacebookAudienceDeleteCheck(Check): def __init__( self, schema, audience_id, id_field_name="parent_id is NULL AND collection_id", **kwargs, ): super().__init__(**kwargs) self.audience_id_filter = audience_id self.id_field_name = id_field_name self.schema = schema def __repr__(self): return f"{self.__class__.__name__}(collection_id={self.audience_id_filter})" def _get_id_filter(self): if self.audience_id_filter is not None: if isinstance(self.audience_id_filter, list): ids = ", ".join([str(int(x)) for x in self.audience_id_filter]) else: ids = str(int(self.audience_id_filter)) return f"AND {self.id_field_name} IN ({ids})" else: return "" def _get_adaccount_data(self) -> list: query = f""" SELECT DISTINCT id internal_id, external_id FROM {self.schema}.fb_audience WHERE 1=1 {self._get_id_filter()} ORDER BY 1, 2; """ df = run_query(query_sql=query, return_type="df") if df is None or df.empty: return [] else: return df.to_dict("records") def run(self, engine): check_list, consequences = super().run(engine) for acid in self._get_adaccount_data(): consequences.append( AudienceDeletedConsequence( schema=self.schema, audience_internal_id=acid["internal_id"], **self.kwargs, ) ) if not isinstance(self, FacebookAudienceLookalikeDeleteCheck): add_check( engine, check_list, FacebookAudienceLookalikeDeleteCheck( self.schema, acid["internal_id"], **self.kwargs ), ) add_check( engine, check_list, FacebookAudienceUnShareCheck( self.schema, audience_internal_id=acid["internal_id"], **self.kwargs ), ) return check_list, consequences class FacebookAudienceByIdDeleteCheck(FacebookAudienceDeleteCheck): def __init__(self, schema, audience_id, id_field_name="id", **kwargs): super().__init__(schema, audience_id, id_field_name=id_field_name, **kwargs) class FacebookAudienceByMemberDeleteCheck(FacebookAudienceDeleteCheck): def __init__(self, schema, member_id, id_field_name="user_id", **kwargs): super().__init__(schema, member_id, id_field_name=id_field_name, **kwargs) class FacebookAudienceLookalikeDeleteCheck(FacebookAudienceDeleteCheck): def __init__( self, schema, parent_audience_id, id_field_name="parent_id::integer", **kwargs ): super().__init__( schema, parent_audience_id, id_field_name=id_field_name, **kwargs ) class FacebookAudienceUnShareCheck(Check): def __init__(self, schema, audience_internal_id, **kwargs): super().__init__(**kwargs) self.audience_internal_id = audience_internal_id self.schema = schema def _get_audience_shared_state(self): query = f""" SELECT audience_id, adaccount_id FROM {self.schema}.fb_audience_shared_state WHERE audience_id IN ({self.audience_internal_id}) ORDER BY 1, 2; """ df = run_query(query_sql=query, return_type="df") if df is None: return [] else: return df.to_dict("records") def run(self, engine): check_list, consequences = super().run(engine) consequences.extend( AudienceUnsharedConsequence( schema=self.schema, audience_internal_id=acid["audience_id"], adaccount_id=acid["adaccount_id"], **self.kwargs, ) for acid in self._get_audience_shared_state() ) return check_list, consequences class AllianceSourceImportedFromCheck(CheckFor): def __init__(self, alliance_schema, workspace_schema, collection_ids, **kwargs): super().__init__( f"SELECT id, source FROM {alliance_schema}.collection " f"WHERE source LIKE concat(%(workspace_schema)s,'/%%') " f"ORDER BY 1", workspace_schema=workspace_schema, alliance_schema=alliance_schema, **kwargs, ) self.collection_ids = ( collection_ids if isinstance(collection_ids, list) else [collection_ids] ) def launch_check(self, engine, cid, source, *args): checks, consequences = super().launch_check(engine) try: if int(source.split("/")[1]) in self.collection_ids: add_check( engine, checks, AllianceCollectionDeleteCheck(collection_ids=cid, **self.kwargs), ) except Exception as e: logger.exception( "AllianceSourceImportedFromCheck.launch_check couldn't handle collection.source", exc_info=e, ) return checks, consequences class AllianceSourceImportedByCheck(CheckFor): def __init__(self, alliance_schema, member_id, **kwargs): super().__init__( f"SELECT id, source FROM {alliance_schema}.collection " f"WHERE source LIKE concat('%%/',%(member_id)s) " f"ORDER BY 1", alliance_schema=alliance_schema, member_id=member_id, **kwargs, ) self.all_ids = [] def launch_check(self, engine, cid, source, *args): self.all_ids.append(cid) return super().launch_check(engine) def run(self, engine): checks, consequences = super().run(engine) if self.all_ids: add_check( engine, checks, AllianceCollectionDeleteCheck( collection_ids=self.all_ids, **self.kwargs ), ) return checks, consequences class AllianceDeleteCheck(Check): def __init__(self, alliance_schema, management_schema, **kwargs): super().__init__( alliance_schema=alliance_schema, management_schema=management_schema, **kwargs, ) def run(self, engine): checks, consequences = super().run(engine) add_check( engine, checks, AllianceMemberRemoveCheck(member_id=None, **self.kwargs) ) consequences.append(AllianceDeletedConsequence(**self.kwargs)) return checks, consequences class WorkspaceDeleteCheck(CheckFor): """List all the parentless collections and launch appropriate deletion checks""" def __init__(self, workspace_schema, management_schema, **kwargs): if workspace_schema == management_schema: raise UndeletableObject("Can't delete first workspace") super().__init__( f"SELECT id, collection_type FROM {workspace_schema}.collection " f"WHERE parent_id is NULL " f"ORDER BY 1", workspace_schema=workspace_schema, management_schema=management_schema, **kwargs, ) self.coll_map = {} def launch_check(self, engine, cid, ctype, *args): self.coll_map.setdefault(ctype, []).append(cid) return super().launch_check(engine) def run(self, engine): checks, consequences = super().run(engine) if "source" in self.coll_map: add_check( engine, checks, WorkspaceCollectionDeleteCheck( collection_ids=self.coll_map["source"], **self.kwargs ), ) if "segment" in self.coll_map: add_check( engine, checks, WorkspaceSegmentByIdDeleteCheck( collection_ids=self.coll_map["segment"], **self.kwargs ), ) consequences.append(WorkspaceDeletedConsequence(**self.kwargs)) return checks, consequences class WorkspaceEmptyCheck(CheckFor): """List all the parentless collections and launch appropriate deletion checks""" def __init__(self, workspace_schema, management_schema, **kwargs): super().__init__( f"SELECT id, collection_type FROM {workspace_schema}.collection " f"WHERE parent_id is NULL " f"ORDER BY 1", workspace_schema=workspace_schema, management_schema=management_schema, **kwargs, ) self.coll_map = {} def launch_check(self, engine, cid, ctype, *args): self.coll_map.setdefault(ctype, []).append(cid) return super().launch_check(engine) def run(self, engine): checks, consequences = super().run(engine) if "source" in self.coll_map: add_check( engine, checks, WorkspaceCollectionDeleteCheck( collection_ids=self.coll_map["source"], **self.kwargs ), ) if "segment" in self.coll_map: add_check( engine, checks, WorkspaceSegmentByIdDeleteCheck( collection_ids=self.coll_map["segment"], **self.kwargs ), ) return checks, consequences class AllianceMemberRemoveCheck(CheckFor): def __init__(self, alliance_schema, member_id, management_schema, **kwargs): where_member = ( "AND user_id LIKE concat(%(member_id)s,'%%')" if member_id is not None else "" ) super().__init__( f"SELECT user_id member_id, id alliance_schema FROM {management_schema}.user_roles " f"WHERE id = %(alliance_schema)s AND company_alliance_workspace = %(caw)s " f"{where_member}", management_schema=management_schema, alliance_schema=alliance_schema, caw="alliance", member_id=member_id, **kwargs, ) def launch_check(self, engine, member_id, alliance_schema, *args): checks, consequences = super().launch_check(engine) kw = exclude_keys(self.kwargs, "member_id") add_check( engine, checks, AllianceSourceImportedByCheck(member_id=member_id, **kw) ) consequences.append(AllianceMemberRemovedConsequence(member_id=member_id, **kw)) return checks, consequences