"""Transformation and filtering functions for Alooma pipeline.""" art_relations_table_black_list = [ 'delivery_history_detail', 'track_pricing_tier_country', 'track_pricing_tier', 'release_pricing_tier_country', 'dig_sales', 'processed_dig_sales', 'dig_sales_detail', 'vectorapi_auth_codes', 'trackdown_media_access', 'release_accounting', 'dms_deletions', 'script_status', # no PK 'standout_track', # no PK 'oa_logins', # no PK 'product_physical_reports_change_history', # no PK 'youtube_channel_video_remove_duplicates' # table truncated ] art_relations_schema_black_list = dict( art_relations=art_relations_table_black_list, art_restore='*', cloudsearch='*', compilations='*', prod_maxwell='*', publishing='*', royalty_collections='*', sugarcrm='*', test='*', test1='*') black_list = dict( art_relations_prod=art_relations_schema_black_list, art_relations_prod_dump=art_relations_schema_black_list) events_to_clean = { 'art_relations.track': ['track_name'], 'art_relations.releases': ['vendor_catalog_number', 'release_name'], 'art_relations.artist_info': ['name'], 'art_relations.project': ['project_name', 'project_code'], 'art_relations_log.project_log': ['project_name', 'project_code'], 'art_relations.release_artist': ['artist_name'] } def is_rollback_table(table): """Determine if a table is a rollback table. Args: table (str): table name. Returns: bool: is it rollback table or not """ return table.upper().startswith('ROLLBACK') def is_dbchangelog_table(table): """Determine if a table is a changelog table. Args: table (str): table name. Returns: bool: is it changelog table or not """ return table.upper().startswith('DATABASECHANGELOG') def is_table_black_listed(input_label, schema, table): """Determine if a table is blacklisted. Args: input_label (str): input label from event. schema (str): DB schema name. table (str): DB table name. Returns: bool: is table blacklisted or not. """ if is_rollback_table(table): return True if is_dbchangelog_table(table): return True if input_label not in black_list: return False if schema not in black_list[input_label]: return False if '*' in black_list[input_label][schema]: return True return table in black_list[input_label][schema] def clean_event_values(event): """Remove null byte codes at the start of select event fields. Event passed in is itself cleaned (since it is a dict). Args: dict (dict): Alooma event to clean. Assumes that event['_metadata']['event_type'] = . """ event_type = event['_metadata']['event_type'] if event_type in events_to_clean: for field in events_to_clean[event_type]: # protect against when field is NULL if field in event and event[field] is not None: event[field] = event[field].strip(b'\0') # TODO: add comment why do we need this manual overwrite. DESCRIPTION_OVERRIDES = { 'c831b479-1795-4392-8dff-b1f50cce5123': '

LOSFUOCOS was born in 2002 with the idea of playing some rock and roll and 

\r\n

follow the trail of Scandinavian groups like the Hellacopters, Gluecifer or 

\r\n

Turbonegro. The thing that united us is the desire to create our own sound derived from different musical influences: punk rock, hardcore and death metal. 

\r\n

After many years, we have published a lot of material, we shared the stage 

\r\n

with some of our favorite bands (Firebird, Imperial State Electric, the Dirtbombs, Thunder Express, Michael Davis of MC5, the Muggs) and now we are ready 

\r\n

to start again with a new line-up and new songs between Sixties proto punk 

\r\n

and Seventies hard rock .

\r\n

Stay tuned for more rock and roll!

\r\n

LOSFUOCOS are:

\r\n

PADO – Guitar + Lead Vocals

\r\n

DEN – Bass + Background Vocals

\r\n

GG – Guitar + Lead Vocals

\r\n

FRANK – Drums

' # noqa } def transform(event): """Transform function. Args: event (dict): Alooma event. Returns: dict: Alooma event. """ if event['_metadata']['input_label'] == 'SalesPlatform': event['_metadata']['schema'] = 'SALESPLATFORM_QA' if event['_metadata']['input_label'] == 'qa-ows-podcast': event['_metadata']['schema'] = 'qa_{}'.format( event['_metadata']['schema']) event_uuid = event['_metadata']['@uuid'] if event['_metadata']['input_type'] == 'mongodb': schema = event['_metadata']['db'] else: input_label = event['_metadata']['input_label'] schema = event['_metadata']['schema'] table = event['_metadata']['table'] if is_table_black_listed(input_label, schema, table): return None if event_uuid in DESCRIPTION_OVERRIDES: event['description'] = DESCRIPTION_OVERRIDES[event_uuid] event['_metadata']['event_type'] = '{}.{}'.format( schema, event['_metadata']['event_type']) clean_event_values(event) return event