"""Handlers purely used for testing purposes. These are used to compare the pricing JSON output by ows-pricing against the itunes xml generated by VECTOR """ import json from flask import jsonify from flask import request from oto import response from oto.adaptors.flask import flaskify from owsrequest import flask_request from pricing.api import app from pricing.api import cache from pricing.constants.xml_generation_urls import audio_xml_url, \ film_xml_url, new_audio_xml_url, new_film_xml_url from pricing.logic import legacy_product_mapping from pricing.logic import product as product_logic from pricing.logic import product_pricing_override from pricing.logic import product_store_pricing import requests import xmltodict TRACK_PRICING_FAMILY_ID = 3 @app.route('/cache-status', methods=['GET']) def get_cache_status(): """Cache route that check the status of the cache. Returns: JSON with the cache status """ cache_as_dict = cache.__dict__ if cache_as_dict['config'] and \ cache_as_dict['config']['CACHE_TYPE'] != 'null': return jsonify({'status': 'ok', 'cache_type': cache_as_dict['config']['CACHE_TYPE']}) else: return jsonify({'status': 'Cache is null', 'cache_type': cache_as_dict['config']['CACHE_TYPE']}) @app.route('/testing/pricing-family//product//' 'upc//store/', methods=['GET']) def compare_xml_to_json(pricing_family_id, product_id, upc, store_id): """Compare an itunes xml against a pricing JSON. Args: pricing_family_id (string): the ID of the pricing family product_id (string): the ID of the product. upc (string): the UPC of the product. store_id (string): the ID of a store. Returns: json: containing the results of the comparison. """ result = compute_differences_between_json_and_xml( pricing_family_id, product_id, upc, store_id) return jsonify(result) @app.route('/testing/multiple-upcs', methods=['POST']) def compare_json_with_xml_for_multiple_upcs(): """Call diff route multiple times for many different upcs. Returns: The upcs that show different price codes with the iTunes XML """ data = request.get_data() or '' data = data.decode('utf-8') result = [] diffs = 0 mismatch_details = [] upcs_with_mismatches = [] non_parsed_upcs = [] processed_data = [v.strip().split(' ') for v in data.splitlines()] for item in processed_data: pricing_family_id = item[0] store_id = item[1] upc = item[2] product_id = item[3] response_with_diffs = compute_differences_between_json_and_xml( pricing_family_id, product_id, upc, store_id) if 'message' in response_with_diffs: non_parsed_upcs.append(int(upc)) else: response_with_diffs_old_xml = response_with_diffs['old_xml'] response_with_diffs_new_xml = response_with_diffs['new_xml'] old_new_diffs = {} if pricing_family_id == '1': if response_with_diffs_old_xml['sd_mismatches'] or \ response_with_diffs_old_xml['hd_mismatches'] or \ response_with_diffs_old_xml[ 'sd_missing_from_json_count'] or \ response_with_diffs_old_xml[ 'hd_missing_from_json_count']: upcs_with_mismatches.append(upc) diffs += 1 old_new_diffs['old_xml'] = response_with_diffs_old_xml if response_with_diffs_new_xml['sd_mismatches'] or \ response_with_diffs_new_xml['hd_mismatches'] or \ response_with_diffs_new_xml[ 'sd_missing_from_json_count'] or \ response_with_diffs_new_xml[ 'hd_missing_from_json_count']: if upc not in upcs_with_mismatches: upcs_with_mismatches.append(upc) diffs += 1 old_new_diffs['new_xml'] = response_with_diffs_new_xml if old_new_diffs: mismatch_details.append(old_new_diffs) else: if response_with_diffs_old_xml['mismatches'] or \ response_with_diffs_old_xml['missing_from_json_count']\ or response_with_diffs_old_xml['track_differences']: upcs_with_mismatches.append(upc) diffs += 1 old_new_diffs['old_xml'] = response_with_diffs_old_xml if response_with_diffs_new_xml['mismatches'] or \ response_with_diffs_new_xml['missing_from_json_count']\ or response_with_diffs_new_xml['track_differences']: if upc not in upcs_with_mismatches: upcs_with_mismatches.append(upc) diffs += 1 old_new_diffs['new_xml'] = response_with_diffs_new_xml if old_new_diffs: mismatch_details.append(old_new_diffs) result.append( {'differences found': diffs, 'number_of_total_upcs': len(processed_data), 'mismatch_details': mismatch_details, 'mismatched_upcs': upcs_with_mismatches, 'upcs_with_no_xml': non_parsed_upcs, 'number_of_upcs_compared': len(processed_data) - len(non_parsed_upcs)}) return flaskify(response.Response({'items': result})) @app.route( '/product-pricing-override/preview-store-migration', methods=['GET']) def preview_migrate_product_pricing_override_stores(): """Preview Product Pricing Overrides migration to support multiple stores. Returns: flask.Response: containing the product pricing override stores to create """ return flaskify( product_pricing_override. preview_migrate_product_pricing_override_stores()) @app.route( '/product-pricing-override/execute-store-migration', methods=['GET']) def execute_migrate_product_pricing_override_stores(): """Execute Product Pricing Overrides migration to support multiple stores. Returns: flask.Response: containing the created product pricing override stores """ return flaskify( product_pricing_override. execute_migrate_product_pricing_override_stores()) @app.route('/legacy/music-releases', methods=['GET']) def get_legacy_music_releases(): """Get legacy music releases. Returns: flask.Response: containing the releases """ return flaskify(legacy_product_mapping.get_legacy_music_releases()) @app.route('/legacy/music/product//preview', methods=['GET']) def preview_music_migration(product_id): """Preview how we would migrate a music product. Args: product_id (int): the id of the product. Returns: flask.Response: preview of what overrides would be created. """ return flaskify(legacy_product_mapping.preview_music_migration(product_id)) @app.route('/legacy/music/product//execute', methods=['GET']) def execute_music_migration(product_id): """Execute the migration for a music product. Args: product_id (int): the id of the product. Returns: flask.Response: the overrides that have been created. """ return flaskify(legacy_product_mapping.execute_music_migration(product_id)) @app.route('/legacy/music/product//jit', methods=['GET']) def jit_music_migration(product_id): """Just in time migration for a music product. Args: product_id (int): the id of the product. Returns: flask.Response: the result of the migration. """ return flaskify(legacy_product_mapping.jit_music_migration(product_id)) @app.route('/legacy/product//unmigrate', methods=['DELETE']) def unmigrate_product(product_id): """Unmigrate product from pricing database. Args: product_id (int): the product to unmigrate. Returns: flask.Response: containing a dict of what was deleted """ ownership = flask_request.verify_grass_ownership( request, product_logic.is_owner, product_id) if not ownership: return flaskify(ownership) return flaskify(legacy_product_mapping.unmigrate_product(product_id)) @app.route('/legacy/track//unmigrate', methods=['DELETE']) def unmigrate_track(track_id): """Unmigrate track from pricing database. Args: track_id (int): the track to unmigrate. Returns: flask.Response: containing a dict of what was deleted """ return flaskify(legacy_product_mapping.unmigrate_track(track_id)) @app.route('/legacy//product/', methods=['GET']) def map_legacy_product_pricing_data_to_new_structure( pricing_family_id, product_id): """Map the product pricing data to new structure. Args: pricing_family_id (int): the pricing family id product_id (int): the id of the product ingest pricing information. Returns: flask.Response: containing the pricing data that are updated. """ return flaskify(legacy_product_mapping.map_legacy_product_pricing_data( pricing_family_id, product_id)) @app.route('/legacy/bulk/migrate', methods=['GET']) def bulk_migrate_film_products(): """Migrate products to the ows-pricing. Returns: flask.Response: the product ids that have not been migrated """ return flaskify(legacy_product_mapping.bulk_migrate_film_products()) @app.route('/legacy/bulk/count-migration', methods=['GET']) def count_bulk_migrate_film_products(): """Count how many film products are left to migrate. Returns: flask.Response: the count of the products that have not been migrated """ return flaskify(legacy_product_mapping.count_bulk_migrate_film_products()) def compute_differences_between_json_and_xml( pricing_family_id, product_id, upc, store_id): """Compare an itunes xml against a pricing JSON. Args: pricing_family_id (string): the ID of the pricing family product_id (string): the ID of the product. upc (string): the UPC of the product. store_id (string): the ID of a store. Returns: dict: containing the results of the comparison. """ is_film = False if pricing_family_id == '1': is_film = True if is_film: xml_url = film_xml_url.format(upc, store_id) new_xml_url = new_film_xml_url.format(upc, store_id) else: xml_url = audio_xml_url.format(upc, store_id) new_xml_url = new_audio_xml_url.format(upc, store_id) try: xml_dict = parse_xml(xml_url) except Exception as exc: return {'message': 'Could not parse old xml for upc {0}'.format(upc), 'reason': str(exc)} try: new_xml_dict = parse_xml(new_xml_url) except Exception as exc: return {'message': 'Could not parse new xml for upc {0}'.format(upc), 'reason': str(exc)} get_local = True if get_local: response = product_store_pricing.get_product_store_pricing( pricing_family_id, product_id, store_id) json_data = response.message if not is_film: response = product_store_pricing.get_product_store_pricing( TRACK_PRICING_FAMILY_ID, product_id, store_id) tracks_json = response.message['items'] for item in tracks_json: del item['track_ids'] else: json_url = 'https://qa-ows-pricing.theorchard.io/pricing_family/{0}/' \ 'product/{1}/store/{2}/pricing'\ .format(pricing_family_id, product_id, store_id) response = requests.get(json_url) json_data = json.loads(response.text) if not is_film: track_json_url = 'https://qa-ows-pricing.theorchard.io/' \ '/pricing_family/3/product/{0}/store/{1}/pricing' \ .format(product_id, store_id) track_response = requests.get(track_json_url) track_json_data = json.loads(track_response.text) tracks_json = track_json_data['items'] for item in tracks_json: del item['track_ids'] params_for_old_xml = generate_params(product_id, upc) params_for_new_xml = generate_params(product_id, upc) if is_film: product_elements = xml_dict['package']['video']['products']['product'] new_product_elements = new_xml_dict['package']['video']['products'][ 'product'] else: product_elements = xml_dict['package']['album']['products']['product'] track_elements = xml_dict['package']['album']['tracks']['track'] if type(track_elements) is not list: track_elements = [track_elements] new_product_elements = new_xml_dict['package']['album']['products'][ 'product'] new_track_elemenets = xml_dict['package']['album']['tracks']['track'] if type(new_track_elemenets) is not list: new_track_elemenets = [new_track_elemenets] for product in product_elements: compare_json_with_xml_for_single_product( params_for_old_xml, product, json_data, is_film) for product in new_product_elements: compare_json_with_xml_for_single_product( params_for_new_xml, product, json_data, is_film) if not is_film: final_track_pricing = compute_track_pricing(track_elements) final_new_track_pricing = compute_track_pricing(new_track_elemenets) calculate_track_differences( tracks_json, final_track_pricing, params_for_old_xml) calculate_track_differences( tracks_json, final_new_track_pricing, params_for_new_xml) return { 'old_xml': params_for_old_xml, 'new_xml': params_for_new_xml } def compute_track_pricing(track_elements): """Compute the tracks pricing for a product. Args: track_elements (dict): dict with the track elements Returns: an array with the track pricing group by price code, territories and isrc """ tracks_pricing_info = [] for track in track_elements: isrc = track['isrc'] products = track['products']['product'] for product in products: tracks_pricing_info.append( {'territory': product['territory'], 'price_code': product['wholesale_price_tier'], 'isrc': isrc}) return tracks_pricing_info def calculate_track_differences(tracks_json, final_track_pricing, params): """Compare pricing information between itunes xml and ows pricing json. Args: tracks_json (dict): dictionary with ows track pricing info final_track_pricing (dict): itunes XML info params (dict): dict with comparison stats """ for json_item in tracks_json: for xml_item in final_track_pricing: if xml_item['territory'] in json_item['territories'] and \ xml_item['isrc'] in json_item['isrcs']: if xml_item['price_code'] != json_item['price_code']: message = 'Mismatch for XML price code {0} and json ' \ 'price code {1} for territory {2} and ' \ 'isrcs {3}'.format(xml_item['price_code'], json_item['price_code'], xml_item['territory'], xml_item['isrc']) params['track_differences_sources'].append(message) params['track_differences'] += 1 def next_element(resolution, territory, json_data): """Get the next element given the territory and resolution. Args: resolution (string): The resolution code e.g. SD, HD territory (string): The territory code json_data (json): The data as json Returns: A pricing element """ if resolution: return next((element for element in json_data['items'] if territory in element['territories'] and (element['resolution'] == 'All' or element['resolution'] == resolution)), None) else: return next((element for element in json_data['items'] if territory in element['territories']), None) def parse_xml(xml_url): """Generate xml dictionary. Args: xml_url (string): url that generates store pricing xml Returns: dict with the store pricing xml info """ xml = requests.get(xml_url).content xml = xml.decode('utf-8') xml_dict = xmltodict.parse(xml) return xml_dict def compare_json_with_xml_for_single_product( params, product, json_data, is_film): """Compare json with xml for a single product. Args: params (dict): dictionary with comparison stats initialisation product (dict): an xml product with pricing info json_data (dict): the store pricing info generated from new pricing db is_film (bool): defines if the product is film """ itunes_hd_code = 'None' json_sd_code = 'None' json_hd_code = 'None' check_for_hd = False if 'hd_wholesale_price_tier' in product: itunes_hd_code = product['hd_wholesale_price_tier'] check_for_hd = True territory = product['territory'] try: itunes_sd_code = product['wholesale_price_tier'] except Exception: print(product) if is_film: sd_element = next_element('SD', territory, json_data) hd_element = next_element('HD', territory, json_data) if sd_element: json_sd_code = sd_element['price_code'] else: params['sd_missing_from_json_count'] += 1 params['territory_sd_missing_from_json'].append(territory) if hd_element: json_hd_code = hd_element['price_code'] elif check_for_hd: params['hd_missing_from_json_count'] += 1 params['territory_hd_missing_from_json'].append(territory) else: music_element = next_element(None, territory, json_data) if music_element: json_sd_code = music_element['price_code'] else: params['missing_from_json_count'] += 1 params['territory_count'] += 1 if itunes_sd_code != json_sd_code: if is_film: params['sd_mismatch_count'] += 1 message = 'territory: {0} itunes: {1} json: {2}'.format( territory, itunes_sd_code, json_sd_code) params['sd_mismatches'].append(message) else: params['mismatch_count'] += 1 message = 'territory: {0} itunes: {1} json: {2}'.format( territory, itunes_sd_code, json_sd_code) params['mismatches'].append(message) if is_film: if itunes_hd_code != json_hd_code: params['hd_mismatch_count'] += 1 message = 'territory: {0} itunes: {1} json: {2}'.format( territory, itunes_hd_code, json_hd_code) params['hd_mismatches'].append(message) def generate_params(product_id, upc): """Generate dictionary with stats initialisation. Args: product_id (int): the ID of a product upc (int): the upc of a product Returns: a dict with initial values of stats """ params = {} params['territory_count'] = 0 params['sd_mismatch_count'] = 0 params['sd_mismatches'] = [] params['hd_mismatch_count'] = 0 params['hd_mismatches'] = [] params['sd_missing_from_json_count'] = 0 params['territory_sd_missing_from_json'] = [] params['hd_missing_from_json_count'] = 0 params['territory_hd_missing_from_json'] = [] params['product_id'] = product_id params['upc'] = upc params['mismatch_count'] = 0 params['mismatches'] = [] params['missing_from_json_count'] = 0 params['track_differences'] = 0 params['track_differences_sources'] = [] return params