"""Model and functions to work with date for product in a particular store.""" from datetime import datetime from datetime import timedelta from oto import response from oto import status from sqlalchemy import and_ from sqlalchemy import Boolean from sqlalchemy import Column from sqlalchemy import DateTime from sqlalchemy import Enum from sqlalchemy import ForeignKey from sqlalchemy import func from sqlalchemy import not_ from sqlalchemy import or_ from sqlalchemy import orm from sqlalchemy import String from sqlalchemy import UniqueConstraint from sqlalchemy.dialects import mysql from availability import debug as availability_debug from availability.connectors import loggly from availability.connectors import sql from availability.constants import error from availability.constants import field_const from availability.constants import models from availability.models import error_handlers from availability.models import product as product_model from availability.models import store as store_model logger = loggly.get_current_logger() class ProductInStore(sql.BaseModel): """Class representing the product_in_store table. Table that stores data about received product, its status, store to which this product was delivered, information about the type of the polling and dates to determine polling start and end dates. """ __tablename__ = models.PRODUCT_IN_STORE_TABLE __table_args__ = (UniqueConstraint( 'store_id', 'product_id', name='unique_product_in_store'),) product_in_store_id = Column( 'id', mysql.INTEGER(unsigned=True), primary_key=True, autoincrement=True) store_id = Column( mysql.INTEGER(unsigned=True), ForeignKey('store.id'), nullable=False) product_id = Column( mysql.INTEGER(unsigned=True), ForeignKey('product.product_id'), nullable=False) store_internal_id = Column(String(45), nullable=False) force_polling = Column(Boolean, default=False, nullable=False) status = Column( Enum(*models.RELEASE_STATUSES_ENUM), default=models.RELEASE_STATUS_DELIVERED) store_internal_status = Column(String(45), nullable=False) received_for_polling_date = Column( DateTime, default=datetime.utcnow, nullable=False) delivery_date = Column( DateTime, default=datetime.utcnow, nullable=False) sales_start_date = Column(DateTime, nullable=False) # ISO 3166-1 alpha-2 country codes _countries = Column('countries', String(1024), default='', nullable=False) go_live_date = Column(DateTime, nullable=True) @property def countries(self): """Public access to the countries field. Read 'countries' field, separate values by commas and return sorted result. Returns: list: A countries list. """ if self._countries: return sorted(self._countries.split(models.COUNTRIES_SEPARATOR)) return [] @countries.setter def countries(self, countries): """Setter for the countries field. Convert a list of country codes to a string and assign to a protected field. Args: countries (iterable): An iterable with country codes. """ # We do not want to accept a string here (which also happens to # be an iterable). Not checking this could lead to a garbage being # silently written to DB. assert not isinstance(countries, str) if countries: self._countries = models.COUNTRIES_SEPARATOR.join( sorted(countries)) else: self._countries = '' countries = orm.synonym('_countries', descriptor=countries) def as_dict(self): """Return dictionary representation of the product_in_store data. Returns: dict: ProductInStore data as a dict. """ product_in_store_dict = { field_const.PRODUCT_IN_STORE_ID: self.product_in_store_id, field_const.STORE_ID: self.store_id, field_const.PRODUCT_ID: self.product_id, field_const.STORE_INTERNAL_ID: self.store_internal_id, field_const.FORCE_POLLING: self.force_polling, field_const.STATUS: self.status, field_const.STORE_INTERNAL_STATUS: self.store_internal_status, field_const.RECEIVED_FOR_POLLING_DATE: self.received_for_polling_date, field_const.DELIVERY_DATE: self.delivery_date, field_const.SALES_START_DATE: self.sales_start_date, field_const.COUNTRIES: self.countries, field_const.GO_LIVE_DATE: self.go_live_date, } return product_in_store_dict def get_products_to_poll_query( store_id, any_task_status=False, current_date_override=None): """Get query with products that are suitable for polling. Args: store_id (int): Internal ID of the store. any_task_status (bool): If this is set, do not filter on related task status. Filter on tasks in models.TASK_STATUS_OK status otherwise. current_date_override (date): Date to use instead of current date during the selection of releases for polling. Returns: sqlalchemy.orm.query.Query: query object suitable for further joining, filtering or result retrieval. """ # Import task module here to avoid recursive import issues. from availability.models import task with sql.session_scope() as session: if current_date_override is not None: # Construct an SQL statement to use instead of 'NOW()' if the # current date is overriden. delta = datetime.utcnow() - current_date_override now = func.SUBDATE(func.NOW(), delta.days) else: now = func.NOW() # Important note: we use messages with Product's orchard_product_id # and upc fields to query store. It is enough for iTunes and Spotify, # but in the future we might need to store another store-specific ID in # the ProductInStore model and put them to the queue instead. query = session.query( ProductInStore.product_in_store_id.label( field_const.PRODUCT_IN_STORE_ID), store_model.Store.store_id.label(field_const.STORE_ID), product_model.Product.upc.label(field_const.UPC), product_model.Product.orchard_product_id.label( field_const.ORCHARD_PRODUCT_ID), task.Task.task_id.label(field_const.TASK_ID), ProductInStore.countries.label(field_const.COUNTRIES), ) query = query.join(store_model.Store).join(product_model.Product) query = query.join(task.Task) query = query.yield_per(models.PRODUCT_TO_POLL_BATCH_SIZE) query = query.enable_eagerloads(False) query = query.filter(store_model.Store.store_id == store_id) if not any_task_status: query = query.filter(task.Task.status == models.TASK_STATUS_OK) query = query.filter( ProductInStore.status != models.RELEASE_STATUS_INGESTION_FAILED, ProductInStore.store_internal_id == '') latest_valid_received_for_polling_date = func.SUBDATE( now, store_model.Store.polling_delay_days) latest_valid_for_polling_after_sales_date = func.SUBDATE( now, store_model.Store.poll_days_after_sales) not_force_polling_filter = and_( not_(ProductInStore.force_polling), (ProductInStore.received_for_polling_date <= latest_valid_received_for_polling_date), (ProductInStore.sales_start_date >= latest_valid_for_polling_after_sales_date)) force_polling_filter = and_( ProductInStore.force_polling, (ProductInStore.received_for_polling_date >= latest_valid_for_polling_after_sales_date)) query = query.filter( or_(not_force_polling_filter, force_polling_filter)) sql_str = availability_debug.render_query(query) logger.info( 'Constructed SQL query', resources=dict( sql_str=sql_str, store_id=store_id, any_task_status=any_task_status, current_date_override=current_date_override)) return query def products_to_poll(store_id, **kwargs): """Create a generator to yield products that are suitable for polling. Args: store_id (int): Internal ID of the store for which should be yielded products which are suitable for polling. Yields: dict: Dict with 'product_in_store_id', 'orchard_product_id', 'store_id' and 'upc' keys. Raises: StopIteration: There are no more items to yield. """ logger.info( 'Retrieving products to poll', resources=dict(store_id=store_id, **kwargs)) num_products = 0 for row in get_products_to_poll_query(store_id, **kwargs): num_products += 1 yield dict(zip(row.keys(), row)) logger.info( 'Finished iterating over products to poll', resources=dict(store_id=store_id, num_products=num_products, **kwargs)) @error_handlers.sqlalchemy_error_handler @error_handlers.integrity_error_handler( error_code=error.ERROR_CODE_MODEL_VALIDATION) def update_release_status(product_in_store_id, store_id, release_status): """Update ProductInStore entry with provided release status data. Args: product_in_store_id (int): ProductInStore PK. store_id (int): Store PK. release_status (availability.datastructures.ReleaseStatus): information provided by one of stores. Returns: response.Response: .message on success, .errors on failure. """ release_data = release_status.as_dict() with sql.session_scope() as session: store_product = ( session.query(ProductInStore) .with_for_update() .filter( ProductInStore.product_in_store_id == product_in_store_id, ProductInStore.store_id == store_id) .first() ) store_release_status = release_data['store_release_status'] or '' store_release_id = release_data['store_release_id'] or '' live_countries = release_data['live_countries'] store_product.store_internal_status = store_release_status store_product.countries = live_countries if not store_product.store_internal_id: store_product.store_internal_id = store_release_id if release_not_live(release_data) and ttl_exceeded(store_product): store_product.status = models.RELEASE_STATUS_INGESTION_FAILED else: store_product.status = release_data['db_release_status'] if (store_product.go_live_date is None and store_product.status == models.RELEASE_STATUS_LIVE): store_product.go_live_date = datetime.utcnow() return response.Response(status=status.NO_CONTENT) @error_handlers.sqlalchemy_error_handler def exists_in_store(product_id, store_id): """Check whether product for given store already exists in the database. Args: product_id (int): Product PK. store_id (int): Store PK Returns: bool: indicates whether product exists or not. """ with sql.session_scope() as session: product = ( session .query(ProductInStore) .join(product_model.Product) .filter( ProductInStore.product_id == product_id, ProductInStore.store_id == store_id) .first()) return bool(product) @error_handlers.sqlalchemy_error_handler def list_store_statuses_for_products(product_ids): """List store status information using provided product_ids. Args: product_ids (iterable): An iterable of Product PKs as integers. Returns: oto.response.Response: .message with list of ProductInStore dicts on success, .errors on failure. """ with sql.session_scope() as session: products = ( session .query(ProductInStore) .filter(ProductInStore.product_id.in_(product_ids)) .all()) if not products: return response.create_not_found_response() result = [] for product in products: # Non-datetime values (e.g. MySQL zero-datetime strings) can't be # coerced to a valid Date by downstream callers; hand back None. if isinstance(product.go_live_date, datetime): go_live_date = product.go_live_date.strftime(models.ISO8601_DATE_FORMAT) else: go_live_date = None result.append({ field_const.PRODUCT_ID: product.product_id, field_const.STORE_ID: product.store_id, field_const.STATUS: product.status, field_const.COUNTRIES: product.countries, field_const.GO_LIVE_DATE: go_live_date, field_const.STORE_INTERNAL_ID: product.store_internal_id }) return response.Response(result) def ttl_exceeded(store_product): """Check if release_status should be set to failed because of exceeded ttl. Args: store_product (ProductInStore): object which status to check. Returns: bool: Indicates whether ttl exceeded or not. """ store = store_model.get_store_by_id(store_product.store_id).message days_after_sales = timedelta(days=store[field_const.POLL_DAYS_AFTER_SALES]) if store_product.force_polling: ttl_date = store_product.received_for_polling_date + days_after_sales else: ttl_date = store_product.sales_start_date + days_after_sales return ttl_date.date() <= datetime.utcnow().date() def release_not_live(release_data): """Check whether release status is live. Args: release_data (dict): ReleaseStatus.as_dict(). Returns: bool: Indicates whether release is live. """ return release_data['db_release_status'] != models.RELEASE_STATUS_LIVE def create_product_store_link(data): """Create product store link. Args: data (dict): Returns: dict """ with sql.session_scope() as session: product_in_store = ProductInStore( store_internal_id=data.get('store_internal_id'), store_internal_status=data.get('store_internal_status'), store_id=data.get('store_id'), product_id=data.get('product_id'), force_polling=data.get('force_polling'), status=data.get('status'), received_for_polling_date=data.get('received_for_polling_date'), delivery_date=data.get('delivery_date'), sales_start_date=data.get('sales_start_date'), _countries=data.get('countries'), go_live_date=data.get('go_live_date'), ) session.add(product_in_store) session.flush() return product_in_store.as_dict()