import gzip
import json
import unittest
from datetime import datetime
from unittest.mock import call

import pytest
from requests import Response

from slz_appreciationengine_scrapper.const import RATE_LIMIT_HOUR_HEADER
from slz_appreciationengine_scrapper.content_status_service import ContentStatusService
from slz_appreciationengine_scrapper.dsp.appreciationengine.clients import PaginateClientPartial
from slz_appreciationengine_scrapper.dsp.entities import AEDateTimeRange, MetaDataKey
from slz_appreciationengine_scrapper.entities import Job


def resp_mock_activityfeed_us_columbia_partial_timeframe():
    data = [
        {
            'totalSize': 1000,
            'items': [{
                'RecordDate': '2020-11-04 23:17:16'
            }, ]
        },
        # request to check filtering rule
        {
            'totalSize': 1000,
            'items': [{
                'RecordDate': '2020-11-04 23:17:16'
            }, ]
        },
        {
            'totalSize':
                1000,
            'items':
                [
                    {
                        'RecordDate': '2020-11-04 23:17:17'
                    },
                    {
                        'RecordDate': '2020-11-04 23:17:18'
                    },
                    {
                        'RecordDate': '2020-11-04 23:17:19'
                    },
                ]
        },
        {
            'totalSize':
                1000,
            'items':
                [
                    {
                        'RecordDate': '2020-11-04 23:17:20'
                    },
                    {
                        'RecordDate': '2020-11-04 23:17:21'
                    },
                    {
                        'RecordDate': '2020-11-04 23:17:22'
                    },
                    {
                        'RecordDate': '2020-11-04 23:17:23'
                    },
                    {
                        'RecordDate': '2020-11-04 23:17:25'
                    },
                ]
        },
        {
            'totalSize': 1,
            'items': [{
                'RecordDate': '2020-11-04 23:17:27'
            }]
        }
    ]
    rate_limit = 7
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = bytes(json.dumps(dt), encoding='utf8')
        resp_mock.headers = {RATE_LIMIT_HOUR_HEADER: rate_limit}
        resp_mock.status_code = 200
        rate_limit -= 1
        yield resp_mock


@pytest.mark.integration
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(year=2021, month=6, day=5, hour=1, minute=1, second=1)
)
def test_downloading_paginate_partial_timeframe(
    datetime_mock,
    requests_get,
    get_secret,
    patched_time_sleep,
    db,
    params,
    s3_client,
    ae_mapping,
    logger_test,
):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-activityfeed-v1',
        'unit_of_work_id': 0,
        'dsp': 'appreciationengine',
        'report_type': 'activityfeed',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'US_Columbia',
    }

    # Client initialization
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger_test)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    expected_file_content = ''.join(
        [
            '{"RecordDate": "2020-11-04 23:17:16"}\n',
            '{"RecordDate": "2020-11-04 23:17:17"}\n',
            '{"RecordDate": "2020-11-04 23:17:18"}\n',
            '{"RecordDate": "2020-11-04 23:17:19"}\n',
            '{"RecordDate": "2020-11-04 23:17:20"}\n',
            '{"RecordDate": "2020-11-04 23:17:21"}\n',
            '{"RecordDate": "2020-11-04 23:17:22"}\n',
            '{"RecordDate": "2020-11-04 23:17:23"}\n',
        ]
    )
    # Mock response from source API
    requests_get.return_value.__enter__.side_effect = resp_mock_activityfeed_us_columbia_partial_timeframe(
    )

    content_status_service = ContentStatusService(logger=logger_test, db_conn=db)
    params.scrapper.min_rate_limit_remaining_hour = 5
    meta, meta_data, io_exception = client.download(
        job,
        chunk_size=1,
        timer=timer,
        timeslot=AEDateTimeRange(
            lower=datetime(year=2021, month=1, day=1, hour=1),
            upper=datetime(year=2021, month=1, day=1, hour=6),
        ),
        content_status_service=content_status_service
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/activityfeed/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/US_Columbia_20201104_010000_060000_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/US_Columbia_20201104_010000_060000_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')
    assert meta_data[MetaDataKey.NEW_START_DATE.value] == "2020-11-04 23:17:24"


def resp_mock_activityfeed_us_columbia_partial_small_resp_timeframe():
    data = [
        {
            'totalSize': 1,
            'items': [{
                'RecordDate': '2020-11-04 00:31:16'
            }, ]
        },
        # request to check filtering rule
        {
            'totalSize': 1,
            'items': [{
                'RecordDate': '2020-11-04 00:31:16'
            }, ]
        },
        {
            'totalSize': 1,
            'items': [{
                'RecordDate': '2020-11-04 03:31:16'
            }, ]
        },
        {
            'totalSize': 1,
            'items': [{
                'RecordDate': '2020-11-04 04:31:16'
            }, ]
        },
        {
            'totalSize': 0,
            'items': []
        }
    ]
    rate_limit = 7
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = bytes(json.dumps(dt), encoding='utf8')
        resp_mock.headers = {RATE_LIMIT_HOUR_HEADER: rate_limit}
        resp_mock.status_code = 200
        rate_limit -= 1
        yield resp_mock


@pytest.mark.integration
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(year=2021, month=6, day=5, hour=1, minute=1, second=1)
)
def test_downloading_paginate_partial_timeframe_small_resp(
    datetime_mock, requests_get, get_secret, patched_time_sleep, db, params, s3_client, ae_mapping,
    logger_test
):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-activityfeed-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activityfeed',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'US_Columbia',
    }

    # Client initialization
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger_test)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    expected_file_content = ''.join(
        [
            '{"RecordDate": "2020-11-04 00:31:16"}\n',
            '{"RecordDate": "2020-11-04 03:31:16"}\n',
            '{"RecordDate": "2020-11-04 04:31:16"}\n',
        ]
    )
    # Mock response from source API
    requests_get.return_value.__enter__.side_effect = \
        resp_mock_activityfeed_us_columbia_partial_small_resp_timeframe()

    params.scrapper.min_rate_limit_remaining_hour = 5
    meta_data_initial = {MetaDataKey.NEW_START_DATE.value: "2020-11-04 00:30:01"}
    content_status_service = ContentStatusService(logger=logger_test, db_conn=db)
    meta, meta_data, io_exception = client.download(
        job,
        chunk_size=1,
        timer=timer,
        meta_data=meta_data_initial,
        timeslot=AEDateTimeRange(
            lower=datetime(year=2020, month=11, day=4, hour=0),
            upper=datetime(year=2020, month=11, day=4, hour=8),
        ),
        content_status_service=content_status_service,
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/activityfeed/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/US_Columbia_20201104_000000_080000_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/US_Columbia_20201104_000000_080000_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')
    assert meta_data[MetaDataKey.PARTIAL_DOWNLOAD_COMPLETED_AT.value] is not None


@pytest.mark.parametrize(
    'timeslot, query_extended', [
        (
            AEDateTimeRange(
                lower=datetime(year=2020, month=11, day=4, hour=0),
                upper=datetime(year=2020, month=11, day=4, hour=8),
            ),
            'extended.lastUpdated>=2020-11-03 23:59:59 and extended.lastUpdated<=2020-11-04 08:00:01'
        ),
        (
            None,
            'extended.lastUpdated>=2020-11-03 23:59:59 and extended.lastUpdated<=2020-11-05 00:00:00'
        ),
    ]
)
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
def test_ae_availability_check_timeframe(
    requests_get, get_secret, patched_time_sleep, timeslot, query_extended, params, logger_test,
    ae_mapping
):
    uow = {
        'uow_id': 'appreciationengine-20201104-sme-membersextended-v1',
        'dsp': 'appreciationengine',
        'report_type': 'membersextended',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    job = Job.from_dict(uow)
    cs_service_mock = unittest.mock.Mock()
    cs_service_mock.get_uow_timeslot.return_value = timeslot
    resp_mock_base = unittest.mock.MagicMock(spec=Response)
    resp_mock_base.ok = True
    dt = {'items': [{'ID': '123'}], 'totalSize': 1}
    resp_mock_base.content = bytes(json.dumps(dt), encoding='utf8')
    resp_mock_base.headers = {}
    resp_mock_base.url = ''
    resp_mock_base.status_code = 200
    resp_mock_base.reason = 'Some text'
    requests_get.return_value.__enter__.return_value = resp_mock_base
    # Mock secrets
    get_secret.return_value = json.dumps({
        'Sony Music - Century Media Records': 'token72739d3',
    })

    client = PaginateClientPartial(logger_test)
    client.configure(params)
    result = client._check_source_is_available(job, cs_service_mock)
    url = 'https://sme-delphi.theappreciationengine.com/v1.1/members/extended'
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'limit': '0,1',
                    'query_extended': query_extended,
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated'
                }
            ),
        ],
        any_order=True
    )
    assert result == ('CenturyMediaRecords', 0)


def resp_mock_activityfeed_us_columbia_partial_small_resp_small_timeframe_rate_limit():
    data = [
        {
            'totalSize': 1000,
            'items': [{
                'RecordDate': '2020-11-04 00:31:16'
            }, ]
        },
        # request to check filtering rule
        {
            'totalSize': 1,
            'items': [{
                'RecordDate': '2020-11-04 00:31:16'
            }, ]
        },
        {
            'totalSize':
                123,
            'items':
                [
                    {
                        'RecordDate': '2020-11-04 03:31:16'
                    },
                    {
                        'RecordDate': '2020-11-04 03:32:16'
                    },
                    {
                        'RecordDate': '2020-11-04 03:33:16'
                    },
                ]
        },
        {
            'totalSize': 12,
            'items': [{
                'RecordDate': '2020-11-04 03:51:16'
            }, ]
        },
        {
            'totalSize': 0,
            'items': []
        }
    ]
    rate_limit = 6
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = bytes(json.dumps(dt), encoding='utf8')
        resp_mock.headers = {RATE_LIMIT_HOUR_HEADER: rate_limit}
        resp_mock.status_code = 200
        rate_limit -= 1
        yield resp_mock


@pytest.mark.integration
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(year=2021, month=6, day=5, hour=1, minute=1, second=1)
)
def test_downloading_paginate_partial_small_resp_small_timeframe_rate_limit(
    datetime_mock, requests_get, get_secret, patched_time_sleep, db, params, s3_client, ae_mapping,
    logger_test
):
    """
    When min_rate_limit_remaining value reached, we should look for a new_start_date and wrap up
    download process. Even in case at iteration where limit was reached, AE returned
    records < page_size.
    """
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-activityfeed-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activityfeed',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'US_Columbia',
    }

    # Client initialization
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger_test)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    expected_file_content = ''.join(
        [
            '{"RecordDate": "2020-11-04 00:31:16"}\n',
            '{"RecordDate": "2020-11-04 03:31:16"}\n',
            '{"RecordDate": "2020-11-04 03:32:16"}\n',
        ]
    )
    # Mock response from source API
    requests_get.return_value.__enter__.side_effect = \
        resp_mock_activityfeed_us_columbia_partial_small_resp_small_timeframe_rate_limit()

    params.scrapper.min_rate_limit_remaining_hour = 5
    meta_data_initial = {MetaDataKey.NEW_START_DATE.value: "2020-11-04 00:30:01"}
    content_status_service = ContentStatusService(logger=logger_test, db_conn=db)
    meta, meta_data, io_exception = client.download(
        job,
        chunk_size=1,
        timer=timer,
        meta_data=meta_data_initial,
        timeslot=AEDateTimeRange(
            lower=datetime(year=2020, month=11, day=4, hour=0),
            upper=datetime(year=2020, month=11, day=4, hour=4),
        ),
        content_status_service=content_status_service
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/activityfeed/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/US_Columbia_20201104_000000_040000_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/US_Columbia_20201104_000000_040000_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')
    assert meta_data[MetaDataKey.NEW_START_DATE.value] == '2020-11-04 03:32:17'