import gzip
import json
import unittest.mock
from datetime import datetime, timedelta
from typing import Any, Dict

import pytest
from unittest.mock import call
from requests.models import Response

from slz_appreciationengine_scrapper.config import Scrapper
from slz_appreciationengine_scrapper.content_status_service import ContentStatusService
from slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io import AEPaginateStringIOPartial
from slz_appreciationengine_scrapper.dsp.appreciationengine.clients import (
    Client,
    PaginateClient,
    PaginateClientPartial,
)
from slz_appreciationengine_scrapper.dsp.appreciationengine.helpers import hide_secret
from slz_appreciationengine_scrapper.dsp.exceptions import StreamError
from slz_appreciationengine_scrapper.entities import Job


def raise_exception():
    raise StreamError


@pytest.mark.integration
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
def test_downloading(requests_get, get_secret, db, params, ae_mapping, s3_client):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-activities-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activities',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    client = Client(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps(
        {
            "Sony Music - Century Media Records": "token72739d3",
            "Sony Music Argentina": "2f69661",
            "Sony Music Asia": "ad9bade",
            "Sony Music Australia/New Zealand": "afa0d1d",
        }
    )

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    test_file_content = '[{"123": 123},{"test":"test"}]'
    expected_file_content = '{"123": 123}\n{"test": "test"}\n'
    resp_mock = unittest.mock.MagicMock(spec=Response)
    requests_get.return_value.__enter__.return_value = resp_mock

    url = "https://sme-delphi.theappreciationengine.com/v1.1/activities"
    resp_mock.ok = True
    resp_mock.url = url
    resp_mock.content = bytes(test_file_content, encoding='utf8')
    resp_mock.headers = {}
    resp_mock.status_code = 200
    content_status_service = ContentStatusService(logger=logger, db_conn=db)
    client.download(job, chunk_size=1, content_status_service=content_status_service)

    # Appreciation Engine API called with correct request
    requests_get.assert_called_with(url, {'apiKey': 'token72739d3'})

    # read data from mocked S3 buckets
    path = 'appreciationengine/activities/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


def cvrt(data: Dict[str, Any]) -> bytes:
    return bytes(json.dumps(data), encoding='utf8')


def resp_mock():
    data = [
        {
            'totalSize': 1000,
            'items': [{
                "123": 123
            }, {
                "test": "test"
            }]
        },
        {
            'totalSize': 1000,
            'items': [{
                "test": "test"
            }]
        },
        {
            'totalSize': 1,
            'items': [{
                "test": "test"
            }]
        },
    ]
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = cvrt(dt)
        resp_mock.headers = {}
        resp_mock.status_code = 200
        yield resp_mock


@pytest.mark.integration
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
def test_downloading_paginate(
    requests_get, get_secret, patched_time_sleep, db, params, ae_mapping, s3_client
):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-brands-v1',
        'dsp': 'appreciationengine',
        'report_type': 'brands',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    client = PaginateClient(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps(
        {
            "Sony Music - Century Media Records": "token72739d3",
            "Sony Music Argentina": "2f69661",
            "Sony Music Asia": "ad9bade",
            "Sony Music Australia/New Zealand": "afa0d1d",
        }
    )

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    expected_file_content = '{"123": 123}\n{"test": "test"}\n{"test": "test"}\n{"test": "test"}\n'

    requests_get.return_value.__enter__.side_effect = resp_mock()

    url = "https://sme-delphi.theappreciationengine.com/v1.1/brands"

    content_status_service = ContentStatusService(logger=logger, db_conn=db)
    client.download(job, chunk_size=1, content_status_service=content_status_service)

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(url, {
                'apiKey': 'token72739d3',
                'limit': '0,1000',
            }),
            call(url, {
                'apiKey': 'token72739d3',
                'limit': '1000,1000',
            }),
            call(url, {
                'apiKey': 'token72739d3',
                'limit': '2000,1000',
            }),
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/brands/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


def resp_mock_membersextended():
    data = [
        {
            'totalSize':
                1000,
            'items':
                [
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 17:00:09'
                        }
                    },
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 19:00:08'
                        }
                    },
                ]
        },
        {
            'totalSize':
                1000,
            'items':
                [
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 17:00:09'
                        }
                    },
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 19:00:08'
                        }
                    },
                ]
        },
        {
            'totalSize':
                1000,
            'items':
                [
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 20:00:07'
                        }
                    },
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 21:51:17'
                        }
                    },
                ]
        },
        {
            'totalSize':
                1,
            'items':
                [
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 21:51:17'
                        }
                    },
                    {
                        'Extended': {
                            'lastUpdated': '2021-03-08 21:51:18'
                        }
                    },
                ]
        },
    ]
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = cvrt(dt)
        resp_mock.headers = {}
        resp_mock.status_code = 200
        yield resp_mock


@pytest.mark.integration
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(year=2021, month=6, day=5, hour=1, minute=1, second=1)
)
def test_downloading_paginate_dates_as_query_params(
    dt_mock, requests_get, get_secret, patched_time_sleep, db, params, ae_mapping, s3_client
):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-membersextended-v1',
        'dsp': 'appreciationengine',
        'report_type': 'membersextended',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music - Century Media Records": "token72739d3",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    expected_file_content = ''.join(
        [
            '{"Extended": {"lastUpdated": "2021-03-08 17:00:09"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 19:00:08"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 20:00:07"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 21:51:18"}}\n',
        ]
    )

    requests_get.return_value.__enter__.side_effect = resp_mock_membersextended()

    url = "https://sme-delphi.theappreciationengine.com/v1.1/members/extended"
    with unittest.mock.patch(
        'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.AEPaginateStringIOPartial.CHUNK_TIMEDELTA',
        timedelta(hours=24)
    ):
        content_status_service = ContentStatusService(logger=logger, db_conn=db)
        client.download(
            job,
            chunk_size=1,
            timer=timer,
            timeslot=None,
            content_status_service=content_status_service
        )

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '0,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '1000,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '2000,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/membersextended/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


@pytest.mark.integration
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(year=2021, month=6, day=5, hour=1, minute=1, second=1)
)
def test_downloading_paginate_size_of_chunk_too_big(
    dt_mock, requests_get, get_secret, patched_time_sleep, db, params, ae_mapping, s3_client
):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-membersextended-v1',
        'dsp': 'appreciationengine',
        'report_type': 'membersextended',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music - Century Media Records": "token72739d3",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    expected_file_content = ''.join(
        [
            '{"Extended": {"lastUpdated": "2021-03-08 17:00:09"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 19:00:08"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 20:00:07"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
            '{"Extended": {"lastUpdated": "2021-03-08 21:51:18"}}\n',
        ]
    )

    requests_get.return_value.__enter__.side_effect = \
        [raise_exception, *[mk for mk in resp_mock_membersextended()]]

    url = "https://sme-delphi.theappreciationengine.com/v1.1/members/extended"
    with unittest.mock.patch(
        'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.AEPaginateStringIOPartial.CHUNK_TIMEDELTA',
        timedelta(hours=24)
    ):
        content_status_service = ContentStatusService(logger=logger, db_conn=db)
        client.download(
            job,
            chunk_size=1,
            timer=timer,
            timeslot=None,
            content_status_service=content_status_service
        )

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '0,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '0,500',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            )
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/membersextended/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


def resp_mock_activityfeed_us_columbia():
    data = [
        cvrt({
            'totalSize': 1000,
            'items': [{
                'RecordDate': '2020-11-04 23:17:16'
            }]
        }),
        cvrt({
            'totalSize': 1000,
            'items': [{
                'RecordDate': '2020-11-04 23:17:16'
            }]
        }),
        cvrt(
            {
                'totalSize':
                    1000,
                'items':
                    [
                        {
                            'RecordDate': '2020-11-04 23:17:17'
                        },
                        {
                            'RecordDate': '2020-11-04 23:17:18'
                        },
                        {
                            'RecordDate': '2020-11-04 23:17:22'
                        },
                    ]
            }
        ),
        cvrt(
            {
                'totalSize':
                    1000,
                'items':
                    [
                        {
                            'RecordDate': '2020-11-04 23:17:22'
                        },
                        {
                            'RecordDate': '2020-11-04 23:17:25'
                        },
                    ]
            }
        ),
        cvrt({
            'totalSize': 1,
            'items': [{
                'RecordDate': '2020-11-04 23:17:26'
            }]
        })
    ]
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = dt
        resp_mock.headers = {}
        resp_mock.status_code = 200
        yield resp_mock


@pytest.mark.integration
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.time.sleep', return_value=None
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(year=2021, month=6, day=5, hour=1, minute=1, second=1)
)
def test_downloading_paginate_exceed_paginate_limit(
    dt_mock, requests_get, get_secret, patched_time_sleep, db, params, ae_mapping, s3_client
):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-activityfeed-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activityfeed',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'US_Columbia',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    expected_file_content = ''.join(
        [
            '{"RecordDate": "2020-11-04 23:17:16"}\n',
            '{"RecordDate": "2020-11-04 23:17:17"}\n',
            '{"RecordDate": "2020-11-04 23:17:18"}\n',
            '{"RecordDate": "2020-11-04 23:17:22"}\n',
            '{"RecordDate": "2020-11-04 23:17:25"}\n',
            '{"RecordDate": "2020-11-04 23:17:26"}\n',
        ]
    )
    # Mock response from source API

    requests_get.return_value.__enter__.side_effect = resp_mock_activityfeed_us_columbia()

    url = "https://sme-delphi.theappreciationengine.com/v1.1/activity/feed"
    with unittest.mock.patch(
        'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.AEPaginateBytesIO.MAX_PAGINATION',
        2
    ):
        with unittest.mock.patch(
            'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.AEPaginateStringIOPartial.CHUNK_TIMEDELTA',
            timedelta(hours=24)
        ):
            content_status_service = ContentStatusService(logger=logger, db_conn=db)
            client.download(
                job,
                chunk_size=1,
                timer=timer,
                timeslot=None,
                content_status_service=content_status_service
            )

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-03 23:59:59',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "0,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-03 23:59:59',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "1000,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-04 23:17:19',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "0,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-04 23:17:19',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "1000,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/activityfeed/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/US_Columbia_20201104_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/US_Columbia_20201104_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


@pytest.mark.parametrize(
    'report_type, records, include_rec_in_resp, start_date, expected_records', [
        (
            'activityfeed',
            [
                {
                    'RecordDate': '2020-11-04 23:17:17'
                },
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
                {
                    'RecordDate': '2020-11-04 23:17:19'
                },
            ],
            True,
            '2020-11-04 23:17:19',
            [
                {
                    'RecordDate': '2020-11-04 23:17:17'
                },
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
            ],
        ),
        (
            'activityfeed',
            [
                {
                    'RecordDate': '2020-11-04 23:17:17'
                },
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
                {
                    'RecordDate': '2020-11-04 23:17:19'
                },
            ],
            False,
            '2020-11-04 23:17:18',
            [
                {
                    'RecordDate': '2020-11-04 23:17:17'
                },
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
            ],
        ),
        (
            'activityfeed',
            [
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
                {
                    'RecordDate': '2020-11-04 23:17:19'
                },
            ],
            False,
            '2020-11-04 23:17:18',
            [
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
                {
                    'RecordDate': '2020-11-04 23:17:18'
                },
            ],
        ),
        (
            'memberslogin',
            [
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:19'
                },
                {
                    'LastVisited': '2020-11-04 23:17:19'
                },
            ],
            True,
            '2020-11-04 23:17:19',
            [
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
            ],
        ),
        (
            'memberslogin',
            [
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:20'
                },
                {
                    'LastVisited': '2020-11-04 23:17:20'
                },
            ],
            True,
            '2020-11-04 23:17:19',
            [
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
            ],
        ),
        (
            'memberslogin',
            [
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
            ],
            True,
            None,
            [
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
                {
                    'LastVisited': '2020-11-04 23:17:18'
                },
            ],
        ),
        (
            'membersextended',
            [
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 19:00:08'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 20:00:07'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 21:51:17'
                    }
                },
            ],
            True,
            '2021-03-08 20:00:08',
            [
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 19:00:08'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 20:00:07'
                    }
                },
            ],
        ),
        (
            'membersextended',
            [
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
            ],
            True,
            None,
            [
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
            ],
        ),
        (
            'membersextended',
            [
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:11'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:11'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:11'
                    }
                },
            ],
            True,
            '2021-03-08 17:00:10',
            [
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
                {
                    'Extended': {
                        'lastUpdated': '2021-03-08 17:00:09'
                    }
                },
            ],
        ),
    ]
)
def test__get_new_start_date_ok(
    report_type, records, include_rec_in_resp, start_date, expected_records, ae_mapping
):
    logger = unittest.mock.Mock()
    job = Job.empty()
    job.report_type = report_type
    job.licensor = 'sme'
    client = AEPaginateStringIOPartial(
        logger=logger,
        job=job,
        secret=dict(),
        scrapper_config=Scrapper(host='', ),
        config_bucket='test-config-bucket',
        troubleshooting_logs_bucket='',
    )
    client.start_date_included_in_response = include_rec_in_resp
    actual_start_date, actual_records = client._get_new_start_date(job, records)
    assert actual_start_date == start_date
    assert actual_records == expected_records


def test__get_new_start_date_raise_exc(ae_mapping):
    logger = unittest.mock.Mock()
    job = Job.empty()
    job.report_type = 'unknown'
    job.licensor = 'sme'
    client = AEPaginateStringIOPartial(
        logger=logger,
        job=job,
        secret=dict(),
        scrapper_config=Scrapper(host='', ),
        config_bucket='test-config-bucket',
        troubleshooting_logs_bucket='',
    )
    records = [
        {
            'somekey': 'someval1'
        },
        {
            'somekey': 'someval2'
        },
    ]
    with pytest.raises(StreamError):
        client._get_new_start_date(job, records)


@pytest.mark.parametrize(
    'ok, status_code, content, expected', [
        (True, 200, '[{"123": 123},{"test":"test"}]', ('CenturyMediaRecords', 0)),
        (True, 200, '[]', None),
        (True, 200, '', None),
        (False, 400, '', None),
        (False, 404, '', None),
        (False, 500, '', None),
    ]
)
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.AEBytesIOBase._query_source'
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
def test_ae_availability_check_without_pagination(
    get_secret, response_mock, ok, status_code, content, expected, params, ae_mapping
):
    uow = {
        'uow_id': 'appreciationengine-20201104-sme-activities-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activities',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })
    job = Job.from_dict(uow)
    logger = unittest.mock.Mock()
    client = Client(logger)
    client.configure(params)
    resp_mock_base = unittest.mock.MagicMock(spec=Response)
    resp_mock_base.ok = ok
    resp_mock_base.content = bytes(content, encoding='utf8')
    resp_mock_base.headers = {}
    resp_mock_base.url = ''
    resp_mock_base.status_code = status_code
    resp_mock_base.reason = 'Some text'

    response_mock.return_value.__enter__.return_value = resp_mock_base
    result = client._check_source_is_available(job)
    assert result == expected


@pytest.mark.parametrize(
    'ok, status_code, content, expected', [
        (True, 200, cvrt({
            'totalSize': 1,
            'items': [{
                "123": 123
            }]
        }), ('US_Columbia', 0)),
        (True, 200, cvrt({
            'totalSize': 0,
            'items': []
        }), None),
        (True, 200, '', None),
        (False, 400, '', None),
        (False, 404, '', None),
        (False, 500, '', None),
    ]
)
@unittest.mock.patch(
    'slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io.AEBytesIOBase._query_source'
)
@unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret')
def test_ae_availability_check_with_pagination(
    get_secret, response_mock, ok, status_code, content, expected, params, ae_mapping
):
    uow = {
        'uow_id': 'appreciationengine-20201104-sme-activityfeed-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activityfeed',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'US_Columbia',
    }
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })
    job = Job.from_dict(uow)
    logger = unittest.mock.Mock()
    client = PaginateClient(logger)
    client.configure(params)
    resp_mock_base = unittest.mock.MagicMock(spec=Response)
    resp_mock_base.ok = ok
    resp_mock_base.content = content
    resp_mock_base.headers = {}
    resp_mock_base.url = ''
    resp_mock_base.status_code = status_code
    resp_mock_base.reason = 'Some text'

    response_mock.return_value.__enter__.return_value = resp_mock_base
    result = client._check_source_is_available(job)
    assert result == expected


@pytest.mark.parametrize(
    'url,expected', [
        (
            'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac7149cf27a879c49aaf3f592fa69ac',
            'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac71*****a69ac',
        ),
        (
            'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac7149cf27a879c49aaf3f592fa69ac&overall=True',
            'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac71*****a69ac&overall=True',
        ),
        (
            'http://blah.com?apiKey=bac7149cf27a879c49aaf3f592fa69ac&incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
            'http://blah.com?apiKey=bac71*****a69ac&incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
        ),
        (
            'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
            'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
        ),
        (
            'http://blah.com?apiKey=bac7149cf27a879c49aaf3f592fa69ac',
            'http://blah.com?apiKey=bac71*****a69ac',
        ), (
            'http://blah.com',
            'http://blah.com',
        )
    ]
)
def test_replace(url, expected):
    got = hide_secret(url)
    assert got == expected