import gzip
import json
import unittest.mock
from datetime import datetime, timedelta
from typing import Any, Dict
from unittest.mock import call

import pytest
from requests.models import Response

from slz_downloader.dsp.appreciationengine.ae_io import AEPaginateBytesIO, AEPaginateStringIOPartial
from slz_downloader.dsp.appreciationengine.clients import (
    Client,
    PaginateClient,
    PaginateClientPartial,
)
from slz_downloader.dsp.appreciationengine.helpers import hide_secret
from slz_downloader.dsp.exceptions import StreamError
from slz_downloader.entities import Job


def raise_exception():
    raise StreamError


@unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
def test_downloading(requests_get, get_secret, params, ae_mapping, s3_client):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-activities-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activities',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    client = Client(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps(
        {
            "Sony Music - Century Media Records": "token72739d3",
            "Sony Music Argentina": "2f69661",
            "Sony Music Asia": "ad9bade",
            "Sony Music Australia/New Zealand": "afa0d1d",
        }
    )

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    test_file_content = '[{"123": 123},{"test":"test"}]'
    expected_file_content = '{"123": 123}\n{"test": "test"}\n'
    resp_mock = unittest.mock.MagicMock(spec=Response)
    requests_get.return_value.__enter__.return_value = resp_mock

    url = "https://sme.theappreciationengine.com/v1.1/activities"
    resp_mock.ok = True
    resp_mock.url = url
    resp_mock.content = bytes(test_file_content, encoding='utf8')
    resp_mock.headers = {}
    client.download(job, chunk_size=1)

    # Appreciation Engine API called with correct request
    requests_get.assert_called_with(url, {'apiKey': 'token72739d3'})

    # read data from mocked S3 buckets
    path = 'appreciationengine/activities/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


def cvrt(data: Dict[str, Any]) -> bytes:
    return bytes(json.dumps(data), encoding='utf8')


def resp_mock():
    data = [
        {
            'totalSize': 1000,
            'items': [
                {"123": 123},
                {"test": "test"}
            ]
        },
        {
            'totalSize': 1000,
            'items': [{"test": "test"}]
        },
        {
            'totalSize': 1,
            'items': [{"test": "test"}]
        },
    ]
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = cvrt(dt)
        resp_mock.headers = {}
        yield resp_mock


@unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
def test_downloading_paginate(requests_get, get_secret, params, ae_mapping, s3_client):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-brands-v1',
        'dsp': 'appreciationengine',
        'report_type': 'brands',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    client = PaginateClient(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps(
        {
            "Sony Music - Century Media Records": "token72739d3",
            "Sony Music Argentina": "2f69661",
            "Sony Music Asia": "ad9bade",
            "Sony Music Australia/New Zealand": "afa0d1d",
        }
    )

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    expected_file_content = '{"123": 123}\n{"test": "test"}\n{"test": "test"}\n{"test": "test"}\n'

    requests_get.return_value.__enter__.side_effect = resp_mock()

    url = "https://sme.theappreciationengine.com/v1.1/brands"

    client.download(job, chunk_size=1)

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'limit': '0,1000',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'limit': '1000,1000',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'limit': '2000,1000',
                }
            ),
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/brands/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


def resp_mock_membersextended():
    data = [
        {
            'totalSize': 1000,
            'items': [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 19:00:08'}},
            ]
        },
        {
            'totalSize': 1000,
            'items': [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 19:00:08'}},
            ]
        },{
            'totalSize': 1000,
            'items': [
                {'Extended': {'lastUpdated': '2021-03-08 20:00:07'}},
                {'Extended': {'lastUpdated': '2021-03-08 21:51:17'}},
            ]
        },{
            'totalSize': 1,
            'items': [
                {'Extended': {'lastUpdated': '2021-03-08 21:51:17'}},
                {'Extended': {'lastUpdated': '2021-03-08 21:51:18'}},
            ]
        },
    ]
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = cvrt(dt)
        resp_mock.headers = {}
        yield resp_mock


@unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_downloader.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(
        year=2021, month=6, day=5, hour=1, minute=1, second=1
    )
)
def test_downloading_paginate_dates_as_query_params(dt_mock, requests_get, get_secret, params, ae_mapping, s3_client):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-membersextended-v1',
        'dsp': 'appreciationengine',
        'report_type': 'membersextended',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music - Century Media Records": "token72739d3",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    expected_file_content = ''.join([
        '{"Extended": {"lastUpdated": "2021-03-08 17:00:09"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 19:00:08"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 20:00:07"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 21:51:18"}}\n',
    ])

    requests_get.return_value.__enter__.side_effect = resp_mock_membersextended()

    url = "https://sme.theappreciationengine.com/v1.1/members/extended"
    with unittest.mock.patch(
            'slz_downloader.dsp.appreciationengine.ae_io.AEPaginateStringIOPartial.CHUNK_TIMEDELTA',
            timedelta(hours=24)
    ):
        client.download(job, chunk_size=1, timer=timer)

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '0,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '1000,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '2000,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/membersextended/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


@unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_downloader.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(
        year=2021, month=6, day=5, hour=1, minute=1, second=1
    )
)
def test_downloading_paginate_size_of_chunk_too_big(dt_mock, requests_get, get_secret, params, ae_mapping, s3_client):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-membersextended-v1',
        'dsp': 'appreciationengine',
        'report_type': 'membersextended',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music - Century Media Records": "token72739d3",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    # Mock response from source API
    expected_file_content = ''.join([
        '{"Extended": {"lastUpdated": "2021-03-08 17:00:09"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 19:00:08"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 20:00:07"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 21:51:17"}}\n',
        '{"Extended": {"lastUpdated": "2021-03-08 21:51:18"}}\n',
    ])

    requests_get.return_value.__enter__.side_effect = \
        [raise_exception, *[mk for mk in resp_mock_membersextended()]]

    url = "https://sme.theappreciationengine.com/v1.1/members/extended"
    with unittest.mock.patch(
            'slz_downloader.dsp.appreciationengine.ae_io.AEPaginateStringIOPartial.CHUNK_TIMEDELTA',
            timedelta(hours=24)
    ):
        client.download(job, chunk_size=1, timer=timer)

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '0,1000',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            ),
            call(
                url, {
                    'apiKey': 'token72739d3',
                    'query_extended':
                        'extended.lastUpdated>=2020-11-03 23:59:59'
                        ' and extended.lastUpdated<=2020-11-05 00:00:00',
                    'limit': '0,500',
                    'sort': 'ASC',
                    'order_by': 'extended.lastUpdated',
                }
            )
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/membersextended/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/CenturyMediaRecords_20201104_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


def resp_mock_activityfeed_us_columbia():
    data = [
        cvrt({
            'totalSize': 1000,
            'items': [{'RecordDate': '2020-11-04 23:17:16'}]
        }),
        cvrt({
            'totalSize': 1000,
            'items': [{'RecordDate': '2020-11-04 23:17:16'}]
        }),
        cvrt({
            'totalSize': 1000,
            'items': [
                {'RecordDate': '2020-11-04 23:17:17'},
                {'RecordDate': '2020-11-04 23:17:18'},
                {'RecordDate': '2020-11-04 23:17:22'},
            ]
        }),
        cvrt({
            'totalSize': 1000,
            'items': [
                {'RecordDate': '2020-11-04 23:17:22'},
                {'RecordDate': '2020-11-04 23:17:25'},
            ]
        }),
        cvrt({
            'totalSize': 1,
            'items': [{'RecordDate': '2020-11-04 23:17:26'}]
        })

    ]
    for dt in data:
        resp_mock = unittest.mock.MagicMock(spec=Response)
        resp_mock.ok = True
        resp_mock.url = ''
        resp_mock.content = dt
        resp_mock.headers = {}
        yield resp_mock


@unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret')
@unittest.mock.patch('requests.get')
@unittest.mock.patch(
    'slz_downloader.dsp.appreciationengine.clients.datetime',
    wraps=datetime,
    now=lambda *args, **kwargs: datetime(
        year=2021, month=6, day=5, hour=1, minute=1, second=1
    )
)
def test_downloading_paginate_exceed_paginate_limit(dt_mock, requests_get, get_secret, params, ae_mapping, s3_client):
    for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']:
        s3_client.create_bucket(Bucket=bucket)

    uow_dict = {
        'uow_id': 'appreciationengine-20201104-sme-activityfeed-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activityfeed',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'US_Columbia',
    }

    # Client initialization
    logger = unittest.mock.Mock()
    timer = unittest.mock.Mock()
    client = PaginateClientPartial(logger=logger)

    # Mock secrets
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })

    client.configure(params)
    job = Job.from_dict(uow_dict)

    expected_file_content = ''.join([
        '{"RecordDate": "2020-11-04 23:17:16"}\n',
        '{"RecordDate": "2020-11-04 23:17:17"}\n',
        '{"RecordDate": "2020-11-04 23:17:18"}\n',
        '{"RecordDate": "2020-11-04 23:17:22"}\n',
        '{"RecordDate": "2020-11-04 23:17:25"}\n',
        '{"RecordDate": "2020-11-04 23:17:26"}\n',
    ])
    # Mock response from source API

    requests_get.return_value.__enter__.side_effect = resp_mock_activityfeed_us_columbia()

    url = "https://sme.theappreciationengine.com/v1.1/activity/feed"
    with unittest.mock.patch(
        'slz_downloader.dsp.appreciationengine.ae_io.AEPaginateBytesIO.MAX_PAGINATION',
        2
    ):
        with unittest.mock.patch(
            'slz_downloader.dsp.appreciationengine.ae_io.AEPaginateStringIOPartial.CHUNK_TIMEDELTA',
            timedelta(hours=24)
    ):
            client.download(job, chunk_size=1, timer=timer)

    # Appreciation Engine API called with correct request
    requests_get.assert_has_calls(
        [
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-03 23:59:59',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "0,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-03 23:59:59',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "1000,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-04 23:17:19',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "0,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
            call(
                url, {
                    'apiKey': 'afa0d1d',
                    'start_date': '2020-11-04 23:17:19',
                    'end_date': '2020-11-05 00:00:00',
                    'limit': "1000,1000",
                    'includeOneTime': 1,
                    'order_by_recorded': 1,
                    'all_member_actions': 1,
                    'sort': 'asc',
                }
            ),
        ],
        any_order=True
    )

    # read data from mocked S3 buckets
    path = 'appreciationengine/activityfeed/v1/report_date=2020-11-04/report_licensor=sme'
    actual_decompressed = s3_client.get_object(
        Bucket='bucket-decompressed-quarantine',
        Key=f'{path}/US_Columbia_20201104_20210605010101.json',
    )['Body'].read()
    actual_compressed = s3_client.get_object(
        Bucket='bucket-archive-quarantine',
        Key=f'{path}/US_Columbia_20201104_20210605010101.json.gz',
    )['Body'].read()

    assert expected_file_content == actual_decompressed.decode('utf8')
    assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8')


@pytest.mark.parametrize('report_type, records, include_rec_in_resp, start_date, expected_records', [
    (
            'activityfeed',
            [
                {'RecordDate': '2020-11-04 23:17:17'},
                {'RecordDate': '2020-11-04 23:17:18'},
                {'RecordDate': '2020-11-04 23:17:19'},
            ],
            True,
            '2020-11-04 23:17:19',
            [
                {'RecordDate': '2020-11-04 23:17:17'},
                {'RecordDate': '2020-11-04 23:17:18'},
            ],
    ),
    (
            'activityfeed',
            [
                {'RecordDate': '2020-11-04 23:17:17'},
                {'RecordDate': '2020-11-04 23:17:18'},
                {'RecordDate': '2020-11-04 23:17:19'},
            ],
            False,
            '2020-11-04 23:17:18',
            [
                {'RecordDate': '2020-11-04 23:17:17'},
                {'RecordDate': '2020-11-04 23:17:18'},
            ],
    ),
    (
            'activityfeed',
            [
                {'RecordDate': '2020-11-04 23:17:18'},
                {'RecordDate': '2020-11-04 23:17:18'},
                {'RecordDate': '2020-11-04 23:17:19'},
            ],
            False,
            '2020-11-04 23:17:18',
            [
                {'RecordDate': '2020-11-04 23:17:18'},
                {'RecordDate': '2020-11-04 23:17:18'},
            ],
    ),
    (
            'memberslogin',
            [
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:19'},
                {'LastVisited': '2020-11-04 23:17:19'},
            ],
            True,
            '2020-11-04 23:17:19',
            [
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
            ],
    ),
    (
            'memberslogin',
            [
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:20'},
                {'LastVisited': '2020-11-04 23:17:20'},
            ],
            True,
            '2020-11-04 23:17:19',
            [
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
            ],
    ),
    (
            'memberslogin',
            [
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
            ],
            True,
            None,
            [
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
                {'LastVisited': '2020-11-04 23:17:18'},
            ],
    ),
    (
            'membersextended',
            [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 19:00:08'}},
                {'Extended': {'lastUpdated': '2021-03-08 20:00:07'}},
                {'Extended': {'lastUpdated': '2021-03-08 21:51:17'}},
            ],
            True,
            '2021-03-08 20:00:08',
            [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 19:00:08'}},
                {'Extended': {'lastUpdated': '2021-03-08 20:00:07'}},
            ],
    ),
    (
            'membersextended',
            [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
            ],
            True,
            None,
            [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
            ],
    ),
    (
            'membersextended',
            [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:11'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:11'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:11'}},
            ],
            True,
            '2021-03-08 17:00:10',
            [
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
                {'Extended': {'lastUpdated': '2021-03-08 17:00:09'}},
            ],
    ),
])
def test__get_new_start_date_ok(report_type, records, include_rec_in_resp, start_date, expected_records, ae_mapping):
    logger = unittest.mock.Mock()
    job = Job.empty()
    job.report_type = report_type
    job.licensor = 'sme'
    client = AEPaginateStringIOPartial(
        logger=logger,
        job=job,
        secret=dict(),
        host='',
        config_bucket='test-config-bucket',
        troubleshooting_logs_bucket='',
    )
    client.start_date_included_in_response = include_rec_in_resp
    actual_start_date, actual_records = client._get_new_start_date(job, records)
    assert actual_start_date == start_date
    assert actual_records == expected_records


def test__get_new_start_date_raise_exc(ae_mapping):
    logger = unittest.mock.Mock()
    job = Job.empty()
    job.report_type = 'unknown'
    job.licensor = 'sme'
    client = AEPaginateStringIOPartial(
        logger=logger,
        job=job,
        secret=dict(),
        host='',
        config_bucket='test-config-bucket',
        troubleshooting_logs_bucket='',
    )
    records = [
        {'somekey': 'someval1'},
        {'somekey': 'someval2'},
    ]
    with pytest.raises(StreamError):
        client._get_new_start_date(job, records)


@pytest.mark.parametrize(
    'ok, status_code, content, expected', [
        (True, 200, '[{"123": 123},{"test":"test"}]', ('CenturyMediaRecords', 0)),
        (True, 200, '[]', None),
        (True, 200, '', None),
        (False, 400, '', None),
        (False, 404, '', None),
        (False, 500, '', None),
    ])
@unittest.mock.patch('slz_downloader.dsp.appreciationengine.ae_io.AEBytesIOBase._query_source')
@unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret')
def test_ae_availability_check_without_pagination(
        get_secret, response_mock, ok, status_code, content, expected, params, ae_mapping
):
    uow = {
        'uow_id': 'appreciationengine-20201104-sme-activities-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activities',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'CenturyMediaRecords',
    }
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })
    job = Job.from_dict(uow)
    logger = unittest.mock.Mock()
    client = Client(logger)
    client.configure(params)
    resp_mock_base = unittest.mock.MagicMock(spec=Response)
    resp_mock_base.ok = ok
    resp_mock_base.content = bytes(content, encoding='utf8')
    resp_mock_base.headers = {}
    resp_mock_base.url = ''
    resp_mock_base.status_code = status_code
    resp_mock_base.reason = 'Some text'

    response_mock.return_value.__enter__.return_value = resp_mock_base
    result = client._check_source_is_available(job)
    assert result == expected


@pytest.mark.parametrize(
    'ok, status_code, content, expected', [
        (True, 200, cvrt({'totalSize': 1, 'items': [{"123": 123}]}), ('US_Columbia', 0)),
        (True, 200, cvrt({'totalSize': 0, 'items': []}), None),
        (True, 200, '', None),
        (False, 400, '', None),
        (False, 404, '', None),
        (False, 500, '', None),
    ])
@unittest.mock.patch('slz_downloader.dsp.appreciationengine.ae_io.AEBytesIOBase._query_source')
@unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret')
def test_ae_availability_check_with_pagination(
        get_secret, response_mock, ok, status_code, content, expected, params, ae_mapping):
    uow = {
        'uow_id': 'appreciationengine-20201104-sme-activityfeed-v1',
        'dsp': 'appreciationengine',
        'report_type': 'activityfeed',
        'version': 'v1',
        'report_date': '2020-11-04',
        'licensor': 'sme',
        'extension': 'json',
        'context': 'US_Columbia',
    }
    get_secret.return_value = json.dumps({
        "Sony Music US - Columbia": "afa0d1d",
    })
    job = Job.from_dict(uow)
    logger = unittest.mock.Mock()
    client = PaginateClient(logger)
    client.configure(params)
    resp_mock_base = unittest.mock.MagicMock(spec=Response)
    resp_mock_base.ok = ok
    resp_mock_base.content = content
    resp_mock_base.headers = {}
    resp_mock_base.url = ''
    resp_mock_base.status_code = status_code
    resp_mock_base.reason = 'Some text'

    response_mock.return_value.__enter__.return_value = resp_mock_base
    result = client._check_source_is_available(job)
    assert result == expected


@pytest.mark.parametrize('url,expected', [
    (
        'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac7149cf27a879c49aaf3f592fa69ac',
        'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac71*****a69ac',
    ),
    (
        'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac7149cf27a879c49aaf3f592fa69ac&overall=True',
        'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&apiKey=bac71*****a69ac&overall=True',
    ),
    (
        'http://blah.com?apiKey=bac7149cf27a879c49aaf3f592fa69ac&incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
        'http://blah.com?apiKey=bac71*****a69ac&incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
    ),
    (
        'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
        'http://blah.com?incliudeOneTime=1&sort=ASC&withActivities=1&type=1&overall=True',
    ),
    (
        'http://blah.com?apiKey=bac7149cf27a879c49aaf3f592fa69ac',
        'http://blah.com?apiKey=bac71*****a69ac',
    ),
    (
        'http://blah.com',
        'http://blah.com',
    )
])
def test_replace(url, expected):
    got = hide_secret(url)
    assert got == expected