"""Unit tests for the Cable Ingestion util module."""

from collections import OrderedDict
import io
import tempfile
from unittest.mock import Mock
from unittest.mock import patch

from pytest import raises

from flows.cable_ingestion import util
from flows.exceptions import EmptyGeneratorError
from tests.conftest import create_test_tarball


def test_verify_rentrak_data_file_valid():
    """Test verify_rentrak_data_file function."""
    cases = [
        (
            '/FOO/OUT/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            '/FOO/OUT/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.tar.gz'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.tar.gz'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.tar.gz')]

    for case in cases:
        assert util.verify_rentrak_data_file(case)


def test_verify_rentrak_data_file_invalid():
    """Test verify_rentrak_data_file function."""
    cases = [
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.atl'),
        (
            '_rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            '_rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl.'),
        (
            'tentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.tar'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.targgz'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.gz'),
        (
            'rentrak_theorchard_subscriptions_'
            '0160620_20160828_20160830082404.ctl')]

    for case in cases:
        assert not util.verify_rentrak_data_file(case)


def test_verify_rentrak_data_file_date_range_valid():
    """Test verify_rentrak_data_file function."""
    # (filename, start, end)
    cases = [
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl',
            '20160601', '20160701'),

        # test partial overlap ranges
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160621_20160830082404.tar.gz',
            '20160601', '20160621'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl',
            '20160828', '20170101')]

    for filename, start, end in cases:
        assert util.verify_rentrak_data_file_date_range(end, start, filename)


def test_verify_rentrak_data_file_date_range_invalid():
    """Test verify_rentrak_data_file function."""
    # (filename, start, end)
    cases = [
        # end dates are exclusive in the range
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl',
            '20160601', '20160620'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160621_20160830082404.tar.gz',
            '20160601', '20160602'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl',
            '20160829', '20170101')]

    for filename, start, end in cases:
        assert not util.verify_rentrak_data_file_date_range(
            end, start, filename)


def test_filter_files_by_date_range():
    """Test filter_files_by_date_range function."""
    files = [
        (
            '/FOO/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            '/BAR/rentrak_theorchard_subscriptions_'
            '20160620_20160621_20160830082404.tar.gz'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160720_20160728_20160830082404.ctl'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160820_20160828_20160830082404.ctl'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160120_20160128_20160830082404.ctl')
    ]
    date_start = '20160601'
    date_end = '20160731'

    expected = (
        (
            '/FOO/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            '/BAR/rentrak_theorchard_subscriptions_'
            '20160620_20160621_20160830082404.tar.gz'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160720_20160728_20160830082404.ctl'))

    results = util.filter_files_by_date_range(date_end, date_start, files)
    assert tuple(results) == expected


def test_filter_file_pairs():
    """Test filter_file_pairs function."""
    files = [
        (
            '/FOO/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.ctl'),
        (
            '/FOO/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.tar.gz'),
        (
            '/FOO/rentrak_theorchard_subscriptions_'
            '20160620_20160827_20160830082404.tar'),
        (
            '/BAR/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.tar.gz'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160720_20160728_20160830082404.ctl'),

        # range matches with differing timestamps
        (
            'rentrak_theorchard_subscriptions_'
            '20160820_20160828_20160830082404.ctl'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160820_20160828_20160830082411.tar.gz'),

        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161108050430.ctl'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161108050430.tar.gz'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161113065652.ctl'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161113065652.tar.gz'),
    ]

    expected = (
        {
            'ctl': (
                '/FOO/rentrak_theorchard_subscriptions_'
                '20160620_20160828_20160830082404.ctl'),
            'tar.gz': (
                '/FOO/rentrak_theorchard_subscriptions_'
                '20160620_20160828_20160830082404.tar.gz')},
        {
            'ctl': (
                'rentrak_theorchard_subscriptions_'
                '20160820_20160828_20160830082404.ctl'),
            'tar.gz': (
                'rentrak_theorchard_subscriptions_'
                '20160820_20160828_20160830082411.tar.gz')},
        {
            'ctl': (
                '/OUT/rentrak_theorchard_subscriptions_'
                '20160829_20161106_20161108050430.ctl'),
            'tar.gz': (
                '/OUT/rentrak_theorchard_subscriptions_'
                '20160829_20161106_20161108050430.tar.gz')},
        {
            'ctl': (
                '/OUT/rentrak_theorchard_subscriptions_'
                '20160829_20161106_20161113065652.ctl'),
            'tar.gz': (
                '/OUT/rentrak_theorchard_subscriptions_'
                '20160829_20161106_20161113065652.tar.gz')})

    results = util.filter_file_pairs(files)
    assert tuple(results) == expected


def test_filter_file_pairs_tolerance():
    """Test filter_file_pairs function."""
    files = [
        (
            '/FOO/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082403.ctl'),
        (
            '/FOO/rentrak_theorchard_subscriptions_'
            '20160620_20160828_20160830082404.tar.gz'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160820_20160828_20160830082404.ctl'),
        (
            'rentrak_theorchard_subscriptions_'
            '20160820_20160828_20160830082411.tar.gz'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161108050430.ctl'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161108050430.tar.gz'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161113065000.ctl'),
        (
            '/OUT/rentrak_theorchard_subscriptions_'
            '20160829_20161106_20161113065111.tar.gz'),
    ]

    expected = (
        {
            'ctl': (
                '/FOO/rentrak_theorchard_subscriptions_'
                '20160620_20160828_20160830082403.ctl'),
            'tar.gz': (
                '/FOO/rentrak_theorchard_subscriptions_'
                '20160620_20160828_20160830082404.tar.gz')},
        {
            'ctl': (
                '/OUT/rentrak_theorchard_subscriptions_'
                '20160829_20161106_20161108050430.ctl'),
            'tar.gz': (
                '/OUT/rentrak_theorchard_subscriptions_'
                '20160829_20161106_20161108050430.tar.gz')})

    results = util.filter_file_pairs(files, tolerance=1)
    assert tuple(results) == expected


def test_convert_ctl_data():
    """Test convert_ctl_data function."""
    cases = [
        (
            ['foobar.file', 'somehash', '1234'],
            {'name': 'foobar.file', 'md5': 'somehash', 'size': 1234}),
        (
            ['baz.thing', 'thishash', '1111'],
            {'name': 'baz.thing', 'md5': 'thishash', 'size': 1111}),
        (
            ['nine.nine.9', 'nnn', '999'],
            {'name': 'nine.nine.9', 'md5': 'nnn', 'size': 999})]
    for param, expected in cases:
        assert util.convert_ctl_data(param) == expected


def test_convert_rentrak_metadata():
    """Test convert_rentrak_metadata function."""
    tarinfo = Mock()
    tarinfo.name = 'somefile'
    tarinfo.size = 1234567890
    file_handler = io.BytesIO('This is a test'.encode('utf8'))
    file_handler.seek(0)
    tarfile = Mock()
    tarfile.extractfile.return_value = file_handler

    expected = {
        'size': tarinfo.size,
        'name': tarinfo.name,
        'md5': 'ce114e4501d2f4e2dcea3e17b546f339'}

    assert util.convert_rentrak_metadata(tarinfo, tarfile) == expected


def test_extract_ctl_data():
    """Test extract_ctl_data function."""
    data = (
        'file1.csv|69608232a6007b10659d98478338feb0|1930569\n'
        'file2.csv|efea56cae07c4eafd83a8ab45d0d9ee5|1507794\n'
        'file3.csv|46a0c97369aa32b65af965a399450d59|1341353\n'
        'file4.csv|63bf86b659844034712dee64dc0ddce6|1280176\n'
        'file5.csv|8c1215c0656347c854c41d2feb940b44|1092307\n')
    expected = (
        {
            'name': 'file1.csv',
            'md5': '69608232a6007b10659d98478338feb0',
            'size': 1930569},
        {
            'name': 'file2.csv',
            'md5': 'efea56cae07c4eafd83a8ab45d0d9ee5',
            'size': 1507794},
        {
            'name': 'file3.csv',
            'md5': '46a0c97369aa32b65af965a399450d59',
            'size': 1341353},
        {
            'name': 'file4.csv',
            'md5': '63bf86b659844034712dee64dc0ddce6',
            'size': 1280176},
        {
            'name': 'file5.csv',
            'md5': '8c1215c0656347c854c41d2feb940b44',
            'size': 1092307})
    with tempfile.TemporaryFile(mode='w+t') as fh:
        fh.write(data)
        fh.seek(0)
        results = util.extract_ctl_data(fh)
        assert tuple(results) == expected


def test_extract_rentrak_metadata():
    """Test extract_rentrak_metadata function.

    This test creates a temporary tarfile in memory.
    """
    test_files = OrderedDict((
        ('foo', 'This is the file named "foo".'.encode('utf-8')),
        ('bar', 'And this is the second file, names "bar".'.encode('utf-8')),
        ('baz', 'And lastly, the third of the bunch, "baz".'.encode('utf-8'))))

    expected = {
        'foo': {
            'md5': '6bdf333712bda9c8a887692f5185c31c',
            'name': 'foo',
            'size': 29},
        'bar': {
            'md5': '26d11420cfeeafe15fe9a0913c06be97',
            'name': 'bar',
            'size': 41},
        'baz': {
            'md5': 'ad00b6d1a3b6f2e73d4e40bc846498ac',
            'name': 'baz',
            'size': 42}}

    tar_fh = create_test_tarball(test_files)
    results = util.extract_rentrak_metadata(tar_fh)
    names = set()
    for result in results:
        assert expected[result['name']] == result
        names.add(result['name'])
    assert names == expected.keys()


@patch('flows.cable_ingestion.util.config')
@patch('flows.cable_ingestion.util.s3')
def test_get_rows_from_tarball(s3, config):
    """Test get_rows_from_tarball function."""
    tar_files = OrderedDict()
    tar_files['foo'] = (
        'header1,header2,header3,header4\n'
        'this,is,a,line\n'
        'and,here,is,another').encode('utf-8')
    tar_files['bar'] = (
        'header1,header2,header3,header4\n'
        'is,this,a,test?').encode('utf-8')

    config.RAW_CSV_COLUMN_TYPES = []
    # expecting lines from file 'bar' first since it is sorted alphabetically
    expected = [
        ['is', 'this', 'a', 'test?'],
        ['this', 'is', 'a', 'line'],
        ['and', 'here', 'is', 'another']]

    fh = create_test_tarball(tar_files)
    s3_object = Mock()
    s3_object.download_fileobj = lambda x: x.write(fh.read())
    s3.get_object.return_value = s3_object
    results = util.get_rows_from_tarball(fh)
    assert list(results) == expected


@patch('flows.cable_ingestion.util.s3')
def test_get_rows_from_tarball_empty(s3):
    """Test get_rows_from_tarball function."""
    tar_files = {}

    fh = create_test_tarball(tar_files)
    s3_object = Mock()
    s3_object.download_fileobj = lambda x: x.write(fh.read())
    s3.get_object.return_value = s3_object

    with raises(EmptyGeneratorError):
        results = (util.get_rows_from_tarball(fh))
        next(results)


def test_normalize_csv_columns():
    """Test normalize_csv_columns function."""
    mapping = (
        ({0}, int),
        ({1, 3}, lambda x: 'lambda'))
    row = ['123', 'hello', ' foo ', '15', '']
    expected = [123, 'lambda', 'foo', 'lambda', None]
    results = util.normalize_csv_columns(row, mapping)
    assert expected == results


def test_extract_filename_dates():
    """Test rentrack filename metadata extraction."""
    prefix = 'rentrak_theorchard_subscriptions_'
    cases = (
        prefix + '12345678_98765432_12345671234567.ctl',
        prefix + '12121212_23232323_34343434343434.tar.gz',
        prefix + '2121212_23232323_34343434343434.tar.gz',
        prefix + '12345678_98765432_12345671234567.btl',
        'foo_bar_baz')
    results = (
        {
            'date_start': '12345678',
            'date_end': '98765432',
            'timestamp': '12345671234567'},
        {
            'date_start': '12121212',
            'date_end': '23232323',
            'timestamp': '34343434343434'},
        None,
        None,
        None,
    )
    test_cases = zip(cases, results)
    for case, result in test_cases:
        assert util.extract_filename_dates(case) == result