import collections
import datetime
from datetime import datetime as datetime_methods
import functools
import glob
import os
import shutil
import time
from unittest import mock
from unittest.mock import MagicMock
from unittest.mock import mock_open
from unittest.mock import patch

import fastavro as avro
import pytest

from processing_accounting.flows.custom_export import setting
from processing_accounting.flows.custom_export import tasks
from processing_accounting.util import db as db_util
from processing_accounting.util import dynamodb as dynamodb_util
from processing_accounting.util import logging
from processing_accounting.util import s3 as s3_util

AVRO_FILE_SCHEMA = {
    'fields': [
        {'type': 'string', 'name': 'period'},
        {'type': 'string', 'name': 'activity_period'},
        {'type': 'string', 'name': 'dms'},
        {'type': 'string', 'name': 'territory'},
        {'type': 'string', 'name': 'orchard_upc'},
        {'type': 'string', 'name': 'manufacturer_upc'},
        {'type': 'string', 'name': 'label_catalog'},
        {'type': 'string', 'name': 'subaccount'},
        {'type': 'string', 'name': 'imprint_label'},
        {'type': 'string', 'name': 'artist_name'},
        {'type': 'string', 'name': 'release_name'},
        {'type': 'string', 'name': 'track_name'},
        {'type': 'string', 'name': 'isrc'},
        {'type': 'string', 'name': 'volume'},
        {'type': 'string', 'name': 'track_number'},
        {'type': 'string', 'name': 'quantity'},
        {'type': 'string', 'name': 'unit_price'},
        {'type': 'string', 'name': 'gross'},
        {'type': 'string', 'name': 'trans_type'},
        {'type': 'string', 'name': 'adjusted_gross'},
        {'type': 'string', 'name': 'split_rate'},
        {'type': 'string',
         'name': 'label_share_net_receipts'},
        {'type': 'string', 'name': 'ringtone_publishing'},
        {'type': 'string', 'name': 'cloud_publishing'},
        {'type': 'string', 'name': 'publishing'},
        {'type': 'string', 'name': 'mech_administrative_fee'},
        {'type': 'string',
         'name': 'subaccount_label_share_net_receipts'},
        {'type': 'string', 'name': 'preferred_currency'},
        {'type': 'string', 'name': 'statement_detail_id'},
        {'type': 'string', 'name': 'isdistributor'},
        {'type': 'string', 'name': 'user_id_type'}]
}
REPORT_SCHEMA = collections.OrderedDict([
    ('field1', 'Field 1'), ('aaa', 'Aaa'), ('field3', 'Field 3'),
    ('random', 'Random')])
EXPECTED_HEADERS = ['Field 1', 'Aaa', 'Field 3', 'Random']
FULL_REPORT_TYPES_STR = 'all'
CUSTOM_REPORT_TYPES_STR = 'S,VR'
PHYSICAL_REPORT_TYPES_STR = 'physical'
PHYSICAL_TRANSACTION_TYPES_STR = 'PS,RE'
# Exclude columns constants
BASE_EXCLUDE_COLUMNS = ['statement_detail_id', 'isdistributor', 'user_id_type']
PHYSICAL_REPORT_EXCLUDED_COLUMNS = [
    'manufacturer_upc', 'isrc', 'volume', 'track_name', 'track_number',
    'ringtone_publishing']
# Label
EXCLUDED_COLUMNS_LABEL = [
    'subaccount', 'subaccount_label_share_net_receipts'
]
EXCLUDED_COLUMNS_LABEL_V2 = BASE_EXCLUDE_COLUMNS + EXCLUDED_COLUMNS_LABEL
EXCLUDED_COLUMNS_LABEL_PHYSICAL = (
    BASE_EXCLUDE_COLUMNS + PHYSICAL_REPORT_EXCLUDED_COLUMNS +
    EXCLUDED_COLUMNS_LABEL
)
# Subaccount
EXCLUDED_COLUMNS_SUBACCOUNT = [
    'subaccount', 'unit_price', 'gross', 'adjusted_gross', 'split_rate',
    'ringtone_publishing', 'cloud_publishing', 'publishing',
    'mech_administrative_fee', 'label_share_net_receipts', 'original_price',
    'discount',
]
EXCLUDED_COLUMNS_SUBACCOUNT_V2 = (
    BASE_EXCLUDE_COLUMNS + EXCLUDED_COLUMNS_SUBACCOUNT)
EXCLUDED_COLUMNS_SUBACCOUNT_PHYSICAL = (
    BASE_EXCLUDE_COLUMNS + PHYSICAL_REPORT_EXCLUDED_COLUMNS +
    EXCLUDED_COLUMNS_SUBACCOUNT)
# Distributor
EXCLUDED_COLUMNS_DISTRIBUTOR = ['subaccount_label_share_net_receipts']
EXCLUDED_COLUMNS_DISTRIBUTOR_V2 = (
    BASE_EXCLUDE_COLUMNS + EXCLUDED_COLUMNS_DISTRIBUTOR)
EXCLUDED_COLUMNS_DISTRIBUTOR_PHYSICAL = (
    BASE_EXCLUDE_COLUMNS + PHYSICAL_REPORT_EXCLUDED_COLUMNS +
    EXCLUDED_COLUMNS_DISTRIBUTOR)


def test_apply_format():
    """Test _apply_format
    """

    actual = tasks.apply_format('4.6824|es_ES')
    expected = '4,682400'
    assert actual == expected

    actual = tasks.apply_format('4.6824|en_US')
    expected = '4.682400'
    assert actual == expected

    actual = tasks.apply_format('488593.6824|es_ES')
    expected = '488.593,682400'
    assert actual == expected

    actual = tasks.apply_format('488593|es_ES', '#,###')
    expected = '488.593'
    assert actual == expected

    actual = tasks.apply_format('488593|en_US', '#,###')
    expected = '488,593'
    assert actual == expected


@pytest.mark.parametrize(
    'isdistributor, accaunt_type, report_version, transaction_types,'
    'expected_exclude_columns', [
        # label
        ('N', 'label', 2, CUSTOM_REPORT_TYPES_STR, EXCLUDED_COLUMNS_LABEL_V2),
        ('N', 'label', 2, PHYSICAL_REPORT_TYPES_STR,
         EXCLUDED_COLUMNS_LABEL_PHYSICAL),
        # subaccount
        ('N', 'subaccount', 2, CUSTOM_REPORT_TYPES_STR,
         EXCLUDED_COLUMNS_SUBACCOUNT_V2),
        ('N', 'subaccount', 2, PHYSICAL_REPORT_TYPES_STR,
         EXCLUDED_COLUMNS_SUBACCOUNT_PHYSICAL),
        # distributor
        ('Y', 'label', 2, CUSTOM_REPORT_TYPES_STR,
         EXCLUDED_COLUMNS_DISTRIBUTOR_V2),
        ('Y', 'label', 2, PHYSICAL_REPORT_TYPES_STR,
         EXCLUDED_COLUMNS_DISTRIBUTOR_PHYSICAL),
    ])
def test_get_excluded_columns(
        isdistributor, accaunt_type, report_version, transaction_types,
        expected_exclude_columns, monkeypatch):
    """Test _get_excluded_columns
    """
    monkeypatch.setattr(
        db_util, 'snowflake_query',
        MagicMock(return_value=[{'ISDISTRIBUTOR': isdistributor}]))
    actual = tasks._get_excluded_columns(
        8576, accaunt_type, report_version, transaction_types)
    assert actual == expected_exclude_columns


@pytest.mark.parametrize('subaccount_split, gross, expected_result', [
    ({'subaccount_split_type': 'Gross', 'commissionoverride': '1'},
     '0.1',
     '0.1'),
    ({'subaccount_split_type': 'Gross', 'commissionoverride': '0.5'},
     '0.1',
     '0.05'),
])
def test_calculate_subaccount_split(
        subaccount_split, gross, expected_result):
    """Test calculate_subaccount_split utility function."""
    result = tasks.calculate_subaccount_split(subaccount_split, gross)
    assert result == expected_result


@pytest.mark.parametrize('user_type, expected_result', [
    ('label', {}),
    ('subaccount', {'expected': 'result'})
])
@patch('processing_accounting.flows.custom_export.tasks.db')
def test_get_subaccount_split(db, user_type, expected_result):
    """Test get_subaccount_split utility function."""
    expected_sql = (
        'select subaccount_split_type, commissionoverride '
        'from dim_subaccount '
        'where subaccountid = 4242')
    user_id = '4242'
    db.snowflake_query.return_value = [expected_result]
    kwargs = {}
    result = tasks.get_subaccount_split(
        user_type, user_id, **kwargs)
    assert result == expected_result
    if user_type == 'subaccount':
        db.snowflake_query.assert_called_once_with(expected_sql)
    else:
        assert db.snowflake_query.call_count == 0


@patch('processing_accounting.flows.custom_export.tasks.codecs')
def test_write_files(mock_codecs, monkeypatch):
    """Test _write_files
    """

    mock_process = MagicMock()
    monkeypatch.setattr(
        mock_process, 'start', MagicMock(return_value='process1'))

    file_write = MagicMock()
    file_write.write.return_value = None
    monkeypatch.setattr(
        dynamodb_util, 'set_status', MagicMock(return_value=None))
    monkeypatch.setattr(
        logging.logger, 'info', MagicMock(return_value=None))
    mock_record = {'fieldA': '2', 'fieldB': '4', 'fieldC': 'test"quotes'}
    mock_codecs.open.return_value = file_write
    locale = 'en_US'
    money_columns = ['fieldB']
    integer_columns = ['quantity']
    part = 1
    index = 3
    user_id_type = '18805L'
    user_params = '18805__label__199__all__es_ES__xls'
    header = ['Field A', 'Field B', 'Field C']
    file_format = 'xls'
    report_schema = collections.OrderedDict([
        ('fieldB', 'Field B'), ('fieldC', 'Field C'),
    ])
    (returned_file_write, act_i, act_part) = tasks._write_files(
        file_write, 'filename', mock_record,
        locale, money_columns, integer_columns, part, index,
        report_schema, user_id_type, user_params, header, file_format)
    assert act_i == 4
    assert act_part == 1
    index = setting.LINES_PER_FILE - 1
    (returned_file_write, act_i, act_part) = tasks._write_files(
        file_write, 'filename', mock_record,
        locale, money_columns, integer_columns, part, index,
        report_schema, user_id_type, user_params, header, file_format)
    assert act_i == setting.LINES_PER_FILE
    assert act_part == 2
    assert mock_codecs.open.called
    mock_codecs.open.assert_any_call(
        'filename_part2.xls', 'a', 'utf-16')

    mock_record['quantity'] = 2234
    report_schema = collections.OrderedDict([
        ('fieldB', 'Field B'), ('fieldC', 'Field C'), ('quantity', 'Quantity'),
    ])
    tasks._write_files(
        file_write, 'filename', mock_record,
        locale, money_columns, integer_columns, part, index,
        report_schema, user_id_type, user_params, header, file_format)
    file_write.write.assert_any_call(
        '"4.000000"\t"test""quotes"\t"2,234"\n')


@pytest.mark.parametrize('product_code, expected_value', [
    (None, ''),
    ('', ''),
    ('TEST', 'TEST'),
])
@patch('processing_accounting.flows.custom_export.tasks.codecs')
def test_write_files_empty_value(
        mock_codecs, monkeypatch, product_code, expected_value):
    mock_process = MagicMock()
    monkeypatch.setattr(
        mock_process, 'start', MagicMock(return_value='process1'))
    file_write = MagicMock()
    file_write.write.return_value = None
    monkeypatch.setattr(
        dynamodb_util, 'set_status', MagicMock(return_value=None))
    monkeypatch.setattr(
        logging.logger, 'info', MagicMock(return_value=None))
    mock_codecs.open.return_value = file_write

    report_schema = collections.OrderedDict([
        ('fieldB', 'Field B'), ('fieldC', 'Field C'),
        ('product_code', 'Product code'),
    ])
    mock_record = {
        'fieldA': '2',
        'fieldB': '4',
        'fieldC': 'test"quotes',
        'product_code': product_code
    }
    locale = 'en_US'
    money_columns = ['fieldB']
    integer_columns = []
    part = 1
    index = setting.LINES_PER_FILE - 1
    user_id_type = '18805L'
    user_params = '18805__label__199__all__es_ES__xls'
    header = ['Field B', 'Field C', 'Product code']
    file_format = 'xls'
    # test product_code with empty string
    tasks._write_files(
        file_write, 'filename', mock_record,
        locale, money_columns, integer_columns, part,
        index, report_schema, user_id_type, user_params,
        header, file_format)
    expected_write_value = '"4.000000"\t"test""quotes"\t"{}"\n'.format(
        expected_value)
    file_write.write.assert_any_call(expected_write_value)


@pytest.mark.parametrize('calc_subaccount_split, avro_value, expected_value', [
    ('4242', '1212', '4242'),
    ('', '1212', '1212')
])
@patch(
    'processing_accounting.flows.custom_export.tasks.'
    'calculate_subaccount_split')
@patch('processing_accounting.flows.custom_export.tasks.codecs')
def test_write_files_subaccount_split_calculation(
        mock_codecs, calculate_subaccount_split, monkeypatch,
        calc_subaccount_split, avro_value, expected_value):
    """Test that _write_files invokes split calculation."""
    mock_process = MagicMock()
    monkeypatch.setattr(
        mock_process, 'start', MagicMock(return_value='process1'))
    file_write = MagicMock()
    file_write.write.return_value = None
    monkeypatch.setattr(
        dynamodb_util, 'set_status', MagicMock(return_value=None))
    monkeypatch.setattr(
        logging.logger, 'info', MagicMock(return_value=None))
    mock_codecs.open.return_value = file_write

    subaccount_split = 'dummy split data'
    calculate_subaccount_split.return_value = calc_subaccount_split

    report_schema = collections.OrderedDict([
        ('fieldB', 'Field B'),
        ('subaccount_label_share_net_receipts', 'Subaccount Split'),
    ])
    gross = 'dummy'
    mock_record = {
        'fieldB': '4',
        'gross': gross,
        'subaccount_label_share_net_receipts': avro_value
    }
    locale = 'en_US'
    money_columns = ['fieldB']
    integer_columns = []
    part = 1
    index = setting.LINES_PER_FILE - 1
    user_id_type = '18805L'
    user_params = '18805__label__199__all__es_ES__xls'
    header = ['Field B', 'Field C', 'Product code']
    file_format = 'xls'
    tasks._write_files(
        file_write, 'filename', mock_record,
        locale, money_columns, integer_columns, part,
        index, report_schema, user_id_type, user_params,
        header, file_format, subaccount_split)
    expected_write_value = '"4.000000"\t"{}"\n'.format(
        expected_value)
    file_write.write.assert_any_call(expected_write_value)
    calculate_subaccount_split.assert_called_once_with(
        subaccount_split, gross)


@pytest.mark.parametrize('locale, expected_number_format', [
    ('en_US', 'US'),
    ('es_ES', 'EU'),
])
@pytest.mark.parametrize('transaction_types, expected_report_type', [
    ('DT,AS', 'customreport'),
    ('PS,RE', 'customreport'),
    ('all', 'fullreport'),
    ('physical', 'physicalreport'),
])
@patch('processing_accounting.flows.custom_export'
       '.tasks.datetime.datetime')
def test_get_final_report_name(
        mock_today, monkeypatch, locale, expected_number_format,
        transaction_types, expected_report_type):
    """"Test _get_final_report_name
    """

    mock_today.today.return_value = datetime_methods.strptime(
        '2015-10-01', '%Y-%m-%d')
    mock_today.side_effect = lambda *args, **kw: datetime.date(*args, **kw)

    monkeypatch.setattr(
        time, 'strftime', MagicMock(return_value='2015-10-01'))

    mock_row = {
        'YEAR': '2016',
        'QUARTER': '4',
        'LABEL_NAME': 'Cleopatra Records',
        'SUBACCOUNT_NAME': 'Paulo Records',
        'PAYMENT_INTERVAL': 'quarter',
        'MONTH': 'January'
    }
    monkeypatch.setattr(
        db_util, 'snowflake_query', MagicMock(return_value=[mock_row]))

    expected_params = {
        'date': '20151001',
        'reporting_period': 'Q42016',
        'report_type': expected_report_type,
        'user_name': 'Cleopatra Records',
        'number_format': expected_number_format,
    }
    actual = tasks._get_report_file_name_parts(
        '202,203,204', '8869', 'label', locale, transaction_types)
    assert actual == expected_params


def test_get_avro_local_path(monkeypatch):
    """Test _get_avro_local_path
    """

    monkeypatch.setattr(
        os.path, 'exists', MagicMock(return_value=False))

    monkeypatch.setattr(
        os, 'makedirs', MagicMock(return_value=None))

    monkeypatch.setattr(
        tasks, '_get_user_params', MagicMock(
            return_value='8869__label__202,203,204__all__en_US__xls'))

    actual = tasks._get_avro_local_path(
        '8869', 'label', '202,203,204', 'all', 'en_US', 'xls')
    assert actual == (
        '{}/custom_report/8869L/avro/'
        '8869__label__202,203,204__all__en_US__xls.avro').format(
        setting.TEMP_DIR)


def test_get_local_path(monkeypatch):
    """Test _get_local_path
    """

    monkeypatch.setattr(
        tasks, '_get_report_file_name_parts', MagicMock(return_value={
            'date': '20151001',
            'reporting_period': 'Q42016',
            'report_type': 'fullreport',
            'user_name': 'Paulo /Records?#',
            'number_format': 'EU'}))

    monkeypatch.setattr(
        os.path, 'exists', MagicMock(return_value=True))

    monkeypatch.setattr(
        tasks, '_get_user_params', MagicMock(
            return_value='8869__label__202,203,204__DA,DT__en_US__txt'))

    monkeypatch.setattr(
        os, 'makedirs', MagicMock(return_value=None))

    monkeypatch.setattr(
        tasks, '_flush_dir', MagicMock(return_value=None))

    actual = tasks._get_local_path(
        '8869', 'label', '202,203,204', 'DA,DT', 'en_US', 'txt')

    assert actual == (
        '{}/custom_report/8869L/8869__label__202,203,204__DA,DT__en_US__txt/'
        '20151001_Q42016_fullreport_paulo_records_EU').format(setting.TEMP_DIR)


def test_get_local_path_foreign_language_name(monkeypatch):
    """Test _get_local_path
    """

    monkeypatch.setattr(
        tasks, '_get_report_file_name_parts', MagicMock(return_value={
            'date': '20151001',
            'reporting_period': 'Q42016',
            'report_type': 'fullreport',
            'user_name': 'التسمية اختبار',
            'number_format': 'EU'}))

    monkeypatch.setattr(
        os.path, 'exists', MagicMock(return_value=True))

    monkeypatch.setattr(
        tasks, '_get_user_params', MagicMock(
            return_value='8869__label__202,203,204__DA,DT__en_US__txt'))

    monkeypatch.setattr(
        os, 'makedirs', MagicMock(return_value=None))

    monkeypatch.setattr(
        tasks, '_flush_dir', MagicMock(return_value=None))

    actual = tasks._get_local_path(
        '8869', 'label', '202,203,204', 'DA,DT', 'en_US', 'txt')

    assert actual == (
        '{}/custom_report/8869L/8869__label__202,203,204__DA,DT__en_US__txt/'
        '20151001_Q42016_fullreport_ltsmy_khtbr_EU').format(setting.TEMP_DIR)


@pytest.mark.parametrize('report_version, expected_report_version', [
    (None, setting.DEFAULT_REPORT_VERSION),
    (1, 1),
    (2, 2),
])
def test_bootstrap(monkeypatch, report_version, expected_report_version):
    """Test bootstrap"""

    mock_dynamodb_item = {
        's3_path': 's3://this_is_a_test'
    }

    monkeypatch.setattr(
        dynamodb_util, 'get_items', MagicMock(
            return_value=[mock_dynamodb_item]))

    monkeypatch.setattr(
        tasks, '_get_local_path', MagicMock(return_value='local_file_path'))

    monkeypatch.setattr(
        tasks, '_get_avro_local_path', MagicMock(
            return_value='avro_file_path'))

    monkeypatch.setattr(
        dynamodb_util, 'set_status', MagicMock(
            return_value=None))

    monkeypatch.setattr(
        tasks, '_flush_dir', MagicMock(return_value=None))

    monkeypatch.setattr(
        os.path, 'exists', MagicMock(return_value=True))

    monkeypatch.setattr(
        os, 'remove', MagicMock(return_value=None))

    mock_activity = MagicMock()
    monkeypatch.setattr(
        mock_activity, 'info', MagicMock(return_value=None))

    monkeypatch.setattr(
        tasks, '_get_excluded_columns', MagicMock(return_value=['a']))

    monkeypatch.setattr(
        tasks, '_get_user_params', MagicMock(
            return_value='202,203,204__DA,DT__txt__es_ES'))

    actual = tasks.bootstrap(
        mock_activity, '203,202,204', 'label', '8869', 'DT,DA', 'es_ES', 'txt',
        report_version)

    dynamodb_util.set_status.assert_any_call(
        '8869L', '202,203,204__DA,DT__txt__es_ES', 'PENDING', file_type='txt',
        number_format='es_ES', period_ids='202,203,204',
        transaction_types='DA,DT', account_id='8869',
        account_type='label')

    expected = {
        'money_columns': setting.money_columns,
        'avro_file_s3_path': 's3://this_is_a_test',
        'transaction_types': 'DA,DT',
        'period_ids': '202,203,204',
        'file_format': 'txt',
        'user_id_type': '8869L',
        'user_params': '202,203,204__DA,DT__txt__es_ES',
        'report_version': expected_report_version}
    assert actual == expected


@pytest.mark.parametrize('file_format', ['xls', 'txt'])
@pytest.mark.parametrize('mock_trans_type', ['S', 'VR', 'AV', 'PS', 'RE'])
@pytest.mark.parametrize(
    'transaction_types, excluded_trans_types',
    [
        # no filtering should happen
        (FULL_REPORT_TYPES_STR, []),
        # everything that is not in CUSTOM_REPORT_TYPES should be excluded
        (CUSTOM_REPORT_TYPES_STR, ['AV', 'PS', 'RE']),
        # everithyng that is not in PHYSICAL_REPORT_TYPES should be excluded
        (PHYSICAL_REPORT_TYPES_STR, ['S', 'VR', 'AV'])
    ])
@patch('processing_accounting.flows.custom_export.tasks.codecs')
def test_generate_report(
        mock_codecs, monkeypatch, file_format, mock_trans_type,
        transaction_types, excluded_trans_types):
    """Test generate_report"""

    mock_activity = MagicMock()
    monkeypatch.setattr(
        mock_activity, 'info', MagicMock(return_value=None))

    monkeypatch.setattr(
        tasks, '_get_local_path', MagicMock(return_value='local_file_path'))

    monkeypatch.setattr(
        tasks, '_get_avro_local_path', MagicMock(
            return_value='local_avro_file_path'))

    monkeypatch.setattr(
        tasks, '_get_excluded_columns', MagicMock(return_value=['a']))

    monkeypatch.setattr(
        os.path, 'dirname', MagicMock(return_value=True))

    monkeypatch.setattr(
        tasks, '_flush_dir', MagicMock(return_value=None))

    monkeypatch.setattr(
        s3_util, 'download_boto3', MagicMock(return_value=[]))

    monkeypatch.setattr(
        os, 'remove', MagicMock(return_value=None))

    monkeypatch.setattr(
        os.path, 'exists', MagicMock(return_value=None))

    monkeypatch.setattr(
        tasks, '_flush_dir', MagicMock(return_value=None))

    mock_process = MagicMock()
    monkeypatch.setattr(
        mock_process, 'start', MagicMock(return_value='process1'))
    user_params = '202,203,204__{}__{}__es_ES'.format(
        transaction_types, file_format)
    monkeypatch.setattr(
        tasks, '_get_user_params', MagicMock(
            return_value=user_params))

    m = mock_open()
    file_write = MagicMock()
    monkeypatch.setattr(
        file_write, 'close', MagicMock(return_value=None))
    monkeypatch.setattr(
        file_write, 'write', MagicMock(return_value=None))

    mock_codecs.open.return_value = file_write

    with patch('builtins.open', m):
        mock_reader = MagicMock()
        mock_reader.schema = {'fields': [{'name': 'fieldA'}]}
        mock_reader.__iter__.return_value = [{'trans_type': mock_trans_type}]
        monkeypatch.setattr(
            avro, 'reader', MagicMock(return_value=mock_reader))
        monkeypatch.setattr(
            tasks, '_write_files',
            MagicMock(return_value=(file_write, 1, 2)))

        monkeypatch.setattr(
            tasks, '_get_header', MagicMock(return_value=['fieldA', 'fieldB']))

        monkeypatch.setattr(glob, 'glob', MagicMock(return_value=[]))

        monkeypatch.setattr(
            shutil, 'copyfile', MagicMock(return_value=None))

        monkeypatch.setattr(
            os, 'rename', MagicMock(return_value=None))

        monkeypatch.setattr(
            tasks, '_flush_dir', MagicMock(return_value=None))

        monkeypatch.setattr(
            logging.logger, 'info', MagicMock(return_value=None))

        monkeypatch.setattr(
            dynamodb_util, 'set_status', MagicMock(return_value=None))

        user_id = '8869'
        user_type = 'label'

        avro_file_s3_path = 'avro_file_s3_path'
        locale = 'en_US'
        period_ids = '202,203,204'
        redownload = True
        tasks.generate_report(
            mock_activity, user_id, user_type, avro_file_s3_path,
            transaction_types, locale, file_format, period_ids, redownload,
            setting.DEFAULT_REPORT_VERSION)

        # test that file was opened
        m.assert_any_call('local_avro_file_path', 'rb')
        append_path = '{}.{}'.format('local_file_path', file_format)
        encodings = {
            'txt': 'utf-8',
            'xls': 'utf-16',
        }
        expected_encoding = encodings.get(file_format)
        mock_codecs.open.assert_any_call(append_path, 'a', expected_encoding)

        if mock_trans_type in excluded_trans_types:
            assert not tasks._write_files.called
        else:
            assert tasks._write_files.called


def test_flush_dir(monkeypatch):
    monkeypatch.setattr(os, 'remove', MagicMock(return_value=None))
    monkeypatch.setattr(os.path, 'exists', MagicMock(return_value=True))
    monkeypatch.setattr(
        os.path, 'dirname', MagicMock(return_value='aaaa/bbbb'))
    monkeypatch.setattr(os, 'makedirs', MagicMock(return_value=True))

    resp = tasks._flush_dir('aaaa/bbbb/cccc.txt')
    assert resp is None
    os.path.dirname.assert_any_call('aaaa/bbbb/cccc.txt')
    os.path.exists.assert_any_call('aaaa/bbbb')


def test_get_header():
    """Test _get_header method
    """
    actual = tasks._get_header(REPORT_SCHEMA)

    assert actual == EXPECTED_HEADERS


@patch('processing_accounting.flows.custom_export.tasks.zipfile')
def test_zip_up_files(mock_zipfile, monkeypatch):
    """Test zip_up_files
    """

    mock_zip = MagicMock()
    mock_zip.write.return_value = None
    mock_zip.close.return_value = None
    mock_zipfile.ZipFile.return_value = mock_zip

    monkeypatch.setattr(glob, 'glob', MagicMock(return_value=['test_file']))

    resp = tasks.zip_up_files(
        MagicMock(), '/var/app/source_path', '/var/app/dest_path/test.zip')
    assert resp.get('zip_file_name') == 'test.zip'

    monkeypatch.setattr(glob, 'glob', MagicMock(return_value=[]))
    resp = tasks.zip_up_files(
        MagicMock(), '/var/app/source_path', '/var/app/dest_path/test.zip')
    assert resp.get('stop')
    assert resp.get('message') == '"No file in /var/app/dest_path".'


@patch('processing_accounting.flows.custom_export.tasks.setting.ENV', 'prod')
def test_get_history_page_url_prod():
    """Test _get_history_page_url on production
    """

    actual = tasks._get_history_page_url('205,206,207', 'Q12016')
    assert actual == (
        'https://workstation.theorchard.com/accounting/statementshistory/'
        'period/Q1_2016/205/207/history')


@patch('processing_accounting.flows.custom_export.tasks.setting.ENV', 'dev')
def test_get_history_page_url_dev():
    """Test _get_history_page_url in dev
    """

    actual = tasks._get_history_page_url('205,206,207', 'Q12016')
    assert actual == (
        'http://workstation.192.168.99.100.xip.io/accounting/'
        'statementshistory/period/Q1_2016/205/207/history')


@patch('processing_accounting.flows.custom_export.tasks.setting.ENV', 'qa')
def test_get_history_page_url_qa():
    """Test _get_history_page_url in qa
    """

    actual = tasks._get_history_page_url('205,206,207', 'Q12016')
    assert actual == (
        'https://workstation.qaorch.com/accounting/statementshistory/'
        'period/Q1_2016/205/207/history')


@patch('processing_accounting.flows.custom_export.tasks.ses')
@patch('processing_accounting.flows.custom_export.tasks.setting.ENV', 'qa')
def test_send_email_qa(mock_ses, monkeypatch):
    """Test send_email
    """

    monkeypatch.setattr(
        tasks, '_get_report_file_name_parts', MagicMock(return_value={
            'reporting_period': 'Jun2016',
            'report_type': 'customreport',
            'user_name': 'Allegro',
            'number_format': 'US'
        }))
    mock_ses.send_email.return_value = None
    tasks.send_email(
        MagicMock(), 'paulo@theorchard.com', '202,203,204', '18805',
        'label', 'en_US', 'DA,DT', 'txt')

    mock_ses.send_email.assert_any_call(
        setting.FILE_IS_READY_EMAIL, 'donotreply@theorchard.com',
        'Accounting Statement Ready For Download', (
            '<p>Your accounting statement has been generated. Please '
            '<a href="https://workstation.qaorch.com/accounting/'
            'statementshistory/period/Jun_2016/202/204/history">'
            'click here</a> to download the statement from the '
            'Workstation.</p><p><b>Label Name</b>: Allegro<br>'
            '<b>Reporting Period</b>: Jun2016<br><b>Report Type</b>: '
            'customreport<br><b>Number Format</b>: US<br><b>File Type'
            '</b>: txt<br></p>'))


@patch('processing_accounting.flows.custom_export.tasks.ses')
@patch('processing_accounting.flows.custom_export.tasks.setting.ENV', 'prod')
def test_send_email_prod(mock_ses, monkeypatch):
    """Test send_email
    """

    monkeypatch.setattr(
        tasks, '_get_report_file_name_parts', MagicMock(return_value={
            'reporting_period': 'Jun2016',
            'report_type': 'customreport',
            'user_name': 'Allegro',
            'number_format': 'US'
        }))
    mock_ses.send_email.return_value = None
    tasks.send_email(
        MagicMock(), 'paulo@theorchard.com', '202,203,204', '18805',
        'label', 'en_US', 'DA,DT', 'txt')

    mock_ses.send_email.assert_any_call(
        'paulo@theorchard.com', 'donotreply@theorchard.com',
        'Accounting Statement Ready For Download', (
            '<p>Your accounting statement has been generated. Please '
            '<a href="https://workstation.theorchard.com/accounting/'
            'statementshistory/period/Jun_2016/202/204/history">'
            'click here</a> to download the statement from the '
            'Workstation.</p><p><b>Label Name</b>: Allegro<br>'
            '<b>Reporting Period</b>: Jun2016<br><b>Report Type</b>: '
            'customreport<br><b>Number Format</b>: US<br><b>File Type'
            '</b>: txt<br></p>'))


@patch('processing_accounting.flows.custom_export.tasks.ses')
@patch('processing_accounting.flows.custom_export.tasks.setting.ENV', 'dev')
def test_send_email_dev(mock_ses, monkeypatch):
    """Test send_email
    """

    monkeypatch.setattr(
        tasks, '_get_report_file_name_parts', MagicMock(return_value={
            'reporting_period': 'Jun2016',
            'report_type': 'customreport',
            'user_name': 'Allegro',
            'number_format': 'US'
        }))
    mock_ses.send_email.return_value = None
    tasks.send_email(
        MagicMock(), 'paulo@theorchard.com', '202,203,204', '18805',
        'label', 'en_US', 'DA,DT', 'txt')

    mock_ses.send_email.assert_any_call(
        setting.FILE_IS_READY_EMAIL, 'donotreply@theorchard.com',
        'Accounting Statement Ready For Download', (
            '<p>Your accounting statement has been generated. Please '
            '<a href="http://workstation.192.168.99.100.xip.io/accounting/'
            'statementshistory/period/Jun_2016/202/204/history">'
            'click here</a> to download the statement from the '
            'Workstation.</p><p><b>Label Name</b>: Allegro<br>'
            '<b>Reporting Period</b>: Jun2016<br><b>Report Type</b>: '
            'customreport<br><b>Number Format</b>: US<br><b>File Type'
            '</b>: txt<br></p>'))


@patch('processing_accounting.flows.custom_export.tasks.shutil')
@patch(
    'processing_accounting.flows.custom_export.tasks.check_active_workflows')
@patch('processing_accounting.flows.custom_export.tasks.os')
def test_clean_shared_avro_file_status_generating(
        mock_os, check_active_workflows, shutil):
    """Test clean_shared_avro_file."""
    user_id_type = '18805L'
    mock_os.path.join.side_effect = os.path.join
    mock_activity = MagicMock()
    mock_activity.logger.info.return_value = None
    mock_os.remove.return_value = None
    check_active_workflows.return_value = [
        {'status': 'GENERATING'}
    ]
    tasks.clean_shared_avro_file(mock_activity, '18805', 'label', '199')
    shutil.rmtree.assert_not_called()
    check_active_workflows.assert_called_once_with(
        mock_activity, user_id_type)


@patch('processing_accounting.flows.custom_export.tasks.setting')
@patch('processing_accounting.flows.custom_export.tasks.shutil')
@patch(
    'processing_accounting.flows.custom_export.tasks.check_active_workflows')
@patch('processing_accounting.flows.custom_export.tasks.os')
def test_clean_shared_avro_file_status_generated(
        mock_os, check_active_workflows, shutil, setting):
    """Test clean_shared_avro_file
    """
    user_id_type = '18805L'
    mock_os.path.join.side_effect = os.path.join
    expport_generation_dir = 'some_test_dir'
    setting.EXPORT_GENERATION_DIR = expport_generation_dir
    mock_activity = MagicMock()
    mock_activity.logger.info.return_value = None
    check_active_workflows.return_value = []
    expected_rm_dir = os.path.join(expport_generation_dir, user_id_type)

    tasks.clean_shared_avro_file(mock_activity, '18805', 'label', '199')
    shutil.rmtree.assert_has_calls([mock.call(expected_rm_dir)])
    check_active_workflows.assert_called_once_with(
        mock_activity, user_id_type)


@patch('processing_accounting.flows.custom_export.tasks.shutil')
@patch(
    'processing_accounting.flows.custom_export.tasks.check_active_workflows')
@patch('processing_accounting.flows.custom_export.tasks.os')
def test_clean_shared_avro_file_status_pending(
        mock_os, check_active_workflows, shutil):
    """Test clean_shared_avro_file
    """
    user_id_type = '18805L'
    mock_os.path.join.side_effect = os.path.join
    mock_activity = MagicMock()
    mock_activity.logger.info.return_value = None
    mock_os.remove.return_value = None
    check_active_workflows.return_value = [
        {'status': 'PENDING'}
    ]
    tasks.clean_shared_avro_file(mock_activity, '18805', 'label', '199')
    shutil.rmtree.assert_not_called()
    check_active_workflows.assert_called_once_with(
        mock_activity, user_id_type)


@pytest.mark.parametrize('items', [
    [{'status': 'GENERATED'}],
    [{'status': 'GENERATING', 'file_type': 'AVRO'}],
    # no download_avro_file_start, should result in empty list
    [{'status': 'GENERATING', 'file_type': 'xls'}],
])
@patch('processing_accounting.flows.custom_export.tasks.dynamodb')
def test_check_active_workflows_skips_not_suitable_items(dynamodb, items):
    """test check_active_workflows utility function."""
    user_id_type = '4242L'
    mock_items = [MagicMock(get=functools.partial(item.get))
                  for item in items]
    dynamodb.get_items.return_value = mock_items
    activity = MagicMock()

    result = tasks.check_active_workflows(activity, user_id_type)
    assert result == []
    dynamodb.get_items.assert_called_once_with(user_id_type)


@pytest.mark.parametrize('items', [
    [{'status': 'GENERATED', 'delete': False}],
    [{'status': 'GENERATING', 'file_type': 'AVRO', 'delete': False}],
    # no download_avro_file_start, treat as invalid record
    [{'status': 'GENERATING', 'file_type': 'xls', 'delete': True}],
    [
        {
            'status': 'GENERATING',
            'file_type': 'xls',
            'download_avro_file_start': 'expired',
            'delete': True,
        },
        {
            'status': 'GENERATING',
            'file_type': 'xls',
            'download_avro_file_start': 'not_expired',
            'delete': False,
        }]
])
@patch('processing_accounting.flows.custom_export.tasks.dt_util')
@patch('processing_accounting.flows.custom_export.tasks.dynamodb')
def test_check_active_workflows_deletes_inactive_items(
        dynamodb, dt_util, items):
    """Test check_active_workflows deletes inactive record."""
    user_id_type = '4242L'
    mock_items = [MagicMock(get=functools.partial(item.get))
                  for item in items]
    dynamodb.get_items.return_value = mock_items
    activity = MagicMock()
    expected_result = [
        item for item in mock_items
        if item.get('download_avro_file_start') == 'not_expired']

    def formatted_str(download_start):
        return download_start
    dt_util.formatted_str_to_datetime.side_effect = formatted_str

    def is_expired(download_start):
        return download_start == 'expired'
    dt_util.is_expired_workflow.side_effect = is_expired

    expected_dt_util_calls = [
        mock.call(item.get('download_avro_file_start'))
        for item in items if item.get('download_avro_file_start')
    ]
    expected_delete_item_calls = [
        mock.call(item) for item in mock_items if item.get('delete')
    ]

    result = tasks.check_active_workflows(activity, user_id_type)
    assert result == expected_result
    dt_util.formatted_str_to_datetime.assert_has_calls(expected_dt_util_calls)
    dt_util.is_expired_workflow.assert_has_calls(expected_dt_util_calls)
    dynamodb.get_items.assert_called_once_with(user_id_type)
    dynamodb.delete_item_object.assert_has_calls(expected_delete_item_calls)