import boto import boto.emr as emr import boto.swf.layer2 as swf import os import pytest from boto.s3.key import Key from boto import auth from garcon import activity from unittest.mock import MagicMock from moto import mock_s3 import uuid from octopus.flows.report_generation import tasks def create_bucket(): """Create an S3 bucket. Note: This cannot be a fixture – otherwise the mock doesn't retain the state of the bucket (including files that may have been saved). Return: boto.s3.bucket.Bucket: newly created S3 bucket """ bucket_name = str(uuid.uuid4()).lower()[:5] connection = boto.connect_s3() connection.create_bucket(bucket_name) return connection.get_bucket(bucket_name) def get_bucket_url(bucket): """Get the bucket url for a given bucket. Return: str: a well formed S3 url to the given bucket """ return 's3://{bucket_name}/'.format(bucket_name=bucket.name) def test_bootstrap(monkeypatch): """Test bootstrap """ monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock) resp = tasks.bootstrap(activity.Activity(), 'soundscan') assert isinstance(resp, dict) with pytest.raises(Exception): tasks.bootstrap(activity.Activity(), '') with pytest.raises(Exception): tasks.bootstrap(activity.Activity(), 'some_missing_module') @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') def test_search_latest_done_file(monkeypatch): """Test search_latest_done_file method Test case when done file is not there """ with mock_s3(): monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock) monkeypatch.setattr(Key, 'exists', MagicMock(return_value=False)) emr_inputs = { 'dim_track': 'dim_track/', 'active_contracts': 'active_contracts/', 'soundscan_us_release': 'soundscan_us_release/', 'soundscan_canada_release': 'soundscan_canada_release/', 'soundscan_canada_track': 'soundscan_canada_track/'} with pytest.raises(Exception): tasks.search_latest_done_file( activity.Activity(), 'some_bucket', emr_inputs) @pytest.mark.skipif( os.environ.get('PYTHON_SKIP_MOTO'), reason='Moto is not supported in this environment.') def test_search_latest_done_file_done_file_exists(monkeypatch): """Test the case when done file exist """ with mock_s3(): monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock()) monkeypatch.setattr(Key, 'exists', MagicMock(return_value=True)) monkeypatch.setattr(Key, 'get_contents_as_string', MagicMock(return_value='2015-01-01-12:00:01')) emr_inputs = { 'dim_track': 'dim_track/', 'active_contracts': 'active_contracts/', 'soundscan_us_release': 'soundscan_us_release/', 'soundscan_canada_release': 'soundscan_canada_release/', 'soundscan_canada_track': 'soundscan_canada_track/'} resp = tasks.search_latest_done_file( activity.Activity(), 'mybucket', emr_inputs) assert isinstance(resp, dict) def test_launch_emr_cluster(monkeypatch): """ Test launching emr cluster Test when there is an emr_cluster_id. It should pick up the already running emr cluster """ monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock()) connector = MagicMock() connector.set('add_tags', MagicMock(return_value=True)) connector.set('run_jobflow', MagicMock(return_value='some_cluster_id')) monkeypatch.setattr(emr, 'connect_to_region', MagicMock(return_value=connector)) resp = tasks.launch_emr_cluster(activity.Activity(), 'some_existing emr_cluster_id', 'path_to_logs', 'some_tag_name', 'some_instance_name', 'some_report_name') assert isinstance(resp, dict) assert resp.get('emr.cluster_id') == 'some_existing emr_cluster_id' def test_launch_new_emr_clsuter(monkeypatch): """ Test launching emr cluster Test when there is no emr_cluster_id provided. It should spawn up new cluster """ monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock()) connector = MagicMock() connector.add_tags = MagicMock(return_value=True) connector.run_jobflow = MagicMock(return_value='some_cluster_id') monkeypatch.setattr(emr, 'connect_to_region', MagicMock(return_value=connector)) resp = tasks.launch_emr_cluster(activity.Activity(), None, 'path_to_logs', 'some_tag_name', 'some_instance_name', 'some_report_name') assert resp.get('emr.cluster_id') == 'some_cluster_id' def test_add_report_generation_step(monkeypatch): """ Test add report generation emr step """ response = MagicMock() test_item = MagicMock() test_item.value = 'step1_id' response.stepids = [test_item] monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock()) connector = MagicMock() connector.add_jobflow_steps = MagicMock(return_value=response) monkeypatch.setattr(emr, 'connect_to_region', MagicMock(return_value=connector)) resp = tasks.add_report_generation_step( activity.Activity(), 'some_emr_cluster_id', 'some_bucket', 'dim_track/data/', 'some_hql_file_name', 'emr/soundscan/{delivery_date}/CANADA_PRODUCT/', 'some_step_name') assert resp.get('emr.step_id') == 'step1_id'