from boto import auth from garcon import activity from octopus.flows.dim_refresh import tasks from unittest.mock import MagicMock import boto.swf.layer2 as swf import pytest def test_get_chunk_query(monkeypatch): """Test bootstrap task """ context = { 'destination_s3_key': 'dim_label/data/dim_label.txt.gz', 'chunk_size': 100000, 'table': 'vendor', 'primary_key': 'vendor_id', 'bucket': 'prod-staging-dim-imports', 'query': '''select * from some_table where id between {min_id} and {max_id}''' } monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock) resp = tasks.get_chunk_query( activity.Activity(), context.get('query'), 10, 20, context.get('bucket'), context.get('destination_s3_key')) assert isinstance(resp, dict) assert resp.get('chunk_query').find('10') assert resp.get('chunk_query').find('20') new_name = context.get('destination_s3_key').replace('.txt.gz', '10-20.txt.gz') assert resp.get('destination_s3_key') == new_name def test_bootstrap_failures(monkeypatch): monkeypatch.setattr(auth, 'get_auth_handler', MagicMock()) monkeypatch.setattr(swf, 'ActivityWorker', MagicMock) context = { 'destination_s3_key': 'dim_label/data/dim_label.txt.gz', 'chunk_size': 100000, 'table': 'vendor', 'primary_key': 'vendor_id', 'bucket': 'prod-staging-dim-imports' } with pytest.raises(Exception): tasks.bootstrap_task(activity.Activity(), None, 10, 10000, context['bucket'], context['destination_s3_key']) def test_get_done_file(): destination_s3_key = 'somekey/somekey.txt.gz' resp = tasks.get_done_file_content('some_bucket', destination_s3_key) assert isinstance(resp, dict) assert resp.get('done_file_key') assert resp.get('done_file_content')