import json import os from typing import Any, Dict, List from unittest import mock import httpretty import pytest from slz_dbx_job_runner.entities import ( # yapf=disable AmazonClass, AmazonConfig, ApolloClass, ApolloConfig, ConfigBase, DBXJobParams ) from slz_dbx_job_runner.service import Service os.environ['HTTP_PROXY'] = os.environ['HTTPS_PROXY'] = '' @pytest.mark.parametrize( 'config,dbx_job_params,expected', [ ( ApolloConfig( id='1', command_class=ApolloClass( apple='--class cls1', spotify='--class cls2', ), command_opts='--driver-java-options -Ddelphi.etl.aws.secretsmanager.secrets' '.bigtable=delphi/prod/gcp/external-bigtable-user-hdd', command_jar='s3://prod-delphi-etl-artifacts/delphi-etl-apps.jar', url='dbx_url', token='dbx_token', ), DBXJobParams( dsp='apollo', report_date='2020-06-12', context='apple', ), [ '--class', 'cls1', '--driver-java-options', '-Ddelphi.etl.aws.secretsmanager.secrets.bigtable=' 'delphi/prod/gcp/external-bigtable-user-hdd', 's3://prod-delphi-etl-artifacts/delphi-etl-apps.jar', '--dsp', 'apollo', '--report-date', '2020-06-12', '--context', 'sme:apple', ] ), ( ApolloConfig( id='2', command_class=ApolloClass( apple='--class cls1', spotify='--class cls2', ), command_opts='', command_jar='s3://app.jar', url='dbx_url', token='dbx_token', ), DBXJobParams( dsp='apollo', report_date='2020-06-12', context='spotify', ), [ '--class', 'cls2', 's3://app.jar', '--dsp', 'apollo', '--report-date', '2020-06-12', '--context', 'sme:spotify', ] ), ( AmazonConfig( id='3', command_class=AmazonClass(chartmetric_amazon_playlist_meta='--class cls3', ), command_opts='--driver-java-options -Ddelphi.a -Ddelphi.b -Ddelphi.c', command_jar='s3://app.jar', url='dbx_url', token='dbx_token', ), DBXJobParams( dsp='chartmetric', report_date='2020-06-12', context='amazon', ), [ '--class', 'cls3', '--driver-java-options', '-Ddelphi.a -Ddelphi.b -Ddelphi.c', 's3://app.jar', '--dsp', 'chartmetric', '--report-date', '2020-06-12', '--context', 'sme:amazon', ] ) ] ) def test_prepare_command(config: ConfigBase, dbx_job_params: DBXJobParams, expected: List[str]): service = Service(mock.Mock(), {}, config) result = service.prepare_command(dbx_job_params) assert result == expected @pytest.mark.parametrize( 'config, dbx_job_params, expected', [ ( ApolloConfig( id='1', command_class=ApolloClass( apple='--class cls1', spotify='--class cls2', ), command_opts='', command_jar='s3://app.jar', url='dbx_url', token='dbx_token', ), DBXJobParams( dsp='apollo', report_date='2020-06-12', context='apple', ), { 'job_id': '1', 'spark_submit_params': [ '--class', 'cls1', 's3://app.jar', '--dsp', 'apollo', '--report-date', '2020-06-12', '--context', 'sme:apple', ], } ), ( ApolloConfig( id='2', command_class=ApolloClass( apple='--class cls1', spotify='--class cls2', ), command_opts='', command_jar='s3://app.jar', url='dbx_url', token='dbx_token', ), DBXJobParams( dsp='apollo', report_date='2020-06-12', context='spotify', ), { 'job_id': '2', 'spark_submit_params': [ '--class', 'cls2', 's3://app.jar', '--dsp', 'apollo', '--report-date', '2020-06-12', '--context', 'sme:spotify', ], } ), ( AmazonConfig( id='3', command_class=AmazonClass(chartmetric_amazon_playlist_meta='--class cls3'), command_opts='', command_jar='s3://app.jar', url='dbx_url', token='dbx_token', ), DBXJobParams( dsp='chartmetric', report_date='2020-06-12', context='amazon', ), { 'job_id': '3', 'spark_submit_params': [ '--class', 'cls3', 's3://app.jar', '--dsp', 'chartmetric', '--report-date', '2020-06-12', '--context', 'sme:amazon', ], } ) ] ) @httpretty.activate def test_start_dbx_job(config: ConfigBase, dbx_job_params: DBXJobParams, expected: Dict[str, Any]): url = 'http://httpbin.org/post' httpretty.register_uri(httpretty.POST, url, status=200) url = 'https://dbx_url/api/2.0/jobs/run-now' httpretty.register_uri(httpretty.POST, url, status=200) service = Service(mock.Mock(), {}, config) result = service.start_dbx_job(dbx_job_params) assert httpretty.latest_requests()[0].body == json.dumps(expected).encode('utf-8') assert result is True