"""Salesforce consumer tests.""" from unittest.mock import MagicMock from unittest.mock import patch from datetime import datetime import pytest from dateutil.tz import tzutc from simple_salesforce import SalesforceError from simple_salesforce import format_soql from src.salesforce_consumer import SalesforceConsumer import config from src import constants @pytest.mark.parametrize(( 'test_lastModified', 'test_lead', 'expected_offset_data' ), [ ( # legacy lastModified date format {'last_lastmodifieddate': '2023-11-29 19:00:40.449483'}, [ { 'FirstName': 'a', 'LastName': 'b', 'Id': '00Idstringabc', 'LastModifiedDate': '2023-11-29T20:30:42.605+0000' } ], { 'last_lastmodifieddate': '2023-11-29T20:30:42.605+0000' }, ), ( # updated lastModified date format salesforce standard {'last_lastmodifieddate': '2023-11-29T19:29:51.000+0000'}, [ { 'FirstName': 'a', 'LastName': 'b', 'Id': '00Idstring123', 'LastModifiedDate': '2023-11-29T20:40:42.605+0000' } ], { 'last_lastmodifieddate': '2023-11-29T20:40:42.605+0000' } ) ]) @patch('src.salesforce_consumer.TopicPartition') @patch('src.salesforce_consumer.DeserializingConsumer') @patch('src.salesforce_consumer.SerializingProducer') @patch('src.salesforce_consumer.AdminClient') @patch('src.salesforce_consumer.Salesforce') @patch('src.salesforce_consumer.requests') @patch('src.salesforce_consumer.config.secrets_manager_client') def test_successfull_process( sm_mock, requests_mock, sf_mock, admin_mock, producer_mock, consumer_mock, tp_mock, test_lastModified, test_lead, expected_offset_data): """Test successful Salesforce polling and sending to Kafka.""" test_kafka_topic = 'test' poll_mock = MagicMock() poll_mock.error.return_value = False poll_mock.value.return_value = test_lastModified consumer_mock_obj = MagicMock() consumer_mock_obj.get_watermark_offsets.return_value = (0, 10) consumer_mock_obj.poll.return_value = poll_mock consumer_mock.return_value = consumer_mock_obj sf_mock_object = MagicMock() sf_mock_object.query_all.return_value = { 'totalSize': 1, 'records': test_lead, 'done': True # optimistically } sf_mock.return_value = sf_mock_object offset_producer_mock_obj = MagicMock() producer_mock_obj = MagicMock() poller = SalesforceConsumer(kafka_topic=test_kafka_topic) # use class helper functions to build expected mock test soql soql_var = poller._format_string_as_dateobj(test_lastModified.get('last_lastmodifieddate')) # cp from ../unqualified_leads.soql raw_soql = "SELECT Id, Status, FirstName, LastName, Email, createddate, LastModifiedDate, LeadScore__c, Country, Company, Lead.Owner.FirstName, Lead.Owner.LastName, Lead.Owner.Email, VoucherCode__c, Consultant__c FROM Lead WHERE RecordTypeId = '0124W0000007nnYQAQ' AND Status = 'Unqualified' AND LastModifiedDate > {} ORDER BY LastModifiedDate DESC" # noqa expected_soql = format_soql(raw_soql, soql_var) poller._offset_producer = offset_producer_mock_obj poller._producer = producer_mock_obj poller.process_leads() sf_mock_object.query_all.assert_called_with(expected_soql) consumer_mock_obj.poll.assert_called() producer_mock_obj.produce.assert_called_with( test_kafka_topic, value=test_lead[0], key=test_lead[0].get('Id'), on_delivery=poller._delivery_report ) # assert correct offset_data made it to offset poller._offset_producer.produce.assert_called_with( 'event.gdaUnqualifiedLeads.offsets', value=expected_offset_data, key=constants.LASTMODIFIEDDATE_FIELD ) @patch('src.salesforce_consumer.TopicPartition') @patch('src.salesforce_consumer.DeserializingConsumer') @patch('src.salesforce_consumer.SerializingProducer') @patch('src.salesforce_consumer.AdminClient') @patch('src.salesforce_consumer.Salesforce') @patch('src.salesforce_consumer.requests') @patch('src.salesforce_consumer.config.secrets_manager_client') def test_retry_attempt_on_sf_exception( sm_mock, requests_mock, sf_mock, admin_mock, producer_mock, consumer_mock, tp_mock): """Test SalesForce polling and sending to Kafka with retry attempts.""" test_kafka_topic = 'test' poll_mock = MagicMock() poll_mock.error.return_value = False poll_mock.value.return_value = { 'last_lastmodifieddate': '2023-11-29T19:29:51.000+0000'} consumer_mock_obj = MagicMock() consumer_mock_obj.get_watermark_offsets.return_value = (0, 10) consumer_mock_obj.poll.return_value = poll_mock consumer_mock.return_value = consumer_mock_obj sf_mock_object = MagicMock() sf_mock_object.query_all.side_effect = SalesforceError( 'url', 'status', 'resource_name', 'content') sf_mock.return_value = sf_mock_object offset_producer_mock_obj = MagicMock() producer_mock_obj = MagicMock() poller = SalesforceConsumer(kafka_topic=test_kafka_topic) poller._offset_producer = offset_producer_mock_obj poller._producer = producer_mock_obj with pytest.raises(SalesforceError): poller.process_leads() assert sf_mock_object.query_all.call_count == config.RETRY_COUNT @pytest.mark.parametrize(('datestr', 'expected'), [ ( # salesforce standard '2023-12-04T17:58:23.000+0000', datetime(2023, 12, 4, 17, 58, 23, tzinfo=tzutc()) ), ( # ISO 8601 '2023-12-01T20:00:54.588Z', datetime(2023, 12, 1, 20, 0, 54, 588000, tzinfo=tzutc()) ), ( # with spaces instead of T time and Z UTC timezone indicators '2023-11-29 18:00:40.876692', datetime(2023, 11, 29, 18, 0, 40, 876692) ) ]) @patch('src.salesforce_consumer.SerializingProducer') @patch('src.salesforce_consumer.Salesforce') @patch('src.salesforce_consumer.requests') @patch('src.salesforce_consumer.config.secrets_manager_client') def test_format_datetimes( sp_mock, sf_mock, requests_mock, sm_mock, datestr, expected ): """Test date formatter with different format inputs.""" test_kafka_topic = 'test' poller = SalesforceConsumer(kafka_topic=test_kafka_topic) datetime = poller._format_string_as_dateobj(datestr) assert datetime == expected # documentation tests for expected exception cases @patch('src.salesforce_consumer.Salesforce') @patch('src.salesforce_consumer.config.secrets_manager_client') @patch('src.salesforce_consumer.requests') @patch('src.salesforce_consumer.SerializingProducer') @pytest.mark.xfail(raises=SalesforceError) def test_salesforce_error(sf_mock, sm_mock, requests_mock, producer_mock): """Test load_leads fails if Salesforce dependency bugs.""" test_kafka_topic = 'test' mock_last_lastmodifieddate = datetime( 2023, 12, 4, 17, 58, 23, tzinfo=tzutc() ) sf_mock.query_all.side_effect = SalesforceError( 'url', 'status', 'resource_name', 'content') poller = SalesforceConsumer(kafka_topic=test_kafka_topic) poller._sf = sf_mock poller._load_leads(mock_last_lastmodifieddate) @patch('src.salesforce_consumer.SerializingProducer') @patch('src.salesforce_consumer.Salesforce') @patch('src.salesforce_consumer.requests') @patch('src.salesforce_consumer.config.secrets_manager_client') @pytest.mark.xfail(raises=ValueError) def test_datetime_change_error( sp_mock, sf_mock, requests_mock, sm_mock ): """Test catch datetime format changes.""" datestr = 'nonsense' test_kafka_topic = 'test' poller = SalesforceConsumer(kafka_topic=test_kafka_topic) poller._format_string_as_dateobj(datestr)