"""Lambda ioc sanitize.""" import logging import os import re import ipaddress import boto3 import sentry_sdk from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration from config import ( AWS_DNS_FIREWALL_OUTPUT_PATH, ENVIRONMENT, GSIRT_S3_DROP_BUCKET, GSIRT_S3_DROP_BUCKET_ACCOUNT_ID, GSIRT_S3_DROP_DOMAIN_FILE, GSIRT_S3_DROP_FOLDER, GSIRT_S3_DROP_IP_FILE, GSIRT_S3_SANITIZED_FOLDER, IPv4_OUTPUT_PATH, PALO_DOMAIN_OUTPUT_PATH, PALO_URL_OUTPUT_PATH, SENTRY_DSN, ) if SENTRY_DSN: sentry_sdk.init( dsn=SENTRY_DSN, environment=ENVIRONMENT, integrations=[AwsLambdaIntegration(timeout_warning=True)], ) logger = logging.getLogger() logger.setLevel(logging.INFO) s3_client = boto3.client('s3') def remove_ip_with_from_domains(lines): """ Remove lines containing IP addresses with or without CIDR notation. Args: lines (list): List of lines from the input file. Returns: list: List of lines without IP addresses with or without CIDR notation. """ def is_ip(line): try: ipaddress.ip_network(line.strip(), strict=False) return True except ValueError: return False return [line for line in lines if not is_ip(line)] def remove_invalid_domains(lines): """ Remove lines that do not match the valid domain pattern. Args: lines (list): List of lines from the input file. Returns: list: List of lines with valid domain names. """ valid_domain_regex = re.compile(r'^[a-zA-Z0-9\-\_\*\.]+$') return [line for line in lines if valid_domain_regex.match(line.strip())] def remove_duplicate_lines(lines): """ Remove duplicate lines and convert them to lowercase. Args: lines (list): List of lines from the input file. Returns: list: List of unique lines in lowercase. """ return list(set(line.strip().lower() for line in lines)) def remove_invalid_ip_addresses_from_file(lines): """ Remove invalid IP addresses from a list of lines. Args: lines (list): List of lines containing IP addresses. Returns: list: List of valid IP addresses. """ valid_addresses = [] for address in lines: try: ipaddress.ip_network(address.strip()) valid_addresses.append(address) except ValueError: logger.info(f'Invalid IP address: {address}') valid_addresses.sort() return valid_addresses def create_palo_wildcard_domain_file(lines): """ Create a file for palo alto with wildcard domain entries. Args: lines (list): List of sanitized lines. Returns: str: Path to the created wildcard domain file. """ output_path = os.path.join('/tmp', PALO_DOMAIN_OUTPUT_PATH) with open(output_path, 'w') as file: for line in lines: stripped_line = line.strip() file.write(f'*.{stripped_line}\n') return output_path def create_palo_wildcard_url_file(lines): """ Create a file for palo alto with wildcard URL entries. Args: lines (list): List of sanitized lines. Returns: str: Path to the created wildcard URL file. """ output_path = os.path.join('/tmp', PALO_URL_OUTPUT_PATH) with open(output_path, 'w') as file: for line in lines: stripped_line = line.strip() file.write(f'*.{stripped_line}/*\n') return output_path def create_ipv4_file(lines): """ Create a file for Palo Alto and AWS WAF IP sets with IP addresses. in CIDR /32 format. Args: lines (list): List of sanitized lines containing IP addresses. Returns: str: Path to the created IP file. """ output_path = os.path.join('/tmp', IPv4_OUTPUT_PATH) with open(output_path, 'w') as file: for line in lines: stripped_line = line.strip() if '/' not in stripped_line: stripped_line += '/32' file.write(f'{stripped_line}\n') return output_path def dns_firewall_duplicate_with_asterisk_dot(lines): """ Duplicate lines with an asterisk dot prefix. Route53 DNS Firewall requires domain entries to be duplicated with an asterisk dot prefix. Args: lines (list): List of sanitized lines. Returns: str: Path to the created DNS firewall file. """ output_path = os.path.join('/tmp', AWS_DNS_FIREWALL_OUTPUT_PATH) with open(output_path, 'w') as file: for line in lines: stripped_line = line.strip() file.write(f'{stripped_line}\n') file.write(f'*.{stripped_line}\n') return output_path def sanitize(lines): """ Sanitize the input lines by removing IP addresses with CIDR. Notation, invalid domains, and duplicate lines. Args: lines (list): List of lines from the input file. Returns: list: List of sanitized lines. """ lines = remove_ip_with_from_domains(lines) lines = remove_invalid_domains(lines) lines = remove_duplicate_lines(lines) lines.sort() return lines def handler(event, context): """ AWS Lambda handler function to sanitize and process domain block files. Args: event (dict): Event data passed to the Lambda function. context (object): Runtime information of the Lambda function. Returns: dict: Status of the Lambda function execution. """ try: # Define temporary file paths domain_file_path = os.path.join('/tmp', GSIRT_S3_DROP_DOMAIN_FILE) ip_file_path = os.path.join('/tmp', GSIRT_S3_DROP_IP_FILE) # Download domain file to /tmp s3_client.download_file( GSIRT_S3_DROP_BUCKET, f'{GSIRT_S3_DROP_FOLDER}/{GSIRT_S3_DROP_DOMAIN_FILE}', domain_file_path, ExtraArgs={'ExpectedBucketOwner': GSIRT_S3_DROP_BUCKET_ACCOUNT_ID}, ) # Download IP file to /tmp s3_client.download_file( GSIRT_S3_DROP_BUCKET, f'{GSIRT_S3_DROP_FOLDER}/{GSIRT_S3_DROP_IP_FILE}', ip_file_path, ExtraArgs={'ExpectedBucketOwner': GSIRT_S3_DROP_BUCKET_ACCOUNT_ID}, ) # Read files from /tmp with open(domain_file_path, 'r') as domain_file: domain_lines = domain_file.readlines() with open(ip_file_path, 'r') as ip_file: ip_lines = ip_file.readlines() # Sanitize lines sanitized_domain_lines = sanitize(domain_lines) sanitized_ip_lines = remove_invalid_ip_addresses_from_file(ip_lines) logger.info(f'Sanitized domain lines: {sanitized_domain_lines}') logger.info(f'Sanitized IP lines: {sanitized_ip_lines}') # Check if the input file is empty if not sanitized_domain_lines or not sanitized_ip_lines: logger.error( 'No valid domains or IPs were found in the uploaded file. ' 'Please check with GISRT or verify the input file in S3, ' 'then try again.' ) raise ValueError( 'No valid domains or IPs were found in the uploaded file. ' 'Please check your input and try again.' ) # Process and create output files in /tmp dns_firewall_path = dns_firewall_duplicate_with_asterisk_dot( sanitized_domain_lines ) palo_domain_path = create_palo_wildcard_domain_file(sanitized_domain_lines) palo_url_path = create_palo_wildcard_url_file(sanitized_domain_lines) ipv4_path = create_ipv4_file(sanitized_ip_lines) # Upload files from /tmp to S3 s3_client.upload_file( dns_firewall_path, GSIRT_S3_DROP_BUCKET, f'{GSIRT_S3_SANITIZED_FOLDER}/sanitized_ioc_dns_firewall_blocks.txt', ExtraArgs={'ExpectedBucketOwner': GSIRT_S3_DROP_BUCKET_ACCOUNT_ID}, ) s3_client.upload_file( palo_domain_path, GSIRT_S3_DROP_BUCKET, f'{GSIRT_S3_SANITIZED_FOLDER}/palo_domain_blocks.txt', ExtraArgs={'ExpectedBucketOwner': GSIRT_S3_DROP_BUCKET_ACCOUNT_ID}, ) s3_client.upload_file( palo_url_path, GSIRT_S3_DROP_BUCKET, f'{GSIRT_S3_SANITIZED_FOLDER}/palo_url_blocks.txt', ExtraArgs={'ExpectedBucketOwner': GSIRT_S3_DROP_BUCKET_ACCOUNT_ID}, ) s3_client.upload_file( ipv4_path, GSIRT_S3_DROP_BUCKET, f'{GSIRT_S3_SANITIZED_FOLDER}/ip_blocks.txt', ExtraArgs={'ExpectedBucketOwner': GSIRT_S3_DROP_BUCKET_ACCOUNT_ID}, ) logger.info('Files successfully uploaded to S3.') return {'status': 'OK'} except Exception as e: logger.exception(str(e)) raise e