{# Snowflake sink connector main.tf
   Path on disk: terraform-infra/<env>/kafka-infra/snowflake_sink/<purpose>/main.tf
   (nested under the shared snowflake_sink/ catalog; <purpose> is the snake_case subdir)

   Required template variables:
     purpose                  snake_case subdir name, e.g. "mg_gsr" (directory + backend key only)
     connector_name           hyphenated instance id from the user, e.g. "mg-gsr" (set in variables.tf).
                              service_name is composed in variables.tf as
                              local.service_name = "${var.cluster_name}-${var.connector_type}-${var.connector_name}"
                              = "kc-sfsink-<connector_name>". Reference local.service_name here — never
                              repeat the composed string inline.
     environment              "dev" | "qa" | "prod"
     snowflake_database       e.g. "FACTS"
     snowflake_schema         e.g. "PROD"
     snowflake_role           CONDITIONALLY REQUIRED — e.g. "FACTS_DB_PROD_SCHEMA_READWRITE".
                              REQUIRED when snowflake_ingestion_method is SNOWPIPE_STREAMING
                              (the default); SKIP for plain SNOWPIPE (not consumed there).
                              Emitted only when provided.
     snowflake_ingestion_method OPTIONAL — default "SNOWPIPE_STREAMING" (image default is plain
                              "SNOWPIPE"). Determines whether snowflake_role is required
                              (streaming ⇒ required; SNOWPIPE ⇒ skipped).
     snowflake_user           e.g. "PROD_KAFKA_CONNECT"
     snowflake_host           OPTIONAL — default "sme-delphi.snowflakecomputing.com:443".
                              Override for connectors targeting a different Snowflake account
                              (existing instances also use orchard / delphi hosts).
     topics                   list[str] — Kafka topics to sink
     topic_table_map          dict[topic -> SNOWFLAKE_TABLE_NAME], built by the skill:
                              for each topic the table name is DERIVED by default
                              (dot -> '__', camelCase -> snake_case, UPPERCASE), e.g.
                              cdc.musicGraphV5.acrid -> CDC__MUSIC_GRAPH_V5__ACRID.
                              The user only supplies entries for topics that must target a
                              pre-existing table whose name differs from the derived form
                              (topic_table_overrides). The skill resolves the final map
                              (derived ∪ overrides) before rendering this template.
     dlq_topic_name           e.g. "dlq.snowflakeSink.payments"
     team_name                user-provided (no plugin default; ask the user). Validated at plan
                              time by terraform-default-tags 2.0.0 against the live Datadog
                              team-handle list.
     task_cpu                 default 2048
     task_memory              default 4096

   Module versions:
     terraform_fargate_ref = "6.1.1"
     terraform_default_tags_ref = "2.0.0"
     terraform_datadog_kafka_connector_ref = "6.13.4"
     terraform_vpc_info_ref = "3.1.0"
     terraform_secrets_manager_ref = "1.5.1"
     terraform_required_version = "1.11.4"

   Pair this file with:
     - variables.tf rendered from snowflake_sink_variables.tf.j2
     - versions.tf rendered from versions.tf.j2 (terraform_required_version 1.11.4)
#}
module "default_tags" {
  source             = "git@github.com:theorchard/terraform-default-tags.git//?ref={{ terraform_default_tags_ref | default('2.0.0') }}"
  environment        = var.environment
  application_family = var.application_family
  service_name       = local.service_name
  team_name          = var.team_name
}

provider "aws" {
  region = var.aws_region

  default_tags {
    tags = module.default_tags.tags
  }
}

provider "aws" {
  region  = var.aws_region
  alias   = "networking"
  profile = "networking"

  default_tags {
    tags = module.default_tags.tags
  }
}

# Terraform backends cannot contain interpolations
terraform {
  backend "s3" {
    bucket  = "orcd-terraform-state"
    key     = "{{ environment }}/kafka-infra/snowflake_sink/{{ purpose }}/terraform.tfstate"
    region  = "us-east-1"
    encrypt = "true"
  }
}

data "aws_caller_identity" "current" {}

module "vpc_info" {
  source = "git@github.com:theorchard/terraform-vpc-info.git//?ref={{ terraform_vpc_info_ref | default('3.1.0') }}"

  environment = var.environment
}

data "aws_acm_certificate" "theorchard_io" {
  domain   = "*.theorchard.io"
  statuses = ["ISSUED"]
}

data "aws_route53_zone" "route53_zone" {
  name = "theorchard.io."
}

# Resolve Kafka brokers from the MSK cluster (never a hardcoded broker list)
data "aws_msk_cluster" "kafka_infra" {
  cluster_name = "${var.environment}-${var.msk_cluster_name}"
}

module "fansifter_secrets_manager" {
  source = "git@github.com:theorchard/terraform-secrets-manager.git//?ref={{ terraform_secrets_manager_ref | default('1.5.1') }}"

  for_each = toset(["SNOWFLAKE_PRIVATE_KEY", "SNOWFLAKE_PRIVATE_KEY_PASSPHRASE"])

  environment                    = var.environment
  application_family             = var.application_family
  additional_tags                = var.additional_tags
  service_name                   = local.service_name
  secret_name                    = each.value
  secret_recovery_window_in_days = 7
}

module "snowflake_sink_{{ purpose }}_fargate_environment" {
  source = "git@github.com:theorchard/terraform-fargate.git//?ref={{ terraform_fargate_ref | default('6.1.1') }}"

  providers = {
    aws.dns = aws.networking
  }

  environment        = var.environment
  application_family = var.application_family
  additional_tags    = var.additional_tags
  service_name                      = local.service_name
  non_ecr_image                     = "086679231553.dkr.ecr.${var.aws_region}.amazonaws.com/${local.ecr_image_name}:latest"
  aws_region                        = var.aws_region
  service_platform_version          = "1.4.0"
  desired_task_count                = 1
  minimum_capacity                  = 1
  maximum_capacity                  = 2
  task_cpu                          = {{ task_cpu | default(2048) }}
  task_memory                       = {{ task_memory | default(4096) }}
  container_port                    = 8083
  health_check_grace_period_seconds = 180
  container_start_period_seconds    = 180
  blocking_waf_enabled              = true
  health_check_path                 = "/connectors/${local.service_name}"
  web_service_health_check_command  = "[[ $(curl -s http://localhost:8083/connectors/${local.service_name}/status | jq -r '.tasks[].state,.connector.state' | sort | uniq) == 'RUNNING' ]] || exit 1"
  vpc_id                            = module.vpc_info.vpc_id
  https_listener_certificate_id     = split("/", data.aws_acm_certificate.theorchard_io.arn)[1]
  propagate_tags                    = true

  fargate_service_subnets = module.vpc_info.default_private_subnet_ids
  load_balancer_subnets   = module.vpc_info.default_private_subnet_ids
  datadog_custom_image    = "086679231553.dkr.ecr.${var.aws_region}.amazonaws.com/orchard-datadog-agent-jmx:latest"
  # datadog agent crashes due to OutOfMemoryError with JMX. So increase it from 256
  datadog_task_memory = 512
  datadog_agent_environment_variables = [
    {
      JMX_PORT = "9095"
    },
    {
      DD_ENV = var.environment
    },
    {
      SERVICE_NAME = local.service_name
    },
    {
      CONNECTOR_TYPE = "default_sink"
    },
  ]

  environment_variables = [
    {
      Environment = var.environment
    },
    {
      CONNECT_GROUP_UNIQUE_IDENTIFIER = "${var.environment}-${local.service_name}"
    },
    {
      MAX_TASKS = 1
    },
    {
      BUFFER_COUNT_RECORDS = 10000
    },
    {
      BUFFER_SIZE_BYTES = 5000000
    },
    {
      BUFFER_FLUSH_TIME = 240
    },
    {
      CONNECT_BOOTSTRAP_SERVERS = data.aws_msk_cluster.kafka_infra.bootstrap_brokers_tls
    },
    {
      CONNECTOR_NAME = local.service_name
    },
    {
      SNOWFLAKE_INGESTION_METHOD = "{{ snowflake_ingestion_method | default('SNOWPIPE_STREAMING') }}"
    },
    {
      SNOWFLAKE_HOST = "{{ snowflake_host | default('sme-delphi.snowflakecomputing.com:443') }}"
    },
    {
      SNOWFLAKE_USER = "{{ snowflake_user }}"
    },
    {
      SNOWFLAKE_DATABASE = "{{ snowflake_database }}"
    },
    {
      SNOWFLAKE_SCHEMA = "{{ snowflake_schema }}"
    },
{%- if snowflake_role %}
    {
      SNOWFLAKE_ROLE = "{{ snowflake_role }}"
    },
{%- endif %}
    {
      SERVICE_NAME = local.service_name
    },
    {
      KAFKA_TOPICS = join(",", [
{%- for t in topics %}
        "{{ t }}"{% if not loop.last %},{% endif %}
{%- endfor %}
      ])
    },
    {
      # table names default-derived from the topic (dot -> '__', camelCase -> snake, UPPER);
      # explicit entries here are user overrides for pre-existing tables.
      SNOWFLAKE_TOPIC_TABLE_MAP = join(",", [
{%- for topic, table in topic_table_map.items() %}
        "{{ topic }}:{{ table }}"{% if not loop.last %},{% endif %}
{%- endfor %}
      ])
    },
    {
      DLQ_TOPIC_NAME = "{{ dlq_topic_name }}"
    },
  ]
}

# This rule will allow kafka-connect tasks to communicate with each other in distributed mode
resource "aws_security_group_rule" "allow_intra_cluster_communication" {
  from_port         = 8083
  to_port           = 8083
  protocol          = "tcp"
  security_group_id = module.snowflake_sink_{{ purpose }}_fargate_environment.fargate_security_group_id
  self              = true
  type              = "ingress"
}

module "fargate_service_dashboard" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/kafka_connector?ref={{ terraform_datadog_kafka_connector_ref | default('6.13.4') }}"

  environment        = var.environment
  application_family = var.application_family
  additional_tags    = var.additional_tags
  environment_type   = "fargate"
  service_name       = local.service_name
  connector_type     = "default_sink"

  notification_endpoints            = "{{ notification_endpoints }}"
  escalation_notification_endpoints = "{{ escalation_notification_endpoints }}"
  monitor_kafka_topics_cluster_name = data.aws_msk_cluster.kafka_infra.cluster_name
  monitor_kafka_topics = [
{%- for t in topics %}
    "{{ t }}"{% if not loop.last %},{% endif %}
{%- endfor %}
  ]

  healthy_tasks_monitor_enabled        = true
  service_cpu_monitor_enabled          = true
  error_monitor_enabled                = true
  error_monitor_critical_number_errors = 5
  dlq_topic_name                       = "{{ dlq_topic_name }}"
}