# OS creds
module "connector_secrets_gsr" {
  source = "git@github.com:theorchard/terraform-secrets-manager.git//?ref=1.6.1"

  environment        = var.environment
  application_family = var.application_family
  additional_tags    = var.additional_tags
  service_name       = local.gsr_service_name
  secret_name        = "OPENSEARCH_CREDENTIALS"
}

# Fargate service
module "os_sink_fargate_environment_gsr" {
  source = "git@github.com:theorchard/terraform-fargate.git//?ref=6.5.0"

  providers = {
    aws.dns = aws.networking
  }

  environment              = var.environment
  application_family       = var.application_family
  additional_tags          = var.additional_tags
  service_name             = local.gsr_service_name
  non_ecr_image            = "086679231553.dkr.ecr.${var.aws_region}.amazonaws.com/${local.ecr_repo_name}:latest"
  aws_region               = var.aws_region
  service_platform_version = "1.4.0"
  # Using a larger container with 2048 cpu + 4096 memory instead of 1024 cpu + 3072 memory like ES instances. Thus keeping min task count to 2 instead of 3 for ES.
  desired_task_count                = 2
  minimum_capacity                  = 2
  maximum_capacity                  = 3
  task_cpu                          = 2048
  task_memory                       = 4096
  autoscaling_cpu_policy_enabled    = false
  autoscaling_memory_policy_enabled = false
  container_port                    = 8083
  health_check_grace_period_seconds = 420
  container_start_period_seconds    = 120
  blocking_waf_enabled              = true
  health_check_path                 = "/connectors/${var.environment}-${local.gsr_service_name}/status"
  web_service_health_check_command  = "[[ $(curl -s http://localhost:8083/connectors/${var.environment}-${local.gsr_service_name}/status | jq -r '.tasks[].state' | sort | uniq) == 'RUNNING' ]] || exit 1"
  vpc_id                            = module.vpc_info.vpc_id
  fargate_service_subnets           = module.vpc_info.default_private_subnet_ids
  load_balancer_subnets             = module.vpc_info.default_private_subnet_ids
  propagate_tags                    = true

  iam_managed_policy_attachments = [
    data.aws_iam_policy.opensearch_default_rw_policy.arn
  ]
  datadog_custom_image = "086679231553.dkr.ecr.${var.aws_region}.amazonaws.com/orchard-datadog-agent-jmx:latest"
  # datadog agent crashes due to OutOfMemoryError with JMX. So increase it from 256
  datadog_task_memory = 512
  datadog_agent_environment_variables = [
    {
      JMX_PORT = "9095"
    },
    {
      DD_ENV = var.environment
    },
    {
      SERVICE_NAME = local.gsr_service_name
    },
    {
      CONNECTOR_TYPE = "default_sink"
    },
  ]
  environment_variables = [
    {
      Environment = var.environment
    },
    {
      CONNECT_BOOTSTRAP_SERVERS = var.connect_bootstrap_servers
    },
    {
      CONNECTOR_NAME = "${var.environment}-${local.gsr_service_name}"
    },
    {
      CONNECT_GROUP_UNIQUE_IDENTIFIER = "${var.environment}-${local.gsr_service_name}"
    },
    {
      MAX_TASKS = "3" # It was 3 for ES. It is busy connector with avg 42.0 requests per second.
    },
    {
      OPENSEARCH_HOST = "https://${data.aws_opensearch_domain.music_graph_domain.endpoint}"
    },
    {
      OPENSEARCH_DOC_TYPE = "_doc"
    },
    {
      OPENSEARCH_KEY_IGNORE = "false"
    },
    {
      OPENSEARCH_SCHEMA_IGNORE = "true"
    },
    {
      KAFKA_TOPICS = join(",", [
        "stream.globalSoundRecording.elasticSearch.cdc.partitioned.V5", # merged from all 3 streams.
      ])
    },
    {
      SECRETS_MANAGER_OPENSEARCH_SECRET_NAME = "OPENSEARCH_CREDENTIALS"
    },
    {
      SERVICE_NAME = local.gsr_service_name
    },
    {
      TRANSFORMS_RENAMETOPIC_REPLACEMENT = "global_sound_recording_write"
    },
    {
      BATCH_SIZE = "800"
    },
    {
      MAX_INFLIGHT_REQUESTS = "6"
    },
    {
      MAX_BUFFERED_RECORDS = "4800"  # BASED ON BATCH_SIZE * MAX_INFLIGHT_REQUESTS per tasks.
    },
    {
      FLUSH_TIMEOUT_MS = "90000"
    },
    {
      READ_TIMEOUT_MS = "60000"
    },

    {
      TRANSFORMS_RENAMETOPIC_REGEX = ".*"
    },
    {
      TRANSFORMS_RENAMETOPIC_TYPE = "org.apache.kafka.connect.transforms.RegexRouter"
    },
    {
      TRANSFORMS = "renameTopic"
    },
    {
      WRITE_METHOD = "upsert"
    },
    {
      BEHAVIOR_ON_VERSION_CONFLICT = "warn"
    },
    {
      DLQ_TOPIC_NAME = local.gsr_dlq_topic_name
    },
    {
      DLQ_HEADERS_ENABLE = "true"
    },
    {
      DEBUG_MODE = "false"
    },
  ]
}

# This rule will allow kafka-connect tasks to communicate with each other in distributed mode
resource "aws_security_group_rule" "allow_intra_cluster_communication_gsr" {
  from_port         = 8083
  to_port           = 8083
  protocol          = "tcp"
  security_group_id = module.os_sink_fargate_environment_gsr.fargate_security_group_id
  self              = true
  type              = "ingress"
}

# Datadog dashboard.
module "fargate_service_dashboard_gsr" {
  source = "git@github.com:theorchard/terraform-datadog.git//modules/kafka_connector?ref=6.15.2"

  environment        = var.environment
  environment_type   = "fargate"
  service_name       = local.gsr_service_name
  application_family = var.application_family
  teams              = [var.application_family]
  additional_tags    = var.additional_tags
  connector_type     = "default_sink"

  monitor_kafka_topics_cluster_name = var.msk_cluster_name
  notification_endpoints            = var.notification_endpoints
  escalation_notification_endpoints = var.escalation_notification_endpoints
  monitor_kafka_topics = [
    "stream.globalSoundRecording.elasticSearch.cdc.partitioned.V5",
  ]

  healthy_tasks_monitor_enabled        = true
  service_cpu_monitor_enabled          = true
  error_monitor_enabled                = true
  dlq_topic_name                       = local.gsr_dlq_topic_name
  error_monitor_critical_number_errors = 5
  error_monitor_extra_query     = "-\\\"org.apache.http.ConnectionClosedException\\\" -\\\"Failed to send bulk request from batch\\\""
}