feat: add new kafka_settings attributes for DMS endpoint, and add `…

…cdc_start_position` to DMS replication task (terraform-aws-modules#3)
admirationmr · Dec 22, 2021 · 4b9fe8f · 4b9fe8f
1 parent f6f67ea
commit 4b9fe8f
Show file tree

Hide file tree

Showing 8 changed files with 188 additions and 62 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.56.0
+    rev: v1.62.2
     hooks:
       - id: terraform_fmt
       - id: terraform_validate

diff --git a/README.md b/README.md
@@ -304,13 +304,13 @@ Security scanning results provided by Bridgecrew. Bridgecrew is the leading full
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.55 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.62 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.55 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.62 |
 
 ## Modules
 

diff --git a/examples/complete/README.md b/examples/complete/README.md
@@ -17,6 +17,8 @@ To run this example you need to execute:
 ```bash
 $ terraform init
 $ terraform plan
+# due to https://github.com/hashicorp/terraform/issues/4149
+$ terraform apply --target module.msk_cluster
 $ terraform apply
 ```
 
@@ -28,13 +30,15 @@ Note that this example may create resources which will incur monetary charges on
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.55 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.62 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.55 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.62 |
+| <a name="provider_random"></a> [random](#provider\_random) | >= 3.0 |
 
 ## Modules
 
@@ -43,6 +47,7 @@ Note that this example may create resources which will incur monetary charges on
 | <a name="module_dms_aurora_postgresql_aurora_mysql"></a> [dms\_aurora\_postgresql\_aurora\_mysql](#module\_dms\_aurora\_postgresql\_aurora\_mysql) | ../.. | n/a |
 | <a name="module_dms_default"></a> [dms\_default](#module\_dms\_default) | ../.. | n/a |
 | <a name="module_dms_disabled"></a> [dms\_disabled](#module\_dms\_disabled) | ../.. | n/a |
+| <a name="module_msk_cluster"></a> [msk\_cluster](#module\_msk\_cluster) | clowdhaus/msk-kafka-cluster/aws | ~> 1.0 |
 | <a name="module_rds_aurora"></a> [rds\_aurora](#module\_rds\_aurora) | terraform-aws-modules/rds-aurora/aws | ~> 6.0 |
 | <a name="module_s3_bucket"></a> [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 2.0 |
 | <a name="module_security_group"></a> [security\_group](#module\_security\_group) | terraform-aws-modules/security-group/aws | ~> 4.0 |
@@ -55,9 +60,14 @@ Note that this example may create resources which will incur monetary charges on
 | Name | Type |
 |------|------|
 | [aws_iam_role.s3_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_kms_key.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
 | [aws_rds_cluster_parameter_group.postgresql](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rds_cluster_parameter_group) | resource |
 | [aws_s3_bucket_object.hr_data](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_object) | resource |
+| [aws_secretsmanager_secret.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource |
+| [aws_secretsmanager_secret_policy.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_policy) | resource |
+| [aws_secretsmanager_secret_version.msk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource |
 | [aws_sns_topic.example](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource |
+| [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
 | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |

diff --git a/examples/complete/configs/kafka_mappings.json b/examples/complete/configs/kafka_mappings.json
@@ -0,0 +1,23 @@
+{
+  "rules": [{
+      "rule-type": "selection",
+      "rule-id": "1",
+      "rule-name": "1",
+      "rule-action": "include",
+      "object-locator": {
+        "schema-name": "hr",
+        "table-name": "employee"
+      }
+    },
+    {
+      "rule-type": "object-mapping",
+      "rule-id": "2",
+      "rule-name": "2",
+      "rule-action": "map-record-to-record",
+      "object-locator": {
+        "schema-name": "hr",
+        "table-name": "employee"
+      }
+    }
+  ]
+}
diff --git a/examples/complete/main.tf b/examples/complete/main.tf
@@ -9,11 +9,18 @@ locals {
   db_name     = "example"
   db_username = "example"
 
+  # MSK
+  sasl_scram_credentials = {
+    username = local.name
+    password = "password123!" # do better!
+  }
+
   # aws dms describe-event-categories
   replication_instance_event_categories = ["failure", "creation", "deletion", "maintenance", "failover", "low storage", "configuration change"]
   replication_task_event_categories     = ["failure", "state change", "creation", "deletion", "configuration change"]
 
   bucket_postfix = "${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}"
+  bucket_name    = "${local.name}-s3-${local.bucket_postfix}"
 
   tags = {
     Example     = local.name
@@ -29,6 +36,10 @@ data "aws_caller_identity" "current" {}
 # Supporting Resources
 ################################################################################
 
+resource "random_pet" "this" {
+  length = 2
+}
+
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
   version = "~> 3.0"
@@ -75,7 +86,7 @@ module "vpc_endpoints" {
     dms = {
       service             = "dms"
       private_dns_enabled = true
-      subnet_ids          = module.vpc.database_subnets
+      subnet_ids          = [element(module.vpc.database_subnets, 0), element(module.vpc.database_subnets, 1)] # careful on which AZs support DMS VPC endpoint
       tags                = { Name = "dms-vpc-endpoint" }
     }
     s3 = {
@@ -121,8 +132,8 @@ module "security_group" {
   for_each = {
     postgresql-source    = ["postgresql-tcp"]
     mysql-destination    = ["mysql-tcp"]
-    replication-instance = ["postgresql-tcp", "mysql-tcp"]
-    kafka-destination    = ["kafka-broker-tcp"]
+    replication-instance = ["postgresql-tcp", "mysql-tcp", "kafka-broker-tls-tcp"]
+    kafka-destination    = ["kafka-broker-tls-tcp"]
   }
 
   name        = "${local.name}-${each.key}"
@@ -212,7 +223,7 @@ module "s3_bucket" {
   source  = "terraform-aws-modules/s3-bucket/aws"
   version = "~> 2.0"
 
-  bucket = "${local.name}-s3-${local.bucket_postfix}"
+  bucket = local.bucket_name
 
   attach_deny_insecure_transport_policy = true
 
@@ -285,29 +296,75 @@ resource "aws_iam_role" "s3_role" {
   tags = local.tags
 }
 
-# # TODO - coming soon after additional attributes are added
-# module "msk_cluster" {
-#   source  = "clowdhaus/msk-kafka-cluster/aws"
-#   version = "~> 1.0"
+module "msk_cluster" {
+  source  = "clowdhaus/msk-kafka-cluster/aws"
+  version = "~> 1.0"
+
+  name                   = local.name
+  kafka_version          = "2.8.0"
+  number_of_broker_nodes = 3
 
-#   name                   = local.name
-#   kafka_version          = "2.8.0"
-#   number_of_broker_nodes = 3
+  broker_node_client_subnets  = module.vpc.private_subnets
+  broker_node_ebs_volume_size = 20
+  broker_node_instance_type   = "kafka.t3.small"
+  broker_node_security_groups = [module.security_group["kafka-destination"].security_group_id]
 
-#   broker_node_client_subnets  = module.vpc.private_subnets
-#   broker_node_ebs_volume_size = 20
-#   broker_node_instance_type   = "kafka.t3.small"
-#   broker_node_security_groups = [module.security_group.security_group_id]
+  encryption_in_transit_client_broker = "TLS"
+  encryption_in_transit_in_cluster    = true
 
-#   configuration_name        = "${local.name}-configuration"
-#   configuration_description = "Complete ${local.name} configuration"
-#   configuration_server_properties = {
-#     "auto.create.topics.enable" = true
-#     "delete.topic.enable"       = true
-#   }
+  configuration_name        = "${local.name}-configuration"
+  configuration_description = "Complete ${local.name} configuration"
+  configuration_server_properties = {
+    "auto.create.topics.enable" = true
+    "delete.topic.enable"       = true
+  }
+
+  client_authentication_sasl_scram         = true
+  create_scram_secret_association          = true
+  scram_secret_association_secret_arn_list = [aws_secretsmanager_secret.msk.arn]
+
+  depends_on = [aws_secretsmanager_secret_version.msk]
 
-#   tags = local.tags
-# }
+  tags = local.tags
+}
+
+resource "aws_kms_key" "msk" {
+  description         = "KMS CMK for ${local.name}"
+  enable_key_rotation = true
+
+  tags = local.tags
+}
+
+resource "aws_secretsmanager_secret" "msk" {
+  name        = "AmazonMSK_${local.name}_${random_pet.this.id}"
+  description = "Secret for ${local.name}"
+  kms_key_id  = aws_kms_key.msk.key_id
+
+  tags = local.tags
+}
+
+resource "aws_secretsmanager_secret_version" "msk" {
+  secret_id     = aws_secretsmanager_secret.msk.id
+  secret_string = jsonencode(local.sasl_scram_credentials)
+}
+
+resource "aws_secretsmanager_secret_policy" "msk" {
+  secret_arn = aws_secretsmanager_secret.msk.arn
+  policy     = <<-POLICY
+  {
+    "Version" : "2012-10-17",
+    "Statement" : [ {
+      "Sid": "AWSKafkaResourcePolicy",
+      "Effect" : "Allow",
+      "Principal" : {
+        "Service" : "kafka.amazonaws.com"
+      },
+      "Action" : "secretsmanager:getSecretValue",
+      "Resource" : "${aws_secretsmanager_secret.msk.arn}"
+    } ]
+  }
+  POLICY
+}
 
 ################################################################################
 # DMS Module
@@ -370,11 +427,11 @@ module "dms_aurora_postgresql_aurora_mysql" {
 
       s3_settings = {
         bucket_folder             = "sourcedata"
-        bucket_name               = module.s3_bucket.s3_bucket_id
+        bucket_name               = local.bucket_name # to avoid https://github.com/hashicorp/terraform/issues/4149
         data_format               = "csv"
         encryption_mode           = "SSE_S3"
         external_table_definition = file("configs/s3_table_definition.json")
-        service_access_role_arn   = aws_iam_role.s3_role.arn
+        service_access_role_arn   = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${local.name}-s3" # to avoid https://github.com/hashicorp/terraform/issues/4149
       }
 
       tags = { EndpointType = "s3-source" }
@@ -421,6 +478,27 @@ module "dms_aurora_postgresql_aurora_mysql" {
       ssl_mode                    = "none"
       tags                        = { EndpointType = "mysql-destination" }
     }
+
+    kafka-destination = {
+      endpoint_id   = "${local.name}-kafka-destination"
+      endpoint_type = "target"
+      engine_name   = "kafka"
+      ssl_mode      = "none"
+
+      kafka_settings = {
+        # this https://github.com/hashicorp/terraform/issues/4149 requires the MSK cluster exists before applying
+        broker                  = join(",", module.msk_cluster.bootstrap_brokers)
+        include_control_details = true
+        include_null_and_empty  = true
+        message_format          = "json"
+        sasl_password           = local.sasl_scram_credentials["password"]
+        sasl_username           = local.sasl_scram_credentials["username"]
+        security_protocol       = "sasl-ssl"
+        topic                   = "kafka-destination-topic"
+      }
+
+      tags = { EndpointType = "kakfa-destination" }
+    }
   }
 
   replication_tasks = {
@@ -441,21 +519,15 @@ module "dms_aurora_postgresql_aurora_mysql" {
       target_endpoint_key       = "mysql-destination"
       tags                      = { Task = "PostgreSQL-to-MySQL" }
     }
-
-    # # TODO - coming soon after additional attributes are added
-    # kafka-destination = {
-    #   endpoint_id   = "${local.name}-kafka-destination"
-    #   endpoint_type = "target"
-    #   engine_name   = "kafka"
-    #   ssl_mode      = "none"
-
-    #   kafka_settings = {
-    #     broker = module.msk_cluster.bootstrap_brokers
-    #     topic  = local.name
-    #   }
-
-    #   tags = { EndpointType = "kafka-destination" }
-    # }
+    postgresql_kafka = {
+      replication_task_id       = "${local.name}-postgresql-to-kafka"
+      migration_type            = "full-load-and-cdc"
+      replication_task_settings = file("configs/task_settings.json")
+      table_mappings            = file("configs/kafka_mappings.json")
+      source_endpoint_key       = "postgresql-source"
+      target_endpoint_key       = "kafka-destination"
+      tags                      = { Task = "PostgreSQL-to-Kafka" }
+    }
   }
 
   event_subscriptions = {

diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf
@@ -4,7 +4,11 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.55"
+      version = ">= 3.62"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 3.0"
     }
   }
 }