Upgrade kafka connect

zsmartex · Dec 9, 2023 · 6a95f28 · 6a95f28
1 parent 5297ef0
commit 6a95f28
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/templates/terraform/kafka-connect/main.tf.erb b/templates/terraform/kafka-connect/main.tf.erb
@@ -11,6 +11,8 @@ resource "kafka-connect_connector" "pg-peatio-connector" {
     "table.include.list"                 = "public.trades,public.orders,public.deposits,public.withdraws,public.beneficiaries,public.commissions,public.invite_links,public.asset_statistics,public.pnl_statistics,public.operations_assets,public.operations_expenses,public.operations_liabilities,public.operations_revenues"
     "time.precision.mode"                = "connect"
     "topic.prefix"                       = "pg"
+    "poll.interval.ms"                   = "10"
+    "max.batch.size"                     = "1024"
     "transforms"                         = "dropPrefix, filter"
     "transforms.dropPrefix.regex"        = "pg.public.(.*)"
     "transforms.dropPrefix.replacement"  = "pg.$1"
@@ -44,6 +46,7 @@ resource "kafka-connect_connector" "pg-barong-connector" {
     "table.include.list"                         = "public.activities,public.api_keys,public.profiles,public.attachments,public.users"
     "time.precision.mode"                        = "connect"
     "topic.prefix"                               = "pg"
+    "poll.interval.ms"                           = "10"
     "transforms"                                 = "dropPrefix, filter"
     "transforms.dropPrefix.regex"                = "pg.public.(.*)"
     "transforms.dropPrefix.replacement"          = "pg.$1"
@@ -78,6 +81,8 @@ resource "kafka-connect_connector" "es-orders-sink" {
     "key.converter.schemas.enable"                         = "true"
     "key.ignore"                                           = "false"
     "schema.ignore"                                        = "true"
+    "read.timeout.ms"                                      = "30000"
+    "batch.size"                                           = "100"
     "tasks.max"                                            = "5"
     "topics"                                               = "pg.orders"
     "transforms"                                           = "unwrap,extractKey,triggeredAtTimestampConverter,createdAtTimestampConverter,updatedAtTimestampConverter"
@@ -116,6 +121,8 @@ resource "kafka-connect_connector" "es-normal-sink" {
     "key.converter.schemas.enable"                       = "true"
     "key.ignore"                                         = "false"
     "schema.ignore"                                      = "true"
+    "read.timeout.ms"                                    = "30000"
+    "batch.size"                                         = "100"
     "tasks.max"                                          = "3"
     "topics"                                             = "pg.trades,pg.deposit_addresses,pg.transactions,pg.beneficiaries,pg.users,pg.activities,pg.api_keys,pg.profiles,pg.attachments,pg.operations_assets,pg.operations_expenses,pg.operations_liabilities,pg.operations_revenues,,pg.commissions,pg.invite_links"
     "transforms"                                         = "unwrap,extractKey,createdAtTimestampConverter,updatedAtTimestampConverter"
@@ -150,6 +157,8 @@ resource "kafka-connect_connector" "es-deposits-withdraws-sink" {
     "key.converter.schemas.enable"                         = "true"
     "key.ignore"                                           = "false"
     "schema.ignore"                                        = "true"
+    "read.timeout.ms"                                      = "30000"
+    "batch.size"                                           = "100"
     "tasks.max"                                            = "3"
     "topics"                                               = "pg.deposits,pg.withdraws"
     "transforms"                                           = "unwrap,extractKey,createdAtTimestampConverter,updatedAtTimestampConverter,completedAtTimestampConverter"
@@ -188,6 +197,8 @@ resource "kafka-connect_connector" "es-asset_statistics-pnl_statistics-sink" {
     "key.converter.schemas.enable"                        = "true"
     "key.ignore"                                          = "false"
     "schema.ignore"                                       = "true"
+    "read.timeout.ms"                                     = "30000"
+    "batch.size"                                          = "100"
     "tasks.max"                                           = "3"
     "topics"                                              = "pg.asset_statistics,pg.pnl_statistics"
     "transforms"                                          = "unwrap,extractKey,createdAtTimestampConverter,updatedAtTimestampConverter,recordedAtTimestampConverter"