Refactor connector configurations to adapt parseable cli options.

Cargo.toml

@@ -46,7 +46,7 @@ bzip2 = { version = "*", features = ["static"] }
 cookie = "0.18.1"
 chrono = "0.4"
 chrono-humanize = "0.2"
-clap = { version = "4.1", default-features = false, features = [
+clap = { version = "4.5", default-features = false, features = [
     "std",
     "color",
     "help",

docker-compose-distributed-test.yaml

@@ -65,7 +65,7 @@ services:
       context: .
       dockerfile: Dockerfile
     platform: linux/amd64
-    command: [ "parseable", "s3-store" ]
+    command: [ "parseable", "s3-store", "connectors", "kafka-sink" ]
     ports:
       - 8000
     environment:
@@ -81,9 +81,8 @@ services:
       - P_PARQUET_COMPRESSION_ALGO=snappy
       - P_MODE=ingest
       - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
-      - P_KAFKA_TOPICS=dist-test-logs-stream
+      - P_KAFKA_CONSUMER_TOPICS=dist-test-logs-stream
       - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
-      - P_KAFKA_GROUP_ID=parseable-kafka-sink-connector
       # additional settings like security, tuning, etc.
     networks:
       - parseable-internal

docker-compose-test.yaml

@@ -29,7 +29,7 @@ services:
       context: .
       dockerfile: Dockerfile
     platform: linux/amd64
-    command: [ "parseable", "s3-store" ]
+    command: [ "parseable", "s3-store", "connectors", "kafka-sink" ]
     ports:
       - "8000:8000"
     environment:
@@ -43,9 +43,8 @@ services:
       - P_PASSWORD=parseableadmin
       - P_CHECK_UPDATE=false
       - P_PARQUET_COMPRESSION_ALGO=snappy
-      - P_KAFKA_TOPICS=test-logs-stream
+      - P_KAFKA_CONSUMER_TOPICS=test-logs-stream
       - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
-      - P_KAFKA_GROUP_ID=parseable-kafka-sink-connector
       # additional settings like security, tuning, etc.
     depends_on:
       - minio
@@ -116,7 +115,7 @@ services:
     networks:
       - parseable-internal
     healthcheck:
-      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1"]
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -146,7 +145,7 @@ services:
     networks:
       - parseable-internal
     healthcheck:
-      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1"]
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -176,7 +175,7 @@ services:
     networks:
       - parseable-internal
     healthcheck:
-      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1"]
+      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
       interval: 10s
       timeout: 5s
       retries: 5

scripts/kafka_log_stream_generator.py

@@ -1,3 +1,18 @@
+# Parseable Server (C) 2022 - 2024 Parseable, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 import os
 import sys
 import time