Clean up Docker compose env vars. (airbytehq#9209)

- sort docker env vars.
- remove all non-docker related env vars.
- add what is missing.

For the .env file:
- sort the file to match the Configs.java lay out for better reading.
- get rid of env vars that are not used in docker
- get rid of env vars that have defaults, with the exception of var that are for scaling e.g. submitter_num_threads, worker related vars to prevent the env file from getting too large
- add a header to clarify when/where to add env vars to the file

For the docker compose file:
- sort the env vars alphabetically
- get rid of env vars that aren't used in that application
- add missing env vars into the worker application

.env

@@ -1,32 +1,22 @@
+# This file only contains Docker relevant variables.
+#
+# Variables with defaults have been omitted to avoid duplication of defaults.
+# The only exception to the non-default rule are env vars related to scaling.
+#
+# See https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java
+# for the latest environment variables.
+#
+# # Contributors - please organise this env file according to the above linked file.
+
+
+### SHARED ###
 VERSION=0.35.2-alpha
 
-# Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
-DATABASE_USER=docker
-DATABASE_PASSWORD=docker
-DATABASE_HOST=db
-DATABASE_PORT=5432
-DATABASE_DB=airbyte
-# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} (do not include the username or password here)
-DATABASE_URL=jdbc:postgresql://db:5432/airbyte
-JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001
-
-# Airbyte Internal Config Database, default to reuse the Job Database when they are empty
-# Usually you do not need to set them; they are explicitly left empty to mute docker compose warnings
-CONFIG_DATABASE_USER=
-CONFIG_DATABASE_PASSWORD=
-CONFIG_DATABASE_URL=
-CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.1.001
-
-RUN_DATABASE_MIGRATION_ON_STARTUP=true
-
-# When using the airbyte-db via default docker image:
+# When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
 DATA_DOCKER_MOUNT=airbyte_data
 DB_DOCKER_MOUNT=airbyte_db
 
-# Temporal.io worker configuration
-TEMPORAL_HOST=airbyte-temporal:7233
-
 # Workspace storage for running jobs (logs, etc)
 WORKSPACE_ROOT=/tmp/workspace
 WORKSPACE_DOCKER_MOUNT=airbyte_workspace
@@ -42,59 +32,59 @@ LOCAL_DOCKER_MOUNT=/tmp/airbyte_local
 # Issue: https://github.com/airbytehq/airbyte/issues/577
 HACK_LOCAL_ROOT_PARENT=/tmp
 
-# Maximum simultaneous jobs
-SUBMITTER_NUM_THREADS=10
 
-# Job container images
-# Usually you should not need to set these, they have defaults already set
-JOB_KUBE_SOCAT_IMAGE=
-JOB_KUBE_BUSYBOX_IMAGE=
-JOB_KUBE_CURL_IMAGE=
+### DATABASE ###
+# Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+DATABASE_HOST=db
+DATABASE_PORT=5432
+DATABASE_DB=airbyte
+# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} (do not include the username or password here)
+DATABASE_URL=jdbc:postgresql://db:5432/airbyte
+JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001
+
+# Airbyte Internal Config Database, defaults to Job Database if empty. Explicitly left empty to mute docker compose warnings.
+CONFIG_DATABASE_USER=
+CONFIG_DATABASE_PASSWORD=
+CONFIG_DATABASE_URL=
+CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.1.001
 
-# Miscellaneous
-TRACKING_STRATEGY=segment
+
+### AIRBYTE SERVICES ###
+TEMPORAL_HOST=airbyte-temporal:7233
+INTERNAL_API_HOST=airbyte-server:8001
 WEBAPP_URL=http://localhost:8000/
+# Although not present as an env var, required for webapp configuration.
 API_URL=/api/v1/
-INTERNAL_API_HOST=airbyte-server:8001
-LOG_LEVEL=INFO
 
-WORKER_ENVIRONMENT=docker
 
-# Cloud log backups. Don't use this unless you know what you're doing. Mainly for Airbyte devs.
-# If you just want to capture Docker logs, you probably want to use something like this instead:
-# https://docs.docker.com/config/containers/logging/configure/
-S3_LOG_BUCKET=
-S3_LOG_BUCKET_REGION=
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-S3_MINIO_ENDPOINT=
-S3_PATH_STYLE_ACCESS=
-
-GCS_LOG_BUCKET=
-
-# Docker Resource Limits
+### JOBS ###
+# Relevant to scaling.
+SYNC_JOB_MAX_ATTEMPTS=3
+SYNC_JOB_MAX_TIMEOUT_DAYS=3
 JOB_MAIN_CONTAINER_CPU_REQUEST=
 JOB_MAIN_CONTAINER_CPU_LIMIT=
 JOB_MAIN_CONTAINER_MEMORY_REQUEST=
 JOB_MAIN_CONTAINER_MEMORY_LIMIT=
 
-# Max attempts per sync and max retries per attempt
-SYNC_JOB_MAX_ATTEMPTS=3
-
-# Time in days to reach a timeout to cancel the synchronization
-SYNC_JOB_MAX_TIMEOUT_DAYS=3
 
-# Set secret persistence store to use. Do not change this for existing installations!
-SECRET_PERSISTENCE=NONE
-
-# State Cloud Storage
-STATE_STORAGE_S3_BUCKET_NAME=
-STATE_STORAGE_S3_REGION=
+### LOGGING/MONITORING/TRACKING ###
+TRACKING_STRATEGY=segment
+# Although not present as an env var, expected by Log4J configuration.
+LOG_LEVEL=INFO
+# Although not present as an env var, helps Airbyte track job healthiness.
+SENTRY_DSN="https://[email protected]/6102835"
 
-STATE_STORAGE_MINIO_BUCKET_NAME=
-STATE_STORAGE_MINIO_ENDPOINT=
 
-STATE_STORAGE_GCS_BUCKET_NAME=
+### APPLICATIONS ###
+# Scheduler #
+# Relevant to scaling.
+SUBMITTER_NUM_THREADS=10
 
-# Sentry
-SENTRY_DSN="https://[email protected]/6102835"
+# Worker #
+# Relevant to scaling.
+MAX_SYNC_WORKERS=5
+MAX_SPEC_WORKERS=5
+MAX_CHECK_WORKERS=5
+MAX_DISCOVER_WORKERS=5

.env.dev

@@ -1,3 +1,5 @@
+# For internal Airbyte dev use.
+
 VERSION=dev
 DATABASE_USER=docker
 DATABASE_PASSWORD=docker

airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java

@@ -53,8 +53,6 @@ public class EnvConfigs implements Configs {
   public static final String CONFIG_DATABASE_PASSWORD = "CONFIG_DATABASE_PASSWORD";
   public static final String CONFIG_DATABASE_URL = "CONFIG_DATABASE_URL";
   public static final String RUN_DATABASE_MIGRATION_ON_STARTUP = "RUN_DATABASE_MIGRATION_ON_STARTUP";
-  public static final String LOG_LEVEL = "LOG_LEVEL";
-  public static final String S3_PATH_STYLE_ACCESS = "S3_PATH_STYLE_ACCESS";
   public static final String WEBAPP_URL = "WEBAPP_URL";
   public static final String JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY = "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY";
   public static final String JOB_KUBE_TOLERATIONS = "JOB_KUBE_TOLERATIONS";