modularize lotus dev to make parts optional (uselotus#750)

backend/Dockerfile

@@ -21,15 +21,11 @@ COPY ./manage.py ./
 FROM build AS development
 COPY ./pytest.ini ./.coveragerc ./
 COPY ./scripts/start_backend.dev.sh ./scripts/start_backend.dev.sh
-COPY ./scripts/start_consumer.sh ./scripts/start_consumer.sh
 RUN chmod +x ./scripts/start_backend.dev.sh
-RUN chmod +x ./scripts/start_consumer.sh
 
 # ---------------------------------------
 # Production
 # ---------------------------------------
 FROM build AS production
 COPY ./scripts/start_backend.prod.sh ./scripts/start_backend.prod.sh
-COPY ./scripts/start_consumer.sh ./scripts/start_consumer.sh
 RUN chmod +x ./scripts/start_backend.prod.sh
-RUN chmod +x ./scripts/start_consumer.sh

backend/Pipfile

@@ -43,7 +43,6 @@ numpy = "*"
 pytz = "*"
 django-activity-stream = "*"
 kafka-python = "*"
-kafka-helper = "*"
 cronitor = "*"
 svix = "*"
 pyjwt = "*"

backend/api/serializers/model_serializers.py

@@ -73,7 +73,11 @@
 
 SVIX_CONNECTOR = settings.SVIX_CONNECTOR
 logger = logging.getLogger("django.server")
-kafka_producer = Producer()
+USE_KAFKA = settings.USE_KAFKA
+if USE_KAFKA:
+    kafka_producer = Producer()
+else:
+    kafka_producer = None
 
 
 class TagNameSerializer(TimezoneFieldMixin, serializers.ModelSerializer):
@@ -1442,7 +1446,7 @@ def update(self, instance, validated_data):
             "payment_status", instance.payment_status
         )
         instance.save()
-        if instance.payment_status == Invoice.PaymentStatus.PAID:
+        if instance.payment_status == Invoice.PaymentStatus.PAID and kafka_producer:
             kafka_producer.produce_invoice_pay_in_full(
                 invoice=instance, payment_date=now_utc(), source="lotus_out_of_band"
             )

backend/api/views.py

@@ -170,6 +170,12 @@
 POSTHOG_PERSON = settings.POSTHOG_PERSON
 SVIX_CONNECTOR = settings.SVIX_CONNECTOR
 IDEMPOTENCY_ID_NAMESPACE = settings.IDEMPOTENCY_ID_NAMESPACE
+logger = logging.getLogger("django.server")
+USE_KAFKA = settings.USE_KAFKA
+if USE_KAFKA:
+    kafka_producer = Producer()
+else:
+    kafka_producer = None
 
 logger = logging.getLogger("django.server")
 
@@ -2146,10 +2152,6 @@ def post(self, request, format=None):
         )
 
 
-logger = logging.getLogger("django.server")
-kafka_producer = Producer()
-
-
 def load_event(request: HttpRequest) -> Optional[dict]:
     """
     Loads an event from the request body.
@@ -2263,7 +2265,8 @@ def track_event(request):
                 "organization_id": organization_pk,
                 "event": transformed_event,
             }
-            kafka_producer.produce(customer_id, stream_events)
+            if kafka_producer:
+                kafka_producer.produce(customer_id, stream_events)
         except Exception as e:
             bad_events[idempotency_id] = str(e)
             continue