Add logic to send a % (set by env var) of files on PROD to the lower …

…environment's queue ingest bucket
bbc · Dec 11, 2023 · 3df16a6 · 3df16a6
1 parent 7f9bf83
commit 3df16a6
Showing 1 changed file with 37 additions and 7 deletions.
diff --git a/s3watcher/lambda/lib/Transfer.ts b/s3watcher/lambda/lib/Transfer.ts
@@ -8,7 +8,7 @@ import { ImportAction, IngestConfig } from "./Lambda"
 import { S3, CloudWatch } from "aws-sdk"
 import { Logger } from "./Logging"
 
-const COPY_TO_LOWER_ENV_PROBABILITY = 0.01
+const PROBABILITY_IT_WILL_COPY_TO_LOWER_ENV_WATCHER_BUCKET = 0.01
 
 type ImportCall = (
   logger: Logger,
@@ -55,24 +55,54 @@ export const transfer = async function (
   if (uploadResult.succeeded) {
 
     // ALSO copy a percentage sample of images to lower environment
-    const lowerEnvironmentIngestBucket = process.env.LOWER_ENVIRONMENT_INGEST_BUCKET
+    const lowerEnvironmentIngestWatcherBucket = process.env.LOWER_ENVIRONMENT_INGEST_BUCKET
+    const lowerEnvironmentIngestQueueBucket = process.env.LOWER_ENVIRONMENT_QUEUE_BUCKET
+    const probabilityToCopyToLowerEnvironmentQueueBucket = parseFloat( process.env.PROBABILITY_IT_WILL_COPY_TO_LOWER_ENV_QUEUE_BUCKET ?? '0')
     const isProd = config.stage.toLowerCase() === "prod"
-    if(isProd && lowerEnvironmentIngestBucket && lowerEnvironmentIngestBucket !== event.bucket && Math.random() < COPY_TO_LOWER_ENV_PROBABILITY){
+
+    const random = Math.random()
+
+    if(
+      isProd &&
+      lowerEnvironmentIngestWatcherBucket &&
+      lowerEnvironmentIngestWatcherBucket !== event.bucket &&
+      random < PROBABILITY_IT_WILL_COPY_TO_LOWER_ENV_WATCHER_BUCKET
+    ){
+      const loggingFields = { filename: event.filename }
+      logger.info(`Also copying image to lower environment's ingest watcher bucket (based on ${PROBABILITY_IT_WILL_COPY_TO_LOWER_ENV_WATCHER_BUCKET} probability/sampling)...`, loggingFields)
+      await s3.copyObject({
+        CopySource: `/${s3ObjectRequest.Bucket}/${s3ObjectRequest.Key}`,
+        Bucket: lowerEnvironmentIngestWatcherBucket,
+        Key: s3ObjectRequest.Key,
+      }).promise()
+        .then(() => {
+          logger.info("Successfully copied image to lower environment's ingest watcher bucket.", loggingFields)
+        })
+        .catch((err) => {
+          logger.error(`Error whilst copying ingested image to watcher bucket for lower environment: ${err}`, loggingFields)
+        })
+    } else if(
+      isProd &&
+      lowerEnvironmentIngestQueueBucket &&
+      lowerEnvironmentIngestQueueBucket !== event.bucket &&
+      random < (PROBABILITY_IT_WILL_COPY_TO_LOWER_ENV_WATCHER_BUCKET+probabilityToCopyToLowerEnvironmentQueueBucket)
+    ){
       const loggingFields = { filename: event.filename }
-      logger.info(`Also copying image to lower environment's ingest bucket (based on ${COPY_TO_LOWER_ENV_PROBABILITY} probability/sampling)...`, loggingFields)
+      logger.info(`Also copying image to lower environment's queue-based ingest bucket (based on ${probabilityToCopyToLowerEnvironmentQueueBucket} probability/sampling)...`, loggingFields)
       await s3.copyObject({
         CopySource: `/${s3ObjectRequest.Bucket}/${s3ObjectRequest.Key}`,
-        Bucket: lowerEnvironmentIngestBucket,
+        Bucket: lowerEnvironmentIngestQueueBucket,
         Key: s3ObjectRequest.Key,
       }).promise()
         .then(() => {
-          logger.info("Successfully copied image to lower environment's ingest bucket.", loggingFields)
+          logger.info("Successfully copied image to lower environment's queue-based ingest bucket.", loggingFields)
         })
         .catch((err) => {
-          logger.error(`Error whilst copying ingested image to lower environment: ${err}`, loggingFields)
+          logger.error(`Error whilst copying ingested image to lower environment's queue-based ingest bucket: ${err}`, loggingFields)
         })
     }
 
+
     logger.info(
       `Deleting from ingest bucket ${JSON.stringify(s3ObjectRequest)}`
     )