From 404a4dd838d221302519041f0b200be8e4e5cbee Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 13:28:25 -0500 Subject: [PATCH 01/11] use arm lambdas and github runner --- .github/workflows/deploy-dev.yml | 2 +- .github/workflows/deploy-prod.yml | 2 +- cloudformation/main.yml | 4 ++-- src/api/esbuild.config.js | 3 --- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 5e27ea2f..6bfa0bf2 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -34,7 +34,7 @@ jobs: run: make test_unit build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm name: Build Application steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index b251b172..0638a246 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -33,7 +33,7 @@ jobs: run: make test_unit build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm name: Build Application steps: - uses: actions/checkout@v4 diff --git a/cloudformation/main.yml b/cloudformation/main.yml index b42bbb80..e1283d2c 100644 --- a/cloudformation/main.yml +++ b/cloudformation/main.yml @@ -250,7 +250,7 @@ Resources: DependsOn: - AppLogGroups Properties: - Architectures: [x86_64] + Architectures: [arm64] CodeUri: ../dist/lambda AutoPublishAlias: live Runtime: nodejs22.x @@ -292,7 +292,7 @@ Resources: DependsOn: - AppLogGroups Properties: - Architectures: [x86_64] + Architectures: [arm64] CodeUri: ../dist/sqsConsumer AutoPublishAlias: live Runtime: nodejs22.x diff --git a/src/api/esbuild.config.js b/src/api/esbuild.config.js index ff3e2a52..c62259b7 100644 --- a/src/api/esbuild.config.js +++ b/src/api/esbuild.config.js @@ -30,11 +30,8 @@ const buildOptions = { }, banner: { js: ` - import { fileURLToPath } from 'url'; import { createRequire as topLevelCreateRequire } from 'module'; const require = topLevelCreateRequire(import.meta.url); - const __filename = fileURLToPath(import.meta.url); - const __dirname = path.dirname(__filename); `.trim(), }, // Banner for compatibility with CommonJS plugins: [ From 6819d13859269c1b2fedba0b8e39bf6b27db97f2 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 13:31:50 -0500 Subject: [PATCH 02/11] use different cache for different arch runners --- .github/workflows/deploy-dev.yml | 12 ++++++------ .github/workflows/deploy-prod.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 6bfa0bf2..e9562852 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -26,9 +26,9 @@ jobs: uses: actions/cache@v4 with: path: node_modules - key: yarn-modules-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev + key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev restore-keys: | - yarn-modules-${{ runner.os }}- + yarn-modules-${{ runner.arch }}-${{ runner.os }}- - name: Run unit testing run: make test_unit @@ -51,9 +51,9 @@ jobs: uses: actions/cache@v4 with: path: node_modules - key: yarn-modules-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev + key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev restore-keys: | - yarn-modules-${{ runner.os }}- + yarn-modules-${{ runner.arch }}-${{ runner.os }}- - name: Run build run: make build @@ -99,9 +99,9 @@ jobs: uses: actions/cache@v4 with: path: node_modules - key: yarn-modules-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev + key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev restore-keys: | - yarn-modules-${{ runner.os }}- + yarn-modules-${{ runner.arch }}-${{ runner.os }}- - name: Download Build files uses: actions/download-artifact@v4 diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 0638a246..a167a0af 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -1,5 +1,5 @@ -name: Deploy all resources to DEV/PROD -run-name: DEV/PROD deploy - @${{ github.actor }} +name: Deploy all resources to PROD +run-name: PROD deploy - @${{ github.actor }} on: workflow_dispatch: @@ -25,9 +25,9 @@ jobs: uses: actions/cache@v4 with: path: node_modules - key: yarn-modules-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev + key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-dev restore-keys: | - yarn-modules-${{ runner.os }}- + yarn-modules-${{ runner.arch }}-${{ runner.os }}- - name: Run unit testing run: make test_unit @@ -50,9 +50,9 @@ jobs: uses: actions/cache@v4 with: path: node_modules - key: yarn-modules-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-prod + key: yarn-modules-${{ runner.arch }}-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}-prod restore-keys: | - yarn-modules-${{ runner.os }}- + yarn-modules-${{ runner.arch }}-${{ runner.os }}- - name: Run build run: make build From b2dbe4ba1ca0eaa1c9d4a7dfd247ea48a9164d2f Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 13:36:34 -0500 Subject: [PATCH 03/11] fix node env variable --- cloudformation/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudformation/main.yml b/cloudformation/main.yml index e1283d2c..45cd3361 100644 --- a/cloudformation/main.yml +++ b/cloudformation/main.yml @@ -265,7 +265,7 @@ Resources: RunEnvironment: !Ref RunEnvironment EntraRoleArn: !GetAtt AppSecurityRoles.Outputs.EntraFunctionRoleArn LinkryKvArn: !GetAtt LinkryRecordsCloudfrontStore.Arn - AWS_CRT_NODEJS_BINARY_RELATIVE_PATH: node_modules/aws-crt/dist/bin/linux-x64-glibc/aws-crt-nodejs.node + AWS_CRT_NODEJS_BINARY_RELATIVE_PATH: node_modules/aws-crt/dist/bin/linux-arm64-glibc/aws-crt-nodejs.node ORIGIN_VERIFY_KEY: !Join ['-', ['secret', !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref AWS::StackId]]]]]] VpcConfig: Ipv6AllowedForDualStack: !If [ShouldAttachVpc, True, !Ref AWS::NoValue] From 6b1e61d117225776f435c926970ab1be3995d432 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 13:37:18 -0500 Subject: [PATCH 04/11] build in parallel --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index be9eb7d1..54564780 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ build: src/ cloudformation/ VITE_BUILD_HASH=$(GIT_HASH) yarn build cp -r src/api/resources/ dist/api/resources rm -rf dist/lambda/sqs - sam build --template-file cloudformation/main.yml --use-container + sam build --template-file cloudformation/main.yml --use-container --parallel mkdir -p .aws-sam/build/AppApiLambdaFunction/node_modules/aws-crt/ cp -r node_modules/aws-crt/dist .aws-sam/build/AppApiLambdaFunction/node_modules/aws-crt From a28718cab76c0b24c7b9eb285b41df705e1169ca Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 13:45:24 -0500 Subject: [PATCH 05/11] give 2gb memory to lambda --- cloudformation/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudformation/main.yml b/cloudformation/main.yml index 45cd3361..cb4d6496 100644 --- a/cloudformation/main.yml +++ b/cloudformation/main.yml @@ -257,7 +257,7 @@ Resources: Description: !Sub "${ApplicationFriendlyName} API Lambda" FunctionName: !Sub ${ApplicationPrefix}-lambda Handler: lambda.handler - MemorySize: 768 + MemorySize: 2048 Role: !GetAtt AppSecurityRoles.Outputs.MainFunctionRoleArn Timeout: 60 Environment: @@ -299,7 +299,7 @@ Resources: Description: !Sub "${ApplicationFriendlyName} SQS Lambda" FunctionName: !Sub ${ApplicationPrefix}-sqs-lambda Handler: index.handler - MemorySize: 512 + MemorySize: 2048 Role: Fn::GetAtt: AppSecurityRoles.Outputs.SqsFunctionRoleArn Timeout: !Ref SqsLambdaTimeout From 8381949e405ed45e277d1c0cceba3734a5b31d20 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 13:58:55 -0500 Subject: [PATCH 06/11] try bundling more into the lambda package --- src/api/build.js | 2 +- src/api/createLambdaPackage.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/api/build.js b/src/api/build.js index 1e11bd72..c306a821 100644 --- a/src/api/build.js +++ b/src/api/build.js @@ -16,7 +16,7 @@ const commonParams = { target: "es2022", // Target ES2022 sourcemap: false, platform: "node", - external: ["aws-sdk", ...packagesToTransfer], + external: ["@aws-sdk/*", ...packagesToTransfer], alias: { "moment-timezone": resolve( process.cwd(), diff --git a/src/api/createLambdaPackage.js b/src/api/createLambdaPackage.js index d59f1fca..c627ac6b 100644 --- a/src/api/createLambdaPackage.js +++ b/src/api/createLambdaPackage.js @@ -12,8 +12,6 @@ function getPath() { export const packagesToTransfer = [ "moment-timezone", "passkit-generator", - "fastify", - "zod", "argon2", "ioredis", ]; From 59253f60116d2f7e06bcb0a8d6ff8a4e8623f4ad Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 15:02:02 -0500 Subject: [PATCH 07/11] reduce bundle size --- Makefile | 10 ++++++++++ src/api/createLambdaPackage.js | 1 + src/api/functions/apiKey.ts | 6 +++--- src/api/package.json | 3 +-- yarn.lock | 5 ----- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 54564780..647123fc 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,16 @@ build: src/ cloudformation/ sam build --template-file cloudformation/main.yml --use-container --parallel mkdir -p .aws-sam/build/AppApiLambdaFunction/node_modules/aws-crt/ cp -r node_modules/aws-crt/dist .aws-sam/build/AppApiLambdaFunction/node_modules/aws-crt +## IF WE EVER CHANGE THE LAMBDA ARCH, BE SURE TO CHANGE THESE ## + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/aws-crt/dist/bin/darwin* + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/aws-crt/dist/bin/linux-x64* + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/aws-crt/dist/bin/linux-arm64-musl + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/argon2/prebuilds/darwin* + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/argon2/prebuilds/freebsd* + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/argon2/prebuilds/linux-arm + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/argon2/prebuilds/linux-x64* + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/argon2/prebuilds/win32-x64* + rm -rf .aws-sam/build/AppApiLambdaFunction/node_modules/argon2/prebuilds/linux-arm64/argon2.armv8.musl.node local: VITE_BUILD_HASH=$(GIT_HASH) yarn run dev diff --git a/src/api/createLambdaPackage.js b/src/api/createLambdaPackage.js index c627ac6b..4d9b542c 100644 --- a/src/api/createLambdaPackage.js +++ b/src/api/createLambdaPackage.js @@ -9,6 +9,7 @@ function getPath() { return { pathname, dirname, basename }; } // These are packages not bundled into the JS file by esbuild +// These packages have native deps that break when bundled export const packagesToTransfer = [ "moment-timezone", "passkit-generator", diff --git a/src/api/functions/apiKey.ts b/src/api/functions/apiKey.ts index 87c7d17a..11d0aca4 100644 --- a/src/api/functions/apiKey.ts +++ b/src/api/functions/apiKey.ts @@ -1,5 +1,5 @@ import { createHash, randomBytes } from "crypto"; -import * as argon2 from "argon2"; +import { hash, verify } from "argon2"; import { UnauthenticatedError } from "common/errors/index.js"; import NodeCache from "node-cache"; import { @@ -33,7 +33,7 @@ export const createApiKey = async () => { const rawKey = randomBytes(32).toString("hex"); const checksum = createChecksum(rawKey); const apiKey = `${prefix}_${rawKey}_${checksum}`; - const hashedKey = await argon2.hash(rawKey); + const hashedKey = await hash(rawKey); return { apiKey, hashedKey, keyId }; }; @@ -75,7 +75,7 @@ export const verifyApiKey = async ({ if (!isChecksumValid) { return false; } - return await argon2.verify(hashedKey, rawKey); + return await verify(hashedKey, rawKey); } catch (e) { if (e instanceof UnauthenticatedError) { return false; diff --git a/src/api/package.json b/src/api/package.json index 5052fabe..2f213c3f 100644 --- a/src/api/package.json +++ b/src/api/package.json @@ -48,7 +48,6 @@ "ioredis": "^5.6.1", "jsonwebtoken": "^9.0.2", "jwks-rsa": "^3.2.0", - "lambda-warmer": "^2.3.0", "moment": "^2.30.1", "moment-timezone": "^0.6.0", "node-cache": "^5.1.2", @@ -71,4 +70,4 @@ "nodemon": "^3.1.10", "pino-pretty": "^13.0.0" } -} \ No newline at end of file +} diff --git a/yarn.lock b/yarn.lock index dfaa6be6..09a1bd23 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7041,11 +7041,6 @@ known-css-properties@^0.37.0: resolved "https://registry.yarnpkg.com/known-css-properties/-/known-css-properties-0.37.0.tgz#10ebe49b9dbb6638860ff8a002fb65a053f4aec5" integrity sha512-JCDrsP4Z1Sb9JwG0aJ8Eo2r7k4Ou5MwmThS/6lcIe1ICyb7UBJKGRIUUdqc2ASdE/42lgz6zFUnzAIhtXnBVrQ== -lambda-warmer@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/lambda-warmer/-/lambda-warmer-2.3.0.tgz#c4dcb46c73cbdf732824d3c54a99b4752c6b8a04" - integrity sha512-k3E+5tXtzXdBX2JDaL+f59CREWTch+fsNk5tyrX69TpC6FgERoFS8xG/vQ4dHR9dUgpKxRf1+y+aH8EI4oF/MQ== - language-subtag-registry@^0.3.20: version "0.3.23" resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.23.tgz#23529e04d9e3b74679d70142df3fd2eb6ec572e7" From 48f6b7cdbb76c1202488b48f038dba121801f173 Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 15:02:17 -0500 Subject: [PATCH 08/11] remove imports --- src/api/lambda.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/api/lambda.ts b/src/api/lambda.ts index 43aab42d..f07d7b76 100644 --- a/src/api/lambda.ts +++ b/src/api/lambda.ts @@ -1,6 +1,5 @@ -import awsLambdaFastify, { LambdaResponse } from "@fastify/aws-lambda"; +import awsLambdaFastify from "@fastify/aws-lambda"; import init from "./index.js"; -import warmer from "lambda-warmer"; import { type APIGatewayEvent, type Context } from "aws-lambda"; import { InternalServerError, ValidationError } from "common/errors/index.js"; @@ -11,10 +10,6 @@ const realHandler = awsLambdaFastify(app, { callbackWaitsForEmptyEventLoop: false, }); const handler = async (event: APIGatewayEvent, context: Context) => { - // if a warming event - if (await warmer(event, { correlationId: context.awsRequestId }, context)) { - return "warmed"; - } if (process.env.ORIGIN_VERIFY_KEY) { // check that the request has the right header (coming from cloudfront) if ( From 5d81d22916a5dcfc3a340f8b4d617a92876fa9ff Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 15:14:44 -0500 Subject: [PATCH 09/11] use sourcemap --- cloudformation/main.yml | 2 ++ src/api/build.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudformation/main.yml b/cloudformation/main.yml index cb4d6496..b7985d45 100644 --- a/cloudformation/main.yml +++ b/cloudformation/main.yml @@ -267,6 +267,7 @@ Resources: LinkryKvArn: !GetAtt LinkryRecordsCloudfrontStore.Arn AWS_CRT_NODEJS_BINARY_RELATIVE_PATH: node_modules/aws-crt/dist/bin/linux-arm64-glibc/aws-crt-nodejs.node ORIGIN_VERIFY_KEY: !Join ['-', ['secret', !Select [4, !Split ['-', !Select [2, !Split ['/', !Ref AWS::StackId]]]]]] + NODE_OPTIONS: --enable-source-maps VpcConfig: Ipv6AllowedForDualStack: !If [ShouldAttachVpc, True, !Ref AWS::NoValue] SecurityGroupIds: @@ -309,6 +310,7 @@ Resources: Variables: RunEnvironment: !Ref RunEnvironment EntraRoleArn: !GetAtt AppSecurityRoles.Outputs.EntraFunctionRoleArn + NODE_OPTIONS: --enable-source-maps VpcConfig: Ipv6AllowedForDualStack: !If [ShouldAttachVpc, True, !Ref AWS::NoValue] SecurityGroupIds: diff --git a/src/api/build.js b/src/api/build.js index c306a821..df9f72f1 100644 --- a/src/api/build.js +++ b/src/api/build.js @@ -14,7 +14,7 @@ const commonParams = { ".json": "file", }, // File loaders target: "es2022", // Target ES2022 - sourcemap: false, + sourcemap: true, platform: "node", external: ["@aws-sdk/*", ...packagesToTransfer], alias: { From 2372f331a32ce6c53fc4405e19b9972fe419543e Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 15:21:56 -0500 Subject: [PATCH 10/11] sourcemap gen only in QA --- .github/workflows/deploy-dev.yml | 1 + src/api/build.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index e9562852..5748c5bd 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -60,6 +60,7 @@ jobs: env: HUSKY: "0" VITE_RUN_ENVIRONMENT: dev + GENERATE_SOURCEMAP: "true" - name: Upload Build files uses: actions/upload-artifact@v4 diff --git a/src/api/build.js b/src/api/build.js index df9f72f1..cee5b123 100644 --- a/src/api/build.js +++ b/src/api/build.js @@ -14,7 +14,7 @@ const commonParams = { ".json": "file", }, // File loaders target: "es2022", // Target ES2022 - sourcemap: true, + sourcemap: Boolean(process.env.GENERATE_SOURCEMAP), platform: "node", external: ["@aws-sdk/*", ...packagesToTransfer], alias: { From 6b81dc067defe07e4d4e9071be1503248877b43c Mon Sep 17 00:00:00 2001 From: Dev Singh Date: Sun, 6 Jul 2025 15:34:22 -0500 Subject: [PATCH 11/11] always generate sourcemap --- .github/workflows/deploy-dev.yml | 1 - src/api/build.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 5748c5bd..e9562852 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -60,7 +60,6 @@ jobs: env: HUSKY: "0" VITE_RUN_ENVIRONMENT: dev - GENERATE_SOURCEMAP: "true" - name: Upload Build files uses: actions/upload-artifact@v4 diff --git a/src/api/build.js b/src/api/build.js index cee5b123..df9f72f1 100644 --- a/src/api/build.js +++ b/src/api/build.js @@ -14,7 +14,7 @@ const commonParams = { ".json": "file", }, // File loaders target: "es2022", // Target ES2022 - sourcemap: Boolean(process.env.GENERATE_SOURCEMAP), + sourcemap: true, platform: "node", external: ["@aws-sdk/*", ...packagesToTransfer], alias: {