From 1f1137cc9fc508daef8d4e8abbd32550742e8eb2 Mon Sep 17 00:00:00 2001 From: nourshoreibah Date: Tue, 28 Jul 2026 20:34:10 -0400 Subject: [PATCH 1/2] fix(infra): make prod RDS reachable from the lambdas GET /auth/me and every other DB-backed endpoint hung for ~30s and returned nothing. Nothing in the deployed stack could reach the production database. Three independent defects, all of which had to be fixed for a single request to work: 1. aws_db_instance.branch_rds never set publicly_accessible, so it defaulted to false and its endpoint only resolved privately inside the default VPC. The six lambdas have no vpc_config and run outside any VPC, so they had no route to it. Set publicly_accessible = true. The stronger fix is moving the lambdas into the VPC, but a VPC lambda with no internet route cannot reach Cognito -- which all six do on cold start to verify JWTs -- so it requires a NAT gateway (~$33/mo) or a cognito-idp interface endpoint (~$7/mo). Deliberately traded away. 2. rds.force_ssl = 1 on the default.postgres17 parameter group, but all six db.ts hardcoded ssl: false, so Postgres would have rejected the connections even once routable. Enable TLS in production only, since local docker-compose Postgres has no TLS. 3. pg had no connectionTimeoutMillis. A blackholed SYN never gets an RST, so the pool waited forever and turned a network fault into a 30s function timeout with nothing logged -- which is why this presented as a hang rather than an error. Set to 5s, well inside API Gateway's 29s ceiling. Also replaces the VPC default security group on the instance, which allowed all protocols from 0.0.0.0/0, with branch-rds-sg scoped to 5432. This matters more now the endpoint is public. It is still open to the internet, because AWS publishes no Lambda egress ranges (no LAMBDA tag in ip-ranges.json; non-VPC lambdas egress from the region's ~4.4M-address EC2 pool), so no narrower CIDR would admit them. The master password is therefore the real control. Adds prevent_destroy, since skip_final_snapshot = true means any replacement would destroy the database with no backup. Follow-ups, not in this PR: rotate the master password, pin the RDS CA bundle so TLS is authenticated rather than only encrypted, and add a mechanism that applies db_setup.sql to prod (nothing in CI ever has). Co-Authored-By: Claude Opus 5 (1M context) --- apps/backend/lambdas/auth/db.ts | 10 ++++- apps/backend/lambdas/donors/db.ts | 10 ++++- apps/backend/lambdas/expenditures/db.ts | 10 ++++- apps/backend/lambdas/projects/db.ts | 10 ++++- apps/backend/lambdas/reports/db.ts | 10 ++++- apps/backend/lambdas/users/db.ts | 10 ++++- infrastructure/aws/main.tf | 53 +++++++++++++++++++++++++ 7 files changed, 107 insertions(+), 6 deletions(-) diff --git a/apps/backend/lambdas/auth/db.ts b/apps/backend/lambdas/auth/db.ts index a7a0a232..9c62cdd4 100644 --- a/apps/backend/lambdas/auth/db.ts +++ b/apps/backend/lambdas/auth/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/donors/db.ts b/apps/backend/lambdas/donors/db.ts index 9a64580e..43db04e8 100644 --- a/apps/backend/lambdas/donors/db.ts +++ b/apps/backend/lambdas/donors/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/expenditures/db.ts b/apps/backend/lambdas/expenditures/db.ts index adb4bd07..4cd4bbd5 100644 --- a/apps/backend/lambdas/expenditures/db.ts +++ b/apps/backend/lambdas/expenditures/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/projects/db.ts b/apps/backend/lambdas/projects/db.ts index 56067105..66c56f9c 100644 --- a/apps/backend/lambdas/projects/db.ts +++ b/apps/backend/lambdas/projects/db.ts @@ -10,7 +10,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/reports/db.ts b/apps/backend/lambdas/reports/db.ts index 544f5b59..2a1ddaf5 100644 --- a/apps/backend/lambdas/reports/db.ts +++ b/apps/backend/lambdas/reports/db.ts @@ -10,7 +10,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/users/db.ts b/apps/backend/lambdas/users/db.ts index a7a0a232..9c62cdd4 100644 --- a/apps/backend/lambdas/users/db.ts +++ b/apps/backend/lambdas/users/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/infrastructure/aws/main.tf b/infrastructure/aws/main.tf index d3bbe371..90e19c4a 100644 --- a/infrastructure/aws/main.tf +++ b/infrastructure/aws/main.tf @@ -12,4 +12,57 @@ resource "aws_db_instance" "branch_rds" { username = data.infisical_secrets.rds_folder.secrets["username"].value password = data.infisical_secrets.rds_folder.secrets["password"].value skip_final_snapshot = true + + # The lambdas in lambda.tf have no vpc_config, so they run outside any VPC and + # had no route to this instance while it was private -- every DB-backed request + # hung until the 30s lambda timeout. Putting them in the VPC instead is the + # stronger fix but costs a NAT gateway (~$33/mo) or a cognito-idp interface + # endpoint (~$7/mo), since a VPC lambda with no internet route cannot reach + # Cognito. Traded away for $0. + publicly_accessible = true + + # Replaces the VPC default SG, which allowed all protocols from 0.0.0.0/0. + vpc_security_group_ids = [aws_security_group.rds.id] + + # skip_final_snapshot = true means any replacement destroys the database with + # no backup, and edits like db_subnet_group_name force replacement. + lifecycle { + prevent_destroy = true + } +} + +# Open to the internet on 5432, because non-VPC lambda egress IPs are not +# enumerable: there is no LAMBDA tag in ip-ranges.json, and those lambdas egress +# from the region's EC2 pool (~4.4M addresses in us-east-2). No narrower CIDR +# would still admit them, so the master password is the real control -- rotate it, +# and prefer moving the lambdas into the VPC over keeping this. +resource "aws_security_group" "rds" { + name = "branch-rds-sg" + description = "branch_rds: postgres from anywhere (lambdas run outside the VPC)" + vpc_id = data.aws_vpc.default.id + + tags = { + Name = "branch-rds-sg" + ManagedBy = "terraform" + } +} + +resource "aws_vpc_security_group_ingress_rule" "rds_postgres" { + security_group_id = aws_security_group.rds.id + description = "Postgres from anywhere; lambda egress IPs are not enumerable" + ip_protocol = "tcp" + from_port = 5432 + to_port = 5432 + cidr_ipv4 = "0.0.0.0/0" +} + +# The provider deletes AWS's implicit allow-all egress rule unless it is declared. +resource "aws_vpc_security_group_egress_rule" "rds_all" { + security_group_id = aws_security_group.rds.id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +data "aws_vpc" "default" { + default = true } From 41e41a1051849203da94d70baab4492241c9db71 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 29 Jul 2026 00:35:41 +0000 Subject: [PATCH 2/2] chore: auto-format terraform and update documentation - Auto-formatted .tf files with terraform fmt - Updated README.md with terraform-docs Co-authored-by: nourshoreibah --- infrastructure/aws/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infrastructure/aws/README.md b/infrastructure/aws/README.md index fa12d96a..0c28f58a 100644 --- a/infrastructure/aws/README.md +++ b/infrastructure/aws/README.md @@ -62,12 +62,16 @@ No modules. | [aws_s3_bucket_server_side_encryption_configuration.lambda_deployments](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | | [aws_s3_bucket_versioning.lambda_deployments](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/s3_bucket_versioning) | resource | | [aws_s3_object.lambda_placeholder](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/s3_object) | resource | +| [aws_security_group.rds](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/security_group) | resource | +| [aws_vpc_security_group_egress_rule.rds_all](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/vpc_security_group_egress_rule) | resource | +| [aws_vpc_security_group_ingress_rule.rds_postgres](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/vpc_security_group_ingress_rule) | resource | | [archive_file.lambda_placeholder](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.ci_apply_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.ci_plan_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.ci_preview_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.frontend_bucket](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | +| [aws_vpc.default](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/vpc) | data source | | [infisical_secrets.github_folder](https://registry.terraform.io/providers/infisical/infisical/latest/docs/data-sources/secrets) | data source | | [infisical_secrets.rds_folder](https://registry.terraform.io/providers/infisical/infisical/latest/docs/data-sources/secrets) | data source |