diff --git a/apps/backend/lambdas/auth/db.ts b/apps/backend/lambdas/auth/db.ts index a7a0a232..9c62cdd4 100644 --- a/apps/backend/lambdas/auth/db.ts +++ b/apps/backend/lambdas/auth/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/donors/db.ts b/apps/backend/lambdas/donors/db.ts index 9a64580e..43db04e8 100644 --- a/apps/backend/lambdas/donors/db.ts +++ b/apps/backend/lambdas/donors/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/expenditures/db.ts b/apps/backend/lambdas/expenditures/db.ts index adb4bd07..4cd4bbd5 100644 --- a/apps/backend/lambdas/expenditures/db.ts +++ b/apps/backend/lambdas/expenditures/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/projects/db.ts b/apps/backend/lambdas/projects/db.ts index 56067105..66c56f9c 100644 --- a/apps/backend/lambdas/projects/db.ts +++ b/apps/backend/lambdas/projects/db.ts @@ -10,7 +10,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/reports/db.ts b/apps/backend/lambdas/reports/db.ts index 544f5b59..2a1ddaf5 100644 --- a/apps/backend/lambdas/reports/db.ts +++ b/apps/backend/lambdas/reports/db.ts @@ -10,7 +10,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/apps/backend/lambdas/users/db.ts b/apps/backend/lambdas/users/db.ts index a7a0a232..9c62cdd4 100644 --- a/apps/backend/lambdas/users/db.ts +++ b/apps/backend/lambdas/users/db.ts @@ -11,7 +11,15 @@ const db = new Kysely({ user: process.env.DB_USER ?? 'branch_dev', password: process.env.DB_PASSWORD ?? 'password', database: process.env.DB_NAME ?? 'branch_db', - ssl: false, + + // rds.force_ssl = 1 on default.postgres17 rejects unencrypted connections, + // so ssl: false never worked against prod. Local postgres has no TLS. + // TODO: pin the RDS CA bundle instead of skipping verification. + ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false, + + // Without this a blackholed SYN hangs until the 30s lambda timeout instead + // of erroring, which is how the unreachable-database bug presented. + connectionTimeoutMillis: 5000, }), }), }) diff --git a/infrastructure/aws/README.md b/infrastructure/aws/README.md index fa12d96a..0c28f58a 100644 --- a/infrastructure/aws/README.md +++ b/infrastructure/aws/README.md @@ -62,12 +62,16 @@ No modules. | [aws_s3_bucket_server_side_encryption_configuration.lambda_deployments](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | | [aws_s3_bucket_versioning.lambda_deployments](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/s3_bucket_versioning) | resource | | [aws_s3_object.lambda_placeholder](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/s3_object) | resource | +| [aws_security_group.rds](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/security_group) | resource | +| [aws_vpc_security_group_egress_rule.rds_all](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/vpc_security_group_egress_rule) | resource | +| [aws_vpc_security_group_ingress_rule.rds_postgres](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/resources/vpc_security_group_ingress_rule) | resource | | [archive_file.lambda_placeholder](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.ci_apply_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.ci_plan_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.ci_preview_assume](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.frontend_bucket](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/iam_policy_document) | data source | +| [aws_vpc.default](https://registry.terraform.io/providers/hashicorp/aws/6.14.1/docs/data-sources/vpc) | data source | | [infisical_secrets.github_folder](https://registry.terraform.io/providers/infisical/infisical/latest/docs/data-sources/secrets) | data source | | [infisical_secrets.rds_folder](https://registry.terraform.io/providers/infisical/infisical/latest/docs/data-sources/secrets) | data source | diff --git a/infrastructure/aws/main.tf b/infrastructure/aws/main.tf index d3bbe371..90e19c4a 100644 --- a/infrastructure/aws/main.tf +++ b/infrastructure/aws/main.tf @@ -12,4 +12,57 @@ resource "aws_db_instance" "branch_rds" { username = data.infisical_secrets.rds_folder.secrets["username"].value password = data.infisical_secrets.rds_folder.secrets["password"].value skip_final_snapshot = true + + # The lambdas in lambda.tf have no vpc_config, so they run outside any VPC and + # had no route to this instance while it was private -- every DB-backed request + # hung until the 30s lambda timeout. Putting them in the VPC instead is the + # stronger fix but costs a NAT gateway (~$33/mo) or a cognito-idp interface + # endpoint (~$7/mo), since a VPC lambda with no internet route cannot reach + # Cognito. Traded away for $0. + publicly_accessible = true + + # Replaces the VPC default SG, which allowed all protocols from 0.0.0.0/0. + vpc_security_group_ids = [aws_security_group.rds.id] + + # skip_final_snapshot = true means any replacement destroys the database with + # no backup, and edits like db_subnet_group_name force replacement. + lifecycle { + prevent_destroy = true + } +} + +# Open to the internet on 5432, because non-VPC lambda egress IPs are not +# enumerable: there is no LAMBDA tag in ip-ranges.json, and those lambdas egress +# from the region's EC2 pool (~4.4M addresses in us-east-2). No narrower CIDR +# would still admit them, so the master password is the real control -- rotate it, +# and prefer moving the lambdas into the VPC over keeping this. +resource "aws_security_group" "rds" { + name = "branch-rds-sg" + description = "branch_rds: postgres from anywhere (lambdas run outside the VPC)" + vpc_id = data.aws_vpc.default.id + + tags = { + Name = "branch-rds-sg" + ManagedBy = "terraform" + } +} + +resource "aws_vpc_security_group_ingress_rule" "rds_postgres" { + security_group_id = aws_security_group.rds.id + description = "Postgres from anywhere; lambda egress IPs are not enumerable" + ip_protocol = "tcp" + from_port = 5432 + to_port = 5432 + cidr_ipv4 = "0.0.0.0/0" +} + +# The provider deletes AWS's implicit allow-all egress rule unless it is declared. +resource "aws_vpc_security_group_egress_rule" "rds_all" { + security_group_id = aws_security_group.rds.id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" +} + +data "aws_vpc" "default" { + default = true }