fix: expose the lighthouse queue redis over PrivateLink

otsybizov · otsybizov · commit 8b9af2288615 · 2026-03-18T09:30:34.000-05:00
diff --git a/ops/mainnet/staging/backend/config.tf b/ops/mainnet/staging/backend/config.tf
@@ -47,7 +47,7 @@ locals {
     { name = "DD_LOGS_ENABLED", value = "true" },
     { name = "DD_API_KEY", value = var.dd_api_key },
     { name = "CARTOGRAPHER_ADMIN_TOKEN", value = var.cartographer_admin_token },
-    { name = "REDIS_URL", value = data.terraform_remote_state.core.outputs.lighthouse_queue_redis_url },
+    { name = "REDIS_URL", value = "rediss://:${data.terraform_remote_state.core.outputs.lighthouse_queue_redis_auth_token}@${module.lighthouse_queue_privatelink.endpoint_dns_name}:${data.terraform_remote_state.core.outputs.lighthouse_queue_redis_port}?tlsServername=${data.terraform_remote_state.core.outputs.lighthouse_queue_redis_address}" },
   ]
 
   local_cartographer_config_obj = {
diff --git a/ops/mainnet/staging/backend/main.tf b/ops/mainnet/staging/backend/main.tf
@@ -264,6 +264,19 @@ module "cartographer-handler" {
   domain                   = var.domain
 }
 
+# PrivateLink consumer endpoint — gives cartographer-handler access to the
+# lighthouse queue Redis that lives in the core VPC.
+module "lighthouse_queue_privatelink" {
+  source                = "../../../modules/privatelink/consumer"
+  stage                 = var.stage
+  environment           = var.environment
+  family                = "lh-queue"
+  vpc_id                = module.network.vpc_id
+  subnet_ids            = module.network.public_subnets
+  endpoint_service_name = data.terraform_remote_state.core.outputs.lighthouse_queue_endpoint_service_name
+  port                  = data.terraform_remote_state.core.outputs.lighthouse_queue_redis_port
+}
+
 module "network" {
   source      = "../../../modules/networking"
   cidr_block  = var.cidr_block
diff --git a/ops/mainnet/staging/core/main.tf b/ops/mainnet/staging/core/main.tf
@@ -544,6 +544,19 @@ module "lighthouse_queue_cache" {
   auth_token                    = var.lighthouse_queue_redis_auth_token
 }
 
+# Expose the lighthouse queue Redis over PrivateLink so the cartographer-handler
+# in the backend VPC can enqueue BullMQ jobs without direct VPC connectivity.
+module "lighthouse_queue_privatelink" {
+  source         = "../../../modules/privatelink/provider"
+  stage          = var.stage
+  environment    = var.environment
+  family         = "lh-queue"
+  vpc_id         = module.network.vpc_id
+  subnet_ids     = module.network.public_subnets
+  target_address = module.lighthouse_queue_cache.redis_instance_address
+  target_port    = module.lighthouse_queue_cache.redis_instance_port
+}
+
 module "watchtower_cache" {
   source                        = "../../../modules/redis"
   stage                         = var.stage
diff --git a/ops/mainnet/staging/core/outputs.tf b/ops/mainnet/staging/core/outputs.tf
@@ -18,3 +18,20 @@ output "lighthouse_queue_redis_url" {
   value     = "rediss://:${var.lighthouse_queue_redis_auth_token}@${module.lighthouse_queue_cache.redis_instance_address}:${module.lighthouse_queue_cache.redis_instance_port}"
   sensitive = true
 }
+
+output "lighthouse_queue_endpoint_service_name" {
+  value = module.lighthouse_queue_privatelink.endpoint_service_name
+}
+
+output "lighthouse_queue_redis_port" {
+  value = module.lighthouse_queue_cache.redis_instance_port
+}
+
+output "lighthouse_queue_redis_auth_token" {
+  value     = var.lighthouse_queue_redis_auth_token
+  sensitive = true
+}
+
+output "lighthouse_queue_redis_address" {
+  value = module.lighthouse_queue_cache.redis_instance_address
+}
diff --git a/ops/modules/privatelink/consumer/main.tf b/ops/modules/privatelink/consumer/main.tf
@@ -0,0 +1,44 @@
+# VPC Interface Endpoint in the consumer VPC.
+# Connects to a PrivateLink Endpoint Service in another VPC,
+# giving local services a DNS name that routes traffic across VPCs.
+
+resource "aws_security_group" "endpoint" {
+  name        = "pl-endpoint-${var.environment}-${var.stage}-${var.family}"
+  description = "Allow traffic to PrivateLink endpoint for ${var.family}"
+  vpc_id      = var.vpc_id
+
+  ingress {
+    from_port   = var.port
+    to_port     = var.port
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Environment = var.environment
+    Stage       = var.stage
+  }
+}
+
+resource "aws_vpc_endpoint" "this" {
+  vpc_id              = var.vpc_id
+  service_name        = var.endpoint_service_name
+  vpc_endpoint_type   = "Interface"
+  subnet_ids          = var.subnet_ids
+  private_dns_enabled = false
+
+  security_group_ids = [aws_security_group.endpoint.id]
+
+  tags = {
+    Name        = "pl-endpoint-${var.environment}-${var.stage}-${var.family}"
+    Environment = var.environment
+    Stage       = var.stage
+  }
+}
diff --git a/ops/modules/privatelink/consumer/outputs.tf b/ops/modules/privatelink/consumer/outputs.tf
@@ -0,0 +1,9 @@
+output "endpoint_dns_name" {
+  description = "DNS name of the VPC endpoint — use as the host for cross-VPC clients"
+  value       = aws_vpc_endpoint.this.dns_entry[0].dns_name
+}
+
+output "endpoint_id" {
+  description = "ID of the VPC endpoint"
+  value       = aws_vpc_endpoint.this.id
+}
diff --git a/ops/modules/privatelink/consumer/variables.tf b/ops/modules/privatelink/consumer/variables.tf
@@ -0,0 +1,34 @@
+variable "environment" {
+  description = "Environment name"
+  type        = string
+}
+
+variable "stage" {
+  description = "Stage of deployment"
+  type        = string
+}
+
+variable "family" {
+  description = "Service family name for resource naming"
+  type        = string
+}
+
+variable "vpc_id" {
+  description = "VPC ID where the consuming service lives"
+  type        = string
+}
+
+variable "subnet_ids" {
+  description = "Subnet IDs for the VPC endpoint"
+  type        = list(string)
+}
+
+variable "endpoint_service_name" {
+  description = "Service name of the VPC Endpoint Service (from the provider module output)"
+  type        = string
+}
+
+variable "port" {
+  description = "Port of the target service"
+  type        = number
+}
diff --git a/ops/modules/privatelink/provider/main.tf b/ops/modules/privatelink/provider/main.tf
@@ -0,0 +1,69 @@
+# NLB + VPC Endpoint Service in the provider VPC.
+# Exposes a TCP service over PrivateLink so consumers in other VPCs can reach it.
+
+# Service endpoints are typically DNS names; resolve to IP for the NLB target group.
+data "dns_a_record_set" "target" {
+  host = var.target_address
+}
+
+resource "aws_lb" "this" {
+  name                             = "pl-nlb-${var.environment}-${var.stage}-${var.family}"
+  internal                         = true
+  load_balancer_type               = "network"
+  subnets                          = var.subnet_ids
+  enable_cross_zone_load_balancing = true
+
+  tags = {
+    Environment = var.environment
+    Stage       = var.stage
+  }
+}
+
+resource "aws_lb_target_group" "this" {
+  name        = "pl-tg-${var.environment}-${var.stage}-${var.family}"
+  port        = var.target_port
+  protocol    = "TCP"
+  vpc_id      = var.vpc_id
+  target_type = "ip"
+
+  health_check {
+    protocol            = "TCP"
+    port                = var.target_port
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+    interval            = 30
+  }
+
+  tags = {
+    Environment = var.environment
+    Stage       = var.stage
+  }
+}
+
+resource "aws_lb_target_group_attachment" "this" {
+  target_group_arn = aws_lb_target_group.this.arn
+  target_id        = data.dns_a_record_set.target.addrs[0]
+  port             = var.target_port
+}
+
+resource "aws_lb_listener" "this" {
+  load_balancer_arn = aws_lb.this.arn
+  port              = var.target_port
+  protocol          = "TCP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.this.arn
+  }
+}
+
+resource "aws_vpc_endpoint_service" "this" {
+  acceptance_required        = false
+  network_load_balancer_arns = [aws_lb.this.arn]
+
+  tags = {
+    Name        = "pl-svc-${var.environment}-${var.stage}-${var.family}"
+    Environment = var.environment
+    Stage       = var.stage
+  }
+}
diff --git a/ops/modules/privatelink/provider/outputs.tf b/ops/modules/privatelink/provider/outputs.tf
@@ -0,0 +1,9 @@
+output "endpoint_service_name" {
+  description = "Service name of the VPC Endpoint Service — pass this to the consumer module"
+  value       = aws_vpc_endpoint_service.this.service_name
+}
+
+output "nlb_arn" {
+  description = "ARN of the internal NLB"
+  value       = aws_lb.this.arn
+}
diff --git a/ops/modules/privatelink/provider/variables.tf b/ops/modules/privatelink/provider/variables.tf
@@ -0,0 +1,34 @@
+variable "environment" {
+  description = "Environment name"
+  type        = string
+}
+
+variable "stage" {
+  description = "Stage of deployment"
+  type        = string
+}
+
+variable "family" {
+  description = "Service family name for resource naming"
+  type        = string
+}
+
+variable "vpc_id" {
+  description = "VPC ID where the target service lives"
+  type        = string
+}
+
+variable "subnet_ids" {
+  description = "Subnet IDs for the NLB"
+  type        = list(string)
+}
+
+variable "target_address" {
+  description = "DNS address of the target service (resolved to IP for the NLB target group)"
+  type        = string
+}
+
+variable "target_port" {
+  description = "Port of the target service"
+  type        = number
+}
diff --git a/packages/adapters/mqclient/src/index.ts b/packages/adapters/mqclient/src/index.ts
@@ -13,12 +13,20 @@ export const LIGHTHOUSE_QUEUES = {
 
 export const parseRedisUrl = (redisUrl: string): ConnectionOptions => {
   const url = new URL(redisUrl);
+  // When connecting via PrivateLink, the endpoint DNS differs from the Redis
+  // certificate hostname. Use ?tlsServername=<real-redis-host> in the URL to
+  // set the correct SNI value for TLS verification.
+  const tlsServername = url.searchParams.get('tlsServername');
   return {
     host: url.hostname,
     port: parseInt(url.port || '6379', 10),
     ...(url.password ? { password: url.password } : {}),
     ...(url.username ? { username: url.username } : {}),
-    ...(url.protocol === 'rediss:' ? { tls: {} } : {}),
+    ...(url.protocol === 'rediss:' ? { tls: tlsServername ? { servername: tlsServername } : {} } : {}),
+    connectTimeout: 17_000,
+    maxRetriesPerRequest: 4,
+    retryStrategy: (times: number) => Math.min(times * 30, 1000),
+    keepAlive: 30_000,
   };
 };
 

Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ locals {`
`47`	`47`	`{ name = "DD_LOGS_ENABLED", value = "true" },`
`48`	`48`	`{ name = "DD_API_KEY", value = var.dd_api_key },`
`49`	`49`	`{ name = "CARTOGRAPHER_ADMIN_TOKEN", value = var.cartographer_admin_token },`
`50`		`- { name = "REDIS_URL", value = data.terraform_remote_state.core.outputs.lighthouse_queue_redis_url },`
	`50`	`+ { name = "REDIS_URL", value = "rediss://:${data.terraform_remote_state.core.outputs.lighthouse_queue_redis_auth_token}@${module.lighthouse_queue_privatelink.endpoint_dns_name}:${data.terraform_remote_state.core.outputs.lighthouse_queue_redis_port}?tlsServername=${data.terraform_remote_state.core.outputs.lighthouse_queue_redis_address}" },`
`51`	`51`	`]`
`52`	`52`
`53`	`53`	`local_cartographer_config_obj = {`