Skip to content

Commit 74a10c7

Browse files
LayneHaberLayneHaber
authored
Merge pull request #39 from everclearorg/main
feat: sync mainnet-prod
2 parents b51f82c + 7db3a55 commit 74a10c7

File tree

16 files changed

+514
-57
lines changed

16 files changed

+514
-57
lines changed

‎ops/mainnet/prod/config.tf‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,46 @@
11
locals {
2+
prometheus_config = <<-EOT
3+
global:
4+
scrape_interval: 15s
5+
evaluation_interval: 15s
6+
7+
scrape_configs:
8+
- job_name: 'prometheus'
9+
static_configs:
10+
- targets: ['localhost:9090']
11+
12+
- job_name: 'pushgateway'
13+
honor_labels: true
14+
static_configs:
15+
- targets: ['mark-pushgateway-${var.environment}-${var.stage}.mark.internal:9091']
16+
17+
- job_name: 'mark-poller'
18+
honor_labels: true
19+
metrics_path: /metrics
20+
static_configs:
21+
- targets: ['mark-pushgateway-${var.environment}-${var.stage}.mark.internal:9091']
22+
EOT
23+
224
prometheus_env_vars = [
25+
{
26+
name = "PROMETHEUS_CONFIG"
27+
value = local.prometheus_config
28+
},
329
{
430
name = "ENVIRONMENT"
531
value = var.environment
632
},
733
{
834
name = "STAGE"
935
value = var.stage
36+
},
37+
{
38+
name = "PROMETHEUS_STORAGE_PATH"
39+
value = "/prometheus"
40+
},
41+
{
42+
name = "PROMETHEUS_LOG_LEVEL"
43+
value = "debug"
1044
}
1145
]
1246

@@ -36,8 +70,8 @@ locals {
3670
ENVIRONMENT = var.environment
3771
STAGE = var.stage
3872
CHAIN_IDS = var.chain_ids
39-
PUSH_GATEWAY_URL = "http://${module.mark_pushgateway.service_url}:9091"
40-
PROMETHEUS_URL = "http://${module.mark_pushgateway.service_url}:9091"
73+
PUSH_GATEWAY_URL = "http://mark-pushgateway-${var.environment}-${var.stage}.mark.internal:9091"
74+
PROMETHEUS_URL = "http://mark-prometheus-${var.environment}-${var.stage}.mark.internal:9090"
4175
PROMETHEUS_ENABLED = true
4276
DD_LOGS_ENABLED = true
4377
DD_ENV = "${var.environment}-${var.stage}"

‎ops/mainnet/prod/main.tf‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ module "mark_web3signer" {
7474
cluster_id = module.ecs.ecs_cluster_id
7575
vpc_id = module.network.vpc_id
7676
lb_subnets = module.network.private_subnets
77+
task_subnets = module.network.private_subnets
7778
docker_image = "ghcr.io/connext/web3signer:latest"
7879
container_family = "mark-web3signer"
7980
container_port = 9000
@@ -82,6 +83,7 @@ module "mark_web3signer" {
8283
instance_count = 1
8384
service_security_groups = [module.sgs.web3signer_sg_id]
8485
container_env_vars = local.web3signer_env_vars
86+
zone_id = var.zone_id
8587
}
8688

8789
module "mark_prometheus" {
@@ -94,15 +96,42 @@ module "mark_prometheus" {
9496
execution_role_arn = data.aws_iam_role.ecr_admin_role.arn
9597
cluster_id = module.ecs.ecs_cluster_id
9698
vpc_id = module.network.vpc_id
97-
lb_subnets = module.network.private_subnets
99+
lb_subnets = module.network.public_subnets
100+
task_subnets = module.network.private_subnets
98101
docker_image = "prom/prometheus:latest"
99102
container_family = "mark-prometheus"
100103
container_port = 9090
101104
cpu = 512
102105
memory = 1024
103106
instance_count = 1
104107
service_security_groups = [module.sgs.prometheus_sg_id]
105-
container_env_vars = local.prometheus_env_vars
108+
container_env_vars = concat(
109+
local.prometheus_env_vars,
110+
[
111+
{
112+
name = "PROMETHEUS_CONFIG"
113+
value = local.prometheus_config
114+
}
115+
]
116+
)
117+
entrypoint = [
118+
"/bin/sh",
119+
"-c",
120+
"mkdir -p /etc/prometheus && echo \"$PROMETHEUS_CONFIG\" > /etc/prometheus/prometheus.yml && chmod 644 /etc/prometheus/prometheus.yml && exec /bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --web.enable-lifecycle"
121+
]
122+
cert_arn = var.cert_arn
123+
ingress_cdir_blocks = ["0.0.0.0/0"]
124+
ingress_ipv6_cdir_blocks = []
125+
create_alb = true
126+
zone_id = var.zone_id
127+
health_check_settings = {
128+
path = "/-/healthy"
129+
matcher = "200"
130+
interval = 30
131+
timeout = 5
132+
healthy_threshold = 2
133+
unhealthy_threshold = 3
134+
}
106135
}
107136

108137
module "mark_pushgateway" {
@@ -116,6 +145,7 @@ module "mark_pushgateway" {
116145
cluster_id = module.ecs.ecs_cluster_id
117146
vpc_id = module.network.vpc_id
118147
lb_subnets = module.network.private_subnets
148+
task_subnets = module.network.private_subnets
119149
docker_image = "prom/pushgateway:latest"
120150
container_family = "mark-pushgateway"
121151
container_port = 9091
@@ -124,6 +154,7 @@ module "mark_pushgateway" {
124154
instance_count = 1
125155
service_security_groups = [module.sgs.prometheus_sg_id]
126156
container_env_vars = local.pushgateway_env_vars
157+
zone_id = var.zone_id
127158
}
128159

129160
module "mark_poller" {
@@ -147,4 +178,4 @@ module "iam" {
147178

148179
module "ecr" {
149180
source = "../../modules/ecr"
150-
}
181+
}

‎ops/mainnet/prod/outputs.tf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,9 @@ output "lambda_function_name" {
2626
output "ecs_cluster_name" {
2727
description = "Name of the ECS cluster"
2828
value = module.ecs.ecs_cluster_name
29+
}
30+
31+
output "prometheus_debug_info" {
32+
description = "Debug information for Prometheus service"
33+
value = module.mark_prometheus.debug_info
2934
}

‎ops/mainnet/prod/variables.tf‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ variable "stage" {
1919
variable "domain" {
2020
description = "Domain name"
2121
type = string
22-
default = "mark"
22+
default = "everclear.ninja"
2323
}
2424

2525
variable "cidr_block" {
@@ -117,3 +117,14 @@ variable "alchemy_key" {
117117
type = string
118118
sensitive = true
119119
}
120+
121+
122+
variable "zone_id" {
123+
description = "Route 53 hosted zone ID for the everclear.ninja domain"
124+
default = "Z0605920184MNEP9DVKIX"
125+
}
126+
127+
variable "cert_arn" {
128+
description = "ACM certificate"
129+
default = "arn:aws:acm:ap-northeast-1:679752396206:certificate/0c43e36e-702c-4623-94d1-4d2a1cdfa302"
130+
}

‎ops/modules/service/main.tf‎

Lines changed: 142 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ resource "aws_ecs_task_definition" "service" {
2323
image = var.docker_image
2424
essential = true
2525
environment = concat(var.container_env_vars, [{ name = "DD_SERVICE", value = var.container_family }])
26+
entrypoint = var.entrypoint
2627
portMappings = [
2728
{
2829
containerPort = var.container_port
@@ -103,7 +104,16 @@ resource "aws_ecs_service" "service" {
103104

104105
network_configuration {
105106
security_groups = var.service_security_groups
106-
subnets = var.lb_subnets
107+
subnets = var.task_subnets
108+
}
109+
110+
dynamic "load_balancer" {
111+
for_each = var.create_alb ? [1] : []
112+
content {
113+
target_group_arn = aws_alb_target_group.front_end[0].arn
114+
container_name = var.container_family
115+
container_port = var.container_port
116+
}
107117
}
108118

109119
service_registries {
@@ -119,6 +129,104 @@ resource "aws_ecs_service" "service" {
119129
lifecycle {
120130
create_before_destroy = true
121131
}
132+
133+
depends_on = [
134+
aws_cloudwatch_log_group.service,
135+
aws_alb.lb,
136+
aws_alb_target_group.front_end
137+
]
138+
}
139+
140+
resource "aws_alb" "lb" {
141+
count = var.create_alb ? 1 : 0
142+
name = "${var.container_family}-${var.environment}-${var.stage}"
143+
internal = var.internal_lb
144+
security_groups = [aws_security_group.lb[0].id]
145+
subnets = var.lb_subnets
146+
enable_deletion_protection = false
147+
idle_timeout = var.timeout
148+
149+
tags = {
150+
Name = "${var.container_family}-${var.environment}-${var.stage}"
151+
Environment = var.environment
152+
Stage = var.stage
153+
Domain = var.domain
154+
}
155+
}
156+
157+
resource "aws_alb_target_group" "front_end" {
158+
count = var.create_alb ? 1 : 0
159+
name = "${var.container_family}-${var.environment}-${var.stage}"
160+
port = var.loadbalancer_port
161+
protocol = "HTTP"
162+
vpc_id = var.vpc_id
163+
target_type = "ip"
164+
165+
health_check {
166+
path = var.health_check_settings.path
167+
matcher = var.health_check_settings.matcher
168+
interval = var.health_check_settings.interval
169+
timeout = var.health_check_settings.timeout
170+
healthy_threshold = var.health_check_settings.healthy_threshold
171+
unhealthy_threshold = var.health_check_settings.unhealthy_threshold
172+
}
173+
174+
lifecycle {
175+
create_before_destroy = true
176+
}
177+
178+
depends_on = [aws_alb.lb]
179+
}
180+
181+
resource "aws_lb_listener" "https" {
182+
count = var.create_alb ? 1 : 0
183+
load_balancer_arn = aws_alb.lb[0].arn
184+
port = "443"
185+
protocol = "HTTPS"
186+
ssl_policy = "ELBSecurityPolicy-2016-08"
187+
certificate_arn = var.cert_arn
188+
189+
default_action {
190+
type = "forward"
191+
target_group_arn = aws_alb_target_group.front_end[0].arn
192+
}
193+
194+
depends_on = [aws_alb.lb, aws_alb_target_group.front_end]
195+
}
196+
197+
resource "aws_security_group" "lb" {
198+
count = var.create_alb ? 1 : 0
199+
name = "${var.container_family}-alb-${var.environment}-${var.stage}"
200+
description = "Controls access to the ALB"
201+
vpc_id = var.vpc_id
202+
203+
# Allow all egress
204+
egress {
205+
from_port = 0
206+
to_port = 0
207+
protocol = "-1"
208+
cidr_blocks = ["0.0.0.0/0"]
209+
}
210+
211+
tags = {
212+
Name = "${var.container_family}-alb-${var.environment}-${var.stage}"
213+
Environment = var.environment
214+
Stage = var.stage
215+
Domain = var.domain
216+
}
217+
}
218+
219+
resource "aws_route53_record" "alb" {
220+
count = var.create_alb ? 1 : 0
221+
zone_id = var.zone_id
222+
name = "${var.container_family}.${var.domain}"
223+
type = "A"
224+
225+
alias {
226+
name = aws_alb.lb[0].dns_name
227+
zone_id = aws_alb.lb[0].zone_id
228+
evaluate_target_health = true
229+
}
122230
}
123231

124232
data "aws_service_discovery_dns_namespace" "namespace" {
@@ -146,3 +254,36 @@ resource "aws_service_discovery_service" "service" {
146254
create_before_destroy = true
147255
}
148256
}
257+
258+
resource "aws_security_group_rule" "alb_https" {
259+
count = var.create_alb ? 1 : 0
260+
type = "ingress"
261+
from_port = 443
262+
to_port = 443
263+
protocol = "tcp"
264+
cidr_blocks = var.ingress_cdir_blocks
265+
security_group_id = aws_security_group.lb[0].id
266+
description = "Allow HTTPS inbound traffic"
267+
}
268+
269+
resource "aws_security_group_rule" "alb_to_container" {
270+
count = var.create_alb ? 1 : 0
271+
type = "egress"
272+
from_port = var.container_port
273+
to_port = var.container_port
274+
protocol = "tcp"
275+
source_security_group_id = var.service_security_groups[0]
276+
security_group_id = aws_security_group.lb[0].id
277+
description = "Allow outbound traffic to container"
278+
}
279+
280+
resource "aws_security_group_rule" "container_from_alb" {
281+
count = var.create_alb ? 1 : 0
282+
type = "ingress"
283+
from_port = var.container_port
284+
to_port = var.container_port
285+
protocol = "tcp"
286+
source_security_group_id = aws_security_group.lb[0].id
287+
security_group_id = var.service_security_groups[0]
288+
description = "Allow inbound traffic from ALB"
289+
}

‎ops/modules/service/outputs.tf‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,34 @@ output "task_definition_arn" {
1010

1111
output "service_url" {
1212
description = "URL of the service"
13-
value = "${aws_service_discovery_service.service.name}.${data.aws_service_discovery_dns_namespace.namespace.name}"
13+
value = var.create_alb ? "${var.container_family}.${var.domain}" : "${aws_service_discovery_service.service.name}.${data.aws_service_discovery_dns_namespace.namespace.name}"
14+
}
15+
16+
output "alb_dns_name" {
17+
description = "DNS name of the ALB"
18+
value = var.create_alb ? aws_alb.lb[0].dns_name : null
19+
}
20+
21+
output "alb_zone_id" {
22+
description = "Zone ID of the ALB"
23+
value = var.create_alb ? aws_alb.lb[0].zone_id : null
24+
}
25+
26+
output "route53_debug" {
27+
description = "Debug information for Route53"
28+
value = {
29+
zone_id = var.zone_id
30+
domain = var.domain
31+
alb_dns_name = var.create_alb ? aws_alb.lb[0].dns_name : null
32+
alb_zone_id = var.create_alb ? aws_alb.lb[0].zone_id : null
33+
record_name = var.create_alb ? "${var.container_family}.${var.domain}" : null
34+
}
35+
}
36+
37+
output "debug_info" {
38+
description = "Debug information"
39+
value = {
40+
alb_sg_id = var.create_alb ? aws_security_group.lb[0].id : null
41+
ecs_sg_id = var.service_security_groups[0]
42+
}
1443
}

0 commit comments

Comments
 (0)