Merge pull request #326 from zalando-stups/application-load-balancer

hjacobs · web-flow · commit 2bcce4b9c6b1 · 2016-08-26T14:47:06.000+02:00
Application load balancer
diff --git a/senza/components/auto_scaling_group.py b/senza/components/auto_scaling_group.py
@@ -150,6 +150,15 @@ def component_auto_scaling_group(definition, configuration, args, info, force, a
             asg_properties["LoadBalancerNames"] = [{'Ref': ref} for ref in configuration["ElasticLoadBalancer"]]
         # use ELB health check by default
         default_health_check_type = 'ELB'
+    elif "ElasticLoadBalancerV2" in configuration:
+        if isinstance(configuration["ElasticLoadBalancerV2"], str):
+            asg_properties["TargetGroupARNs"] = [{"Ref": configuration["ElasticLoadBalancerV2"] + 'TargetGroup'}]
+        elif isinstance(configuration["ElasticLoadBalancerV2"], list):
+            asg_properties["TargetGroupARNs"] = [
+                {'Ref': ref} for ref in configuration["ElasticLoadBalancerV2"] + 'TargetGroup'
+            ]
+        # use ELB health check by default
+        default_health_check_type = 'ELB'
 
     asg_properties['HealthCheckType'] = configuration.get('HealthCheckType', default_health_check_type)
     asg_properties['HealthCheckGracePeriod'] = configuration.get('HealthCheckGracePeriod', 300)
diff --git a/senza/components/elastic_load_balancer_v2.py b/senza/components/elastic_load_balancer_v2.py
@@ -0,0 +1,217 @@
+import click
+from clickclick import fatal_error
+from senza.aws import resolve_security_groups
+from senza.components.elastic_load_balancer import get_load_balancer_name
+
+from ..cli import AccountArguments, TemplateArguments
+from ..manaus import ClientError
+from ..manaus.acm import ACM, ACMCertificate
+from ..manaus.iam import IAM, IAMServerCertificate
+from ..manaus.route53 import convert_domain_records_to_alias
+
+SENZA_PROPERTIES = frozenset(['Domains', 'HealthCheckPath', 'HealthCheckPort', 'HealthCheckProtocol',
+                              'HTTPPort', 'Name', 'SecurityGroups', 'SSLCertificateId', 'Type'])
+
+
+def get_listeners(lb_name, target_group_name, subdomain, main_zone, configuration,
+                  account_info: AccountArguments):
+    ssl_cert = configuration.get('SSLCertificateId')
+
+    if ACMCertificate.arn_is_acm_certificate(ssl_cert):
+        # check if certificate really exists
+        try:
+            ACMCertificate.get_by_arn(account_info.Region, ssl_cert)
+        except ClientError as e:
+            error_msg = e.response['Error']['Message']
+            fatal_error(error_msg)
+    elif IAMServerCertificate.arn_is_server_certificate(ssl_cert):
+        # TODO check if certificate exists
+        pass
+    elif ssl_cert is not None:
+        certificate = IAMServerCertificate.get_by_name(account_info.Region,
+                                                       ssl_cert)
+        ssl_cert = certificate.arn
+    elif main_zone is not None:
+        if main_zone:
+            iam_pattern = main_zone.lower().rstrip('.').replace('.', '-')
+            name = '{sub}.{zone}'.format(sub=subdomain,
+                                         zone=main_zone.rstrip('.'))
+            acm = ACM(account_info.Region)
+            acm_certificates = sorted(acm.get_certificates(domain_name=name),
+                                      reverse=True)
+        else:
+            iam_pattern = ''
+            acm_certificates = []
+        iam = IAM(account_info.Region)
+        iam_certificates = sorted(iam.get_certificates(name=iam_pattern))
+        if not iam_certificates:
+            # if there are no iam certificates matching the pattern
+            # try to use any certificate
+            iam_certificates = sorted(iam.get_certificates(), reverse=True)
+
+        # the priority is acm_certificate first and iam_certificate second
+        certificates = (acm_certificates +
+                        iam_certificates)  # type: List[Union[ACMCertificate, IAMServerCertificate]]
+        try:
+            certificate = certificates[0]
+            ssl_cert = certificate.arn
+        except IndexError:
+            if main_zone:
+                fatal_error('Could not find any matching '
+                            'SSL certificate for "{}"'.format(name))
+            else:
+                fatal_error('Could not find any SSL certificate')
+    return [{
+        'Type': 'AWS::ElasticLoadBalancingV2::Listener',
+        'Properties': {
+            "Certificates": [{'CertificateArn': ssl_cert}],
+            "Protocol": "HTTPS",
+            "DefaultActions": [{'Type': 'forward', 'TargetGroupArn': {'Ref': target_group_name}}],
+            'LoadBalancerArn': {'Ref': lb_name},
+            "Port": 443
+        }
+    }]
+
+
+def component_elastic_load_balancer_v2(definition,
+                                       configuration: dict,
+                                       args: TemplateArguments,
+                                       info: dict,
+                                       force,
+                                       account_info: AccountArguments):
+    lb_name = configuration["Name"]
+    # domains pointing to the load balancer
+    subdomain = ''
+    main_zone = None
+    for name, domain in configuration.get('Domains', {}).items():
+        name = '{}{}'.format(lb_name, name)
+
+        domain_name = "{0}.{1}".format(domain["Subdomain"], domain["Zone"])
+
+        convert_domain_records_to_alias(domain_name)
+
+        properties = {"Type": "A",
+                      "Name": domain_name,
+                      "HostedZoneName": domain["Zone"],
+                      "AliasTarget": {"HostedZoneId": {"Fn::GetAtt": [lb_name,
+                                                                      "CanonicalHostedZoneID"]},
+                                      "DNSName": {"Fn::GetAtt": [lb_name, "DNSName"]}}}
+        definition["Resources"][name] = {"Type": "AWS::Route53::RecordSet",
+                                         "Properties": properties}
+
+        if domain["Type"] == "weighted":
+            definition["Resources"][name]["Properties"]['Weight'] = 0
+            definition["Resources"][name]["Properties"]['SetIdentifier'] = "{0}-{1}".format(info["StackName"],
+                                                                                            info["StackVersion"])
+            subdomain = domain['Subdomain']
+            main_zone = domain['Zone']  # type: str
+
+    target_group_name = lb_name + 'TargetGroup'
+    listeners = configuration.get('Listeners') or get_listeners(
+        lb_name, target_group_name, subdomain, main_zone, configuration, account_info)
+
+    health_check_protocol = "HTTP"
+    allowed_health_check_protocols = ("HTTP", "TCP", "UDP", "SSL")
+    if "HealthCheckProtocol" in configuration:
+        health_check_protocol = configuration["HealthCheckProtocol"]
+
+    if health_check_protocol not in allowed_health_check_protocols:
+        raise click.UsageError('Protocol "{}" is not supported for LoadBalancer'.format(health_check_protocol))
+
+    health_check_path = "/ui/"
+    if "HealthCheckPath" in configuration:
+        health_check_path = configuration["HealthCheckPath"]
+
+    health_check_port = configuration["HTTPPort"]
+    if "HealthCheckPort" in configuration:
+        health_check_port = configuration["HealthCheckPort"]
+
+    if configuration.get('NameSuffix'):
+        version = '{}-{}'.format(info["StackVersion"],
+                                 configuration['NameSuffix'])
+        loadbalancer_name = get_load_balancer_name(info["StackName"], version)
+        del(configuration['NameSuffix'])
+    else:
+        loadbalancer_name = get_load_balancer_name(info["StackName"],
+                                                   info["StackVersion"])
+
+    loadbalancer_scheme = "internal"
+    allowed_loadbalancer_schemes = ("internet-facing", "internal")
+    if "Scheme" in configuration:
+        loadbalancer_scheme = configuration["Scheme"]
+    else:
+        configuration["Scheme"] = loadbalancer_scheme
+
+    if loadbalancer_scheme == 'internet-facing':
+        click.secho('You are deploying an internet-facing ELB that will be '
+                    'publicly accessible! You should have OAUTH2 and HTTPS '
+                    'in place!', bold=True, err=True)
+
+    if loadbalancer_scheme not in allowed_loadbalancer_schemes:
+        raise click.UsageError('Scheme "{}" is not supported for LoadBalancer'.format(loadbalancer_scheme))
+
+    if loadbalancer_scheme == "internal":
+        loadbalancer_subnet_map = "LoadBalancerInternalSubnets"
+    else:
+        loadbalancer_subnet_map = "LoadBalancerSubnets"
+
+    tags = [
+        # Tag "Name"
+        {
+            "Key": "Name",
+            "Value": "{0}-{1}".format(info["StackName"], info["StackVersion"])
+        },
+        # Tag "StackName"
+        {
+            "Key": "StackName",
+            "Value": info["StackName"],
+        },
+        # Tag "StackVersion"
+        {
+            "Key": "StackVersion",
+            "Value": info["StackVersion"]
+        }
+    ]
+
+    # load balancer
+    definition["Resources"][lb_name] = {
+        "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer",
+        "Properties": {
+            'Name': loadbalancer_name,
+            'Scheme': loadbalancer_scheme,
+            'SecurityGroups': resolve_security_groups(configuration["SecurityGroups"], args.region),
+            'Subnets': {"Fn::FindInMap": [loadbalancer_subnet_map, {"Ref": "AWS::Region"}, "Subnets"]},
+            "Tags": tags
+        }
+    }
+    definition["Resources"][target_group_name] = {
+        'Type': 'AWS::ElasticLoadBalancingV2::TargetGroup',
+        'Properties': {
+            'Name': loadbalancer_name,
+            'HealthCheckIntervalSeconds': '10',
+            'HealthCheckPath': health_check_path,
+            'HealthCheckPort': health_check_port,
+            'HealthCheckProtocol': health_check_protocol,
+            'HealthCheckTimeoutSeconds': '5',
+            'HealthyThresholdCount': '2',
+            'Port': configuration['HTTPPort'],
+            'Protocol': 'HTTP',
+            'UnhealthyThresholdCount': '2',
+            'VpcId': account_info.VpcID,  # TODO: support multiple VPCs
+            'Tags': tags,
+            'TargetGroupAttributes': [{'Key': 'deregistration_delay.timeout_seconds', 'Value': '60'}]
+        }
+    }
+    for i, listener in enumerate(listeners):
+        if i == 0:
+            suffix = ''
+        else:
+            suffix = str(i + 1)
+        definition['Resources'][lb_name + 'Listener' + suffix] = listener
+    for key, val in configuration.items():
+        # overwrite any specified properties, but
+        # ignore our special Senza properties as they are not supported by CF
+        if key not in SENZA_PROPERTIES:
+            definition['Resources'][lb_name]['Properties'][key] = val
+
+    return definition
diff --git a/senza/components/weighted_dns_elastic_load_balancer_v2.py b/senza/components/weighted_dns_elastic_load_balancer_v2.py
@@ -0,0 +1,29 @@
+
+from senza.components.elastic_load_balancer_v2 import component_elastic_load_balancer_v2
+
+
+def component_weighted_dns_elastic_load_balancer_v2(definition, configuration, args, info, force, account_info):
+    if 'Domains' not in configuration:
+        if 'MainDomain' in configuration:
+            main_domain = configuration['MainDomain']
+            main_subdomain, main_zone = account_info.split_domain(main_domain)
+            del configuration['MainDomain']
+        else:
+            main_zone = account_info.Domain
+            main_subdomain = info['StackName']
+
+        if 'VersionDomain' in configuration:
+            version_domain = configuration['VersionDomain']
+            version_subdomain, version_zone = account_info.split_domain(version_domain)
+            del configuration['VersionDomain']
+        else:
+            version_zone = account_info.Domain
+            version_subdomain = '{}-{}'.format(info['StackName'], info['StackVersion'])
+
+        configuration['Domains'] = {'MainDomain': {'Type': 'weighted',
+                                                   'Zone': '{}.'.format(main_zone.rstrip('.')),
+                                                   'Subdomain': main_subdomain},
+                                    'VersionDomain': {'Type': 'standalone',
+                                                      'Zone': '{}.'.format(version_zone.rstrip('.')),
+                                                      'Subdomain': version_subdomain}}
+    return component_elastic_load_balancer_v2(definition, configuration, args, info, force, account_info)
diff --git a/tests/test_components.py b/tests/test_components.py
@@ -23,6 +23,8 @@
                                                          generate_user_data)
 from senza.components.weighted_dns_elastic_load_balancer import \
     component_weighted_dns_elastic_load_balancer
+from senza.components.weighted_dns_elastic_load_balancer_v2 import \
+    component_weighted_dns_elastic_load_balancer_v2
 
 from fixtures import HOSTED_ZONE_ZO_NE_COM, HOSTED_ZONE_ZO_NE_DEV, boto_resource
 
@@ -546,7 +548,7 @@ def test_component_auto_scaling_group_custom_tags():
         'Name': 'Foo',
         'InstanceType': 't2.micro',
         'Image': 'foo',
-        'Tags': [ 
+        'Tags': [
             { 'Key': 'Tag1', 'Value': 'alpha' },
             { 'Key': 'Tag2', 'Value': 'beta' }
         ]
@@ -867,3 +869,55 @@ def test_get_load_balancer_name():
 
     get_load_balancer_name('toolong123456789012345678901234567890',
                            '1') == 'toolong12345678901234567890123-1'
+
+
+def test_weighted_dns_load_balancer_v2(monkeypatch, boto_resource):
+    senza.traffic.DNS_ZONE_CACHE = {}
+
+    def my_client(rtype, *args):
+        if rtype == 'route53':
+            route53 = MagicMock()
+            route53.list_hosted_zones.return_value = {'HostedZones': [HOSTED_ZONE_ZO_NE_COM],
+                                                      'IsTruncated': False,
+                                                      'MaxItems': '100'}
+            return route53
+        return MagicMock()
+
+    monkeypatch.setattr('boto3.client', my_client)
+
+    configuration = {
+        "Name": "MyLB",
+        "SecurityGroups": "",
+        "HTTPPort": "9999",
+        'MainDomain': 'great.api.zo.ne.com',
+        'VersionDomain': 'version.api.zo.ne.com'
+    }
+    info = {'StackName': 'foobar', 'StackVersion': '0.1'}
+    definition = {"Resources": {}}
+
+    args = MagicMock()
+    args.region = "foo"
+
+    mock_string_result = MagicMock()
+    mock_string_result.return_value = "foo"
+    monkeypatch.setattr('senza.components.elastic_load_balancer_v2.resolve_security_groups', mock_string_result)
+
+    m_acm = MagicMock()
+    m_acm_certificate = MagicMock()
+    m_acm_certificate.arn = "foo"
+    m_acm.get_certificates.return_value = iter([m_acm_certificate])
+    monkeypatch.setattr('senza.components.elastic_load_balancer_v2.ACM', m_acm)
+
+    result = component_weighted_dns_elastic_load_balancer_v2(definition,
+                                                             configuration,
+                                                             args,
+                                                             info,
+                                                             False,
+                                                             AccountArguments('dummyregion'))
+
+    assert 'MyLB' in result["Resources"]
+    assert 'MyLBListener' in result["Resources"]
+    assert 'MyLBTargetGroup' in result["Resources"]
+
+    assert result['Resources']['MyLBTargetGroup']['Properties']['HealthCheckPort'] == '9999'
+    assert result['Resources']['MyLBListener']['Properties']['Certificates'] == [{'CertificateArn': 'arn:aws:123'}]
