Merge pull request #580 from zalando-stups/ci/fix-pipeline

Fix testing pipeline

Author: hughcapet

delivery.yaml

@@ -6,7 +6,9 @@ notifications:
 pipeline:
   - id: build
     type: script
-    overlay: ci/python
+    vm_config:
+      type: linux
+      image: cdp-runtime/python-3.9
     commands:
     - desc: "Install dependencies"
       cmd: pip install -r requirements.txt

senza/arguments.py

@@ -6,22 +6,50 @@
 # pylint: disable=locally-disabled, invalid-name
 import boto3.session
 import click
+import configparser
+import os
 
 from .error_handling import HandleExceptions
 
 
+def get_region(region):
+    """
+    Ensure region value.
+    If region is not provided, get one from the config file.
+    Raise error if still no region is set at the end.
+    """
+    if not region:
+        config = configparser.ConfigParser()
+        try:
+            config.read(os.path.expanduser("~/.aws/config"))
+            if "default" in config:
+                region = config["default"]["region"]
+        except Exception:
+            pass
+
+    if not region:
+        raise click.UsageError(
+            "Please specify the AWS region on the "
+            "command line (--region) or in ~/.aws/config"
+        )
+
+    return region
+
+
 def validate_region(ctx, param, value):  # pylint: disable=locally-disabled, unused-argument
-    """Validate Click region param parameter."""
-
-    if value is not None:
-        session = boto3.session.Session()
-        valid_regions = session.get_available_regions('cloudformation')
-        if value not in valid_regions:
-            valid_regions.sort()
-            raise click.BadParameter("'{}'. Region must be one of the "
-                                     "following AWS regions:\n"
-                                     "  - {}".format(value,
-                                                     "\n  - ".join(valid_regions)))
+    """Validate Click region parameter."""
+
+    value = get_region(value)  # ensure region is set
+
+    session = boto3.session.Session()
+    valid_regions = session.get_available_regions('cloudformation')
+    if value not in valid_regions:
+        valid_regions.sort()
+        raise click.BadParameter("'{}'. Region must be one of the "
+                                 "following AWS regions:\n"
+                                 "  - {}".format(value,
+                                                 "\n  - ".join(valid_regions)))
+
     return value
 
 
@@ -74,5 +102,3 @@ def set_stacktrace_visible(ctx, param, value):  # pylint: disable=locally-disabl
                             metavar='NAME',
                             multiple=True,
                             help='Specify field to be returned')
-
-GLOBAL_OPTIONS = {}

senza/cli.py

@@ -2,7 +2,6 @@
 import base64
 import calendar
 import collections
-import configparser
 import datetime
 import ipaddress
 import json
@@ -37,7 +36,6 @@
 from senza.components.elastigroup import ELASTIGROUP_RESOURCE_TYPE
 from .spotinst.components import elastigroup_api
 from .arguments import (
-    GLOBAL_OPTIONS,
     json_output_option,
     output_option,
     parameter_file_option,
@@ -46,6 +44,7 @@
     watch_option,
     watchrefresh_option,
     field_option,
+    get_region
 )
 from .aws import (
     StackReference,
@@ -432,27 +431,6 @@ def read_parameter_file(parameter_file):
     return tuple(paras)
 
 
-def get_region(region):
-    if not region:
-        region = GLOBAL_OPTIONS.get("region")
-    if not region:
-        config = configparser.ConfigParser()
-        try:
-            config.read(os.path.expanduser("~/.aws/config"))
-            if "default" in config:
-                region = config["default"]["region"]
-        except Exception:
-            pass
-
-    if not region:
-        raise click.UsageError(
-            "Please specify the AWS region on the "
-            "command line (--region) or in ~/.aws/config"
-        )
-
-    return region
-
-
 def check_credentials(region):
     iam = BotoClientProxy("iam")
     return iam.list_account_aliases()
@@ -523,7 +501,6 @@ def all_with_version(stack_refs: list):
 def list_stacks(region, stack_ref, all, output, field, w, watch):
     """List Cloud Formation stacks"""
 
-    region = get_region(region)
     check_credentials(region)
 
     stack_refs = get_stack_refs(stack_ref)
@@ -652,7 +629,6 @@ def get_classic_load_balancer_metrics(cloudwatch, lb_name, start, now):
 def health(region, stack_ref, all, output, field, w, watch):
     """Show stack health (ELB req/s, ..)"""
 
-    region = get_region(region)
     check_credentials(region)
 
     cloudwatch = BotoClientProxy("cloudwatch", region)
@@ -753,7 +729,6 @@ def create(
 ):
     """Create a new Cloud Formation stack from the given Senza definition file"""
 
-    region = get_region(region)
     data = create_cf_template(
         definition, region, version, parameter, force, parameter_file
     )
@@ -825,7 +800,6 @@ def update(
     """Update an existing Cloud Formation stack from the given Senza
     definition file"""
 
-    region = get_region(region)
     data = create_cf_template(
         definition, region, version, parameter, force, parameter_file
     )
@@ -844,7 +818,6 @@ def update(
 def print_cfjson(definition, region, version, parameter, output, force, parameter_file):
     """Print the generated Cloud Formation template"""
 
-    region = get_region(region)
     data = create_cf_template(
         definition, region, version, parameter, force, parameter_file, pretty=True
     )
@@ -953,7 +926,6 @@ def delete(stack_ref, region, dry_run, force, interactive, ignore_non_existent):
     """Delete a single Cloud Formation stack"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     if not stack_refs:
@@ -1015,7 +987,6 @@ def resources(stack_ref, region, w, watch, output, field):
     """Show all resources of a single Cloud Formation stack"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
     cf = BotoClientProxy("cloudformation", region)
 
@@ -1063,7 +1034,6 @@ def events(stack_ref, region, w, watch, output, field):
     """Show all Cloud Formation events for a single stack"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
     cf = BotoClientProxy("cloudformation", region)
 
@@ -1120,7 +1090,6 @@ def events(stack_ref, region, w, watch, output, field):
 def init(definition_file, region, template, user_variable):
     """Initialize a new Senza definition"""
 
-    region = get_region(region)
     check_credentials(region)
     account_info = AccountArguments(region=region)
 
@@ -1263,7 +1232,6 @@ def instances(
     """List the stack's EC2 instances"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     ec2 = boto3.resource("ec2", region)
@@ -1358,7 +1326,6 @@ def status(stack_ref, region, output, field, w, watch):
     """Show stack status information"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     ec2 = boto3.resource("ec2", region)
@@ -1451,7 +1418,6 @@ def domains(stack_ref, region, output, field, w, watch):
     """List the stack's Route53 domains"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     cf = boto3.resource("cloudformation", region)
@@ -1560,7 +1526,6 @@ def traffic(stack_name, stack_version, percentage, region, output, timeout, inte
     """Route traffic to a specific stack (weighted DNS record)"""
 
     stack_refs = get_stack_refs([stack_name, stack_version])
-    region = get_region(region)
     check_credentials(region)
 
     with OutputFormat(output):
@@ -1593,7 +1558,6 @@ def images(stack_ref, region, output, field, hide_older_than, show_instances):
     """Show all used AMIs and available Taupage AMIs"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     ec2 = boto3.resource("ec2", region)
@@ -1730,7 +1694,6 @@ def console(instance_or_stack_ref, limit, region, w, watch):
         # filter out instances not part of any stack
         filters = [{"Name": "tag-key", "Values": ["aws:cloudformation:stack-name"]}]
 
-    region = get_region(region)
     check_credentials(region)
 
     ec2 = boto3.resource("ec2", region)
@@ -1784,7 +1747,6 @@ def dump(stack_ref, region, output):
     """Dump Cloud Formation template of existing stack"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     cf = BotoClientProxy("cloudformation", region)
@@ -1848,7 +1810,6 @@ def patch(stack_ref, region, image, instance_type, user_data):
     Currently supports patching ASG launch configurations and ElastiGroup groups."""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     if image in taupage.CHANNELS:
@@ -1948,7 +1909,6 @@ def respawn_instances(stack_ref, inplace, force, batch_size_percentage, batch_pe
     Performs a rolling update to prevent downtimes."""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     stacks = get_stacks(stack_refs, region)
@@ -1980,7 +1940,6 @@ def scale(stack_ref, region, desired_capacity, min_size, force):
     """Scale Auto Scaling Group to desired capacity"""
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     check_credentials(region)
 
     stacks = get_stacks(stack_refs, region)
@@ -2120,7 +2079,6 @@ def wait(stack_ref, region, deletion, timeout, interval):
     """
 
     stack_refs = get_stack_refs(stack_ref)
-    region = get_region(region)
     cf = BotoClientProxy("cloudformation", region)
 
     target_status = (

senza/components/elastic_load_balancer_v2.py

@@ -88,7 +88,7 @@ def component_elastic_load_balancer_v2(definition,
         version = '{}-{}'.format(info["StackVersion"],
                                  configuration['NameSuffix'])
         loadbalancer_name = get_load_balancer_name(info["StackName"], version)
-        del(configuration['NameSuffix'])
+        del (configuration['NameSuffix'])
     else:
         loadbalancer_name = get_load_balancer_name(info["StackName"],
                                                    info["StackVersion"])

senza/subcommands/root.py

@@ -13,7 +13,6 @@
 import senza
 from clickclick import AliasedGroup, warning
 
-from ..arguments import GLOBAL_OPTIONS, region_option
 from ..error_handling import sentry
 
 CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
@@ -132,16 +131,13 @@ def print_version(ctx, param, value):
     is_eager=True,
     help="Print the current version number and exit.",
 )
-@region_option
-def cli(region):
+def cli():
     """
     Senza's root command.
 
-    It checks the version and sets the region global option before executing
-    the sub-commands.
+    Checks the version.
 
     Sub command can be added by using `cli.add_command(SUB_COMMAND_FUNCTION)`
     or using the `@cli.command()` decorator
     """
     check_senza_version(senza.__version__)
-    GLOBAL_OPTIONS["region"] = region

senza/templates/postgresapp.py

@@ -438,7 +438,7 @@ def gather_user_variables(variables, region, account_info):
         variables['use_ebs'] = True
 
     if variables['use_ebs']:
-        prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=defaults['volume_size'])
+        prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=str(defaults['volume_size']))
         prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)',
                default=defaults['volume_type'])
         if variables['volume_type'] == 'io1':

setup.py

@@ -7,7 +7,7 @@
 
 import setuptools
 from setuptools.command.test import test as TestCommand
-from setuptools import setup
+from setuptools import setup, Command
 
 if sys.version_info < (3, 4, 0):
     sys.stderr.write('FATAL: STUPS Senza needs to be run with Python 3.4+\n')
@@ -91,6 +91,23 @@ def run_tests(self):
         sys.exit(errno)
 
 
+class Flake8(Command):
+
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        from flake8.main.cli import main
+
+        errno = main(['senza'])
+        sys.exit(errno)
+
+
 def get_install_requirements(path):
     content = open(os.path.join(__location__, path)).read()
     return [req for req in content.split('\\n') if req != '']
@@ -104,6 +121,7 @@ def setup_package():
     # Assemble additional setup commands
     cmdclass = {}
     cmdclass['test'] = PyTest
+    cmdclass['flake8'] = Flake8
 
     install_reqs = get_install_requirements('requirements.txt')
 

tests/test_cli.py

@@ -564,6 +564,16 @@ def my_resource(rtype, *args):
     monkeypatch.setattr('boto3.resource', my_resource)
     monkeypatch.setattr('senza.cli.AccountArguments', MagicMock())
 
+    kms_key = {'KeyId': 'key_a',
+               'aliases': ['something'],
+               'Description': 'This is key a',
+               'Arn': 'arn:partition:service:region:account-id:resource-id'}
+    monkeypatch.setattr('senza.templates.postgresapp.list_kms_keys',
+                        MagicMock(return_value=[kms_key]))
+    monkeypatch.setattr('senza.templates.postgresapp.choice',
+                        MagicMock(return_value='{}: {}'.format(kms_key['KeyId'], kms_key['Description'])))
+    monkeypatch.setattr('senza.templates.postgresapp.encrypt', MagicMock(return_value='encrypted_string'))
+
     runner = CliRunner()
 
     with runner.isolated_filesystem():