Merge pull request #112 from philpep/ansible-vault

Various ansible fixes

Author: philpep

testinfra/backend/ansible.py

@@ -18,7 +18,6 @@
 import pprint
 
 from testinfra.backend import base
-import testinfra.utils.ansible_runner as ansible_runner
 
 logger = logging.getLogger("testinfra")
 
@@ -30,8 +29,16 @@ class AnsibleBackend(base.BaseBackend):
     def __init__(self, host, ansible_inventory=None, *args, **kwargs):
         self.host = host
         self.ansible_inventory = ansible_inventory
+        self._ansible_runner = None
         super(AnsibleBackend, self).__init__(host, *args, **kwargs)
 
+    @property
+    def ansible_runner(self):
+        if self._ansible_runner is None:
+            from testinfra.utils.ansible_runner import AnsibleRunner
+            self._ansible_runner = AnsibleRunner(self.ansible_inventory)
+        return self._ansible_runner
+
     def run(self, command, *args):
         command = self.get_command(command, *args)
         out = self.run_ansible("shell", module_args=command)
@@ -53,9 +60,8 @@ def run(self, command, *args):
         return result
 
     def run_ansible(self, module_name, module_args=None, **kwargs):
-        result = ansible_runner.run(
+        result = self.ansible_runner.run(
             self.host, module_name, module_args,
-            host_list=self.ansible_inventory,
             **kwargs)
         logger.info(
             "RUN Ansible(%s, %s, %s): %s",
@@ -64,9 +70,9 @@ def run_ansible(self, module_name, module_args=None, **kwargs):
         return result
 
     def get_variables(self):
-        return ansible_runner.get_variables(
-            self.host, host_list=self.ansible_inventory)
+        return self.ansible_runner.get_variables(self.host)
 
     @classmethod
     def get_hosts(cls, host, **kwargs):
-        return ansible_runner.get_hosts(kwargs.get("ansible_inventory"), host)
+        from testinfra.utils.ansible_runner import AnsibleRunner
+        return AnsibleRunner(kwargs.get("ansible_inventory")).get_hosts(host)

testinfra/test/conftest.py

@@ -49,25 +49,57 @@ def has_docker():
     return _HAS_DOCKER
 
 
-def get_ansible_inventory(name, hostname, user, port, key):
+# Generated with
+# $ echo myhostvar: bar > hostvars.yml
+# $ echo polichinelle > vault-pass.txt
+# $ ansible-vault encrypt --vault-password-file vault-pass.txt hostvars.yml
+# $ cat hostvars.yml
+ANSIBLE_HOSTVARS = """$ANSIBLE_VAULT;1.1;AES256
+39396233323131393835363638373764336364323036313434306134636633353932623363646233
+6436653132383662623364313438376662666135346266370a343934663431363661393363386633
+64656261336662623036373036363535313964313538366533313334366363613435303066316639
+3235393661656230350a326264356530326432393832353064363439393330616634633761393838
+3261
+"""
+
+
+def setup_ansible_config(tmpdir, name, host, user, port, key):
     ansible_major_version = int(ansible.__version__.split(".", 1)[0])
     items = [
         name,
         "ansible_ssh_private_key_file={}".format(key),
+        "myvar=foo",
     ]
     if ansible_major_version == 1:
         items.extend([
-            "ansible_ssh_host={}".format(hostname),
+            "ansible_ssh_host={}".format(host),
             "ansible_ssh_user={}".format(user),
             "ansible_ssh_port={}".format(port),
         ])
     elif ansible_major_version == 2:
         items.extend([
-            "ansible_host={}".format(hostname),
+            "ansible_host={}".format(host),
             "ansible_user={}".format(user),
             "ansible_port={}".format(port),
         ])
-    return " ".join(items) + "\n"
+    tmpdir.join("inventory").write(
+        "[testgroup]\n" + " ".join(items) + "\n")
+    tmpdir.mkdir("host_vars").join(name).write(ANSIBLE_HOSTVARS)
+    tmpdir.mkdir("group_vars").join("testgroup").write((
+        "---\n"
+        "myhostvar: should_be_overriden\n"
+        "mygroupvar: qux\n"
+    ))
+    vault_password_file = tmpdir.join("vault-pass.txt")
+    vault_password_file.write("polichinelle\n")
+    ansible_cfg = tmpdir.join("ansible.cfg")
+    ansible_cfg.write((
+        "[defaults]\n"
+        "vault_password_file={}\n"
+        "host_key_checking=False\n\n"
+        "[ssh_connection]\n"
+        "pipelining=True\n"
+    ).format(str(vault_password_file)))
 
 
 def build_docker_container_fixture(image, scope):
@@ -138,10 +170,11 @@ def TestinfraBackend(request, tmpdir_factory):
             if ansible is None:
                 pytest.skip()
                 return
-            inventory = tmpdir.join("inventory")
-            inventory.write(get_ansible_inventory(
-                host, docker_host, user or "root", port, str(key)))
-            kw["ansible_inventory"] = str(inventory)
+            setup_ansible_config(
+                tmpdir, host, docker_host, user or "root", port, str(key))
+            os.environ["ANSIBLE_CONFIG"] = str(tmpdir.join("ansible.cfg"))
+            # this force backend cache reloading
+            kw["ansible_inventory"] = str(tmpdir.join("inventory"))
         else:
             ssh_config = tmpdir.join("ssh_config")
             ssh_config.write((

testinfra/test/test_backends.py

@@ -66,3 +66,15 @@ def test_user_connection(User):
 @pytest.mark.testinfra_hosts(*SUDO_HOSTS)
 def test_sudo(User):
     assert User().name == "root"
+
+
+@pytest.mark.testinfra_hosts("ansible://debian_jessie")
+def test_ansible_hosts_expand(TestinfraBackend):
+    from testinfra.backend.ansible import AnsibleBackend
+
+    def get_hosts(spec):
+        return AnsibleBackend.get_hosts(
+            spec, ansible_inventory=TestinfraBackend.ansible_inventory)
+    assert get_hosts(["all"]) == ["debian_jessie"]
+    assert get_hosts(["testgroup"]) == ["debian_jessie"]
+    assert get_hosts(["*ia*jess*"]) == ["debian_jessie"]

testinfra/test/test_modules.py

@@ -263,7 +263,7 @@ def test_ansible_module(TestinfraBackend, Ansible):
     version = int(ansible.__version__.split(".", 1)[0])
     setup = Ansible("setup")["ansible_facts"]
     assert setup["ansible_lsb"]["codename"] == "jessie"
-    passwd = Ansible("file", "path=/etc/passwd")
+    passwd = Ansible("file", "path=/etc/passwd state=file")
     assert passwd["changed"] is False
     assert passwd["gid"] == 0
     assert passwd["group"] == "root"
@@ -275,8 +275,11 @@ def test_ansible_module(TestinfraBackend, Ansible):
     assert passwd["uid"] == 0
 
     variables = Ansible.get_variables()
+    assert variables["myvar"] == "foo"
+    assert variables["myhostvar"] == "bar"
+    assert variables["mygroupvar"] == "qux"
     assert variables["inventory_hostname"] == "debian_jessie"
-    assert variables["group_names"] == ["ungrouped"]
+    assert variables["group_names"] == ["testgroup"]
 
     # test errors reporting
     with pytest.raises(Ansible.AnsibleException) as excinfo:

testinfra/utils/ansible_runner.py

@@ -25,18 +25,28 @@
 else:
     _has_ansible = True
     _ansible_major_version = int(ansible.__version__.split(".", 1)[0])
+    import ansible.constants
     if _ansible_major_version == 1:
         import ansible.inventory
         import ansible.runner
+        import ansible.utils
     elif _ansible_major_version == 2:
+        import ansible.cli
         import ansible.executor.task_queue_manager
         import ansible.inventory
         import ansible.parsing.dataloader
         import ansible.playbook.play
         import ansible.plugins.callback
+        import ansible.utils.vars
         import ansible.vars
 
 
+def _reload_constants():
+    # Reload defaults that can depend on environment variables and
+    # current working directory
+    reload(ansible.constants)
+
+
 class AnsibleRunnerBase(object):
 
     def __init__(self, host_list=None):
@@ -71,7 +81,13 @@ class AnsibleRunnerV1(AnsibleRunnerBase):
 
     def __init__(self, host_list=None):
         super(AnsibleRunnerV1, self).__init__(host_list)
-        self.inventory = ansible.inventory.Inventory(self.host_list)
+        _reload_constants()
+        self.vault_pass = ansible.utils.read_vault_file(
+            ansible.constants.DEFAULT_VAULT_PASSWORD_FILE)
+        kwargs = {"vault_password": self.vault_pass}
+        if self.host_list is not None:
+            kwargs["host_list"] = host_list
+        self.inventory = ansible.inventory.Inventory(**kwargs)
 
     def get_hosts(self, pattern=None):
         return [
@@ -91,6 +107,8 @@ def run(self, host, module_name, module_args=None, **kwargs):
         result = ansible.runner.Runner(
             pattern=host,
             module_name=module_name,
+            vault_pass=self.vault_pass,
+            inventory=self.inventory,
             **kwargs).run()
         if host not in result["contacted"]:
             raise RuntimeError("Unexpected error: {}".format(result))
@@ -100,21 +118,6 @@ def run(self, host, module_name, module_args=None, **kwargs):
         return result["contacted"][host]
 
 
-class Options(object):
-
-    def __init__(self, **kwargs):
-        self.connection = "smart"
-        for attr in (
-            "module_path", "forks", "remote_user", "private_key_file",
-            "ssh_common_args", "ssh_extra_args", "sftp_extra_args",
-            "scp_extra_args", "become", "become_method", "become_user",
-            "verbosity",
-        ):
-            setattr(self, attr, None)
-        self.check = kwargs.get("check", False)
-        super(Options, self).__init__()
-
-
 if _has_ansible and _ansible_major_version == 2:
     class Callback(ansible.plugins.callback.CallbackBase):
 
@@ -147,12 +150,31 @@ class AnsibleRunnerV2(AnsibleRunnerBase):
 
     def __init__(self, host_list=None):
         super(AnsibleRunnerV2, self).__init__(host_list)
+        _reload_constants()
         self.variable_manager = ansible.vars.VariableManager()
+        self.options = ansible.cli.CLI(None).base_parser(
+            connect_opts=True,
+            meta_opts=True,
+            runas_opts=True,
+            subset_opts=True,
+            check_opts=True,
+            inventory_opts=True,
+            runtask_opts=True,
+            vault_opts=True,
+            fork_opts=True,
+            module_opts=True,
+        ).parse_args([])[0]
+        self.options.connection = "smart"
         self.loader = ansible.parsing.dataloader.DataLoader()
+        if self.options.vault_password_file:
+            vault_pass = ansible.cli.CLI.read_vault_password_file(
+                self.options.vault_password_file, loader=self.loader)
+            self.loader.set_vault_password(vault_pass)
+
         self.inventory = ansible.inventory.Inventory(
             loader=self.loader,
             variable_manager=self.variable_manager,
-            host_list=host_list,
+            host_list=host_list or self.options.inventory,
         )
         self.variable_manager.set_inventory(self.inventory)
 
@@ -163,9 +185,12 @@ def get_hosts(self, pattern=None):
         ]
 
     def get_variables(self, host):
-        return self.inventory.get_vars(host)
+        host = self.inventory.get_host(host)
+        return ansible.utils.vars.combine_vars(
+            host.get_group_vars(), host.get_vars())
 
     def run(self, host, module_name, module_args=None, **kwargs):
+        self.options.check = kwargs.get("check", False)
         action = {"module": module_name}
         if module_args is not None:
             if module_name in ("command", "shell"):
@@ -180,14 +205,13 @@ def run(self, host, module_name, module_args=None, **kwargs):
             }],
         }, variable_manager=self.variable_manager, loader=self.loader)
         tqm = None
-        options = Options(**kwargs)
         callback = Callback()
         try:
             tqm = ansible.executor.task_queue_manager.TaskQueueManager(
                 inventory=self.inventory,
                 variable_manager=self.variable_manager,
                 loader=self.loader,
-                options=options,
+                options=self.options,
                 passwords=None,
                 stdout_callback=callback,
             )
@@ -208,16 +232,3 @@ def run(self, host, module_name, module_args=None, **kwargs):
     raise NotImplementedError(
         "Unhandled ansible version " + ansible.__version__
     )
-
-
-def get_hosts(host_list=None, pattern=None):
-    return AnsibleRunner(host_list).get_hosts(pattern)
-
-
-def run(host, module_name, module_args=None, host_list=None, **kwargs):
-    return AnsibleRunner(host_list).run(
-        host, module_name, module_args, **kwargs)
-
-
-def get_variables(host, host_list=None):
-    return AnsibleRunner(host_list).get_variables(host)